From 2681078a3629b177127e0c611f0e583ba0bfa981 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Wed, 18 Nov 2026 02:38:07 +0000 Subject: [PATCH 1/2] fix(s3): abort unknown-length multipart upload when the request body fails or the putObject future is cancelled --- .../next-release/bugfix-AmazonS3-d9e250c.json | 6 + ...ntentLengthAsyncRequestBodySubscriber.java | 134 ++++++--- ...3MultipartClientPutObjectWiremockTest.java | 94 +++++- ...yncRequestBodySubscriberLifecycleTest.java | 284 ++++++++++++++++++ ...ngthAsyncRequestBodySubscriberTckTest.java | 114 +++++++ .../utils/ControlledSubscription.java | 6 + 6 files changed, 591 insertions(+), 47 deletions(-) create mode 100644 .changes/next-release/bugfix-AmazonS3-d9e250c.json create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java diff --git a/.changes/next-release/bugfix-AmazonS3-d9e250c.json b/.changes/next-release/bugfix-AmazonS3-d9e250c.json new file mode 100644 index 000000000000..fef3d00a4c93 --- /dev/null +++ b/.changes/next-release/bugfix-AmazonS3-d9e250c.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "Amazon S3", + "contributor": "", + "description": "Fixed `putObject` with an unknown content length on a multipart-enabled `S3AsyncClient` so that a failed request body no longer leaves a partial object in S3 and a cancel between parts aborts the multipart upload." +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java index 40ed3d7ecabb..d94370fdb523 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -43,6 +44,7 @@ import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Pair; +import software.amazon.awssdk.utils.Validate; @SdkInternalApi public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscriber { @@ -58,18 +60,11 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe */ private final AtomicBoolean createMultipartUploadInitiated = new AtomicBoolean(false); - /** - * Indicates whether CompleteMultipart has been initiated or not. - */ - private final AtomicBoolean completedMultipartInitiated = new AtomicBoolean(false); - /** * The number of AsyncRequestBody has been received but yet to be processed */ private final AtomicInteger asyncRequestBodyInFlight = new AtomicInteger(0); - private final AtomicBoolean failureActionInitiated = new AtomicBoolean(false); - private final AtomicInteger partNumber = new AtomicInteger(0); private final AtomicLong contentLength = new AtomicLong(0); @@ -89,8 +84,9 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe private final Object subscriptionLock = new Object(); private Subscription subscription; private CloseableAsyncRequestBody firstRequestBody; - private String uploadId; + private volatile String uploadId; private volatile boolean isDone; + private final AtomicReference state = new AtomicReference<>(State.RUNNING); UnknownContentLengthAsyncRequestBodySubscriber( long partSizeInBytes, @@ -112,9 +108,10 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe @Override public void onSubscribe(Subscription s) { + Validate.paramNotNull(s, "subscription"); if (this.subscription != null) { log.warn(() -> "The subscriber has already been subscribed. Cancelling the incoming subscription"); - subscription.cancel(); + s.cancel(); return; } this.subscription = s; @@ -122,7 +119,10 @@ public void onSubscribe(Subscription s) { returnFuture.whenComplete((r, t) -> { if (t != null) { s.cancel(); - MultipartUploadHelper.cancelingOtherOngoingRequests(futures, t); + if (tryMarkFailing()) { + MultipartUploadHelper.cancelingOtherOngoingRequests(futures, t); + abortWhenUploadIdKnown(); + } } }); } @@ -131,11 +131,14 @@ public void onSubscribe(Subscription s) { public void onNext(CloseableAsyncRequestBody asyncRequestBody) { if (asyncRequestBody == null) { NullPointerException exception = new NullPointerException("asyncRequestBody passed to onNext MUST NOT be null."); - multipartUploadHelper.failRequestsElegantly(futures, exception, uploadId, returnFuture, putObjectRequest); + if (tryMarkFailing()) { + failUpload(exception); + } throw exception; } - if (isDone) { + if (state.get() != State.RUNNING || isDone) { + discard(asyncRequestBody); return; } @@ -145,8 +148,10 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { Optional sdkClientException = validatePart(asyncRequestBody, currentPartNum); if (sdkClientException.isPresent()) { - multipartUploadHelper.failRequestsElegantly( - futures, sdkClientException.get(), uploadId, returnFuture, putObjectRequest); + if (tryMarkFailing()) { + failUpload(sdkClientException.get()); + } + discard(asyncRequestBody); subscription.cancel(); return; } @@ -169,15 +174,27 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> { if (throwable != null) { - genericMultipartHelper.handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable); + uploadIdFuture.completeExceptionally(throwable); + if (tryMarkFailing()) { + genericMultipartHelper.handleException(returnFuture, () -> "Failed to initiate multipart upload", + throwable); + } subscription.cancel(); + discard(firstRequestBody); + discard(asyncRequestBody); } else { uploadId = createMultipartUploadResponse.uploadId(); log.debug(() -> "Initiated a new multipart upload, uploadId: " + uploadId); + // Complete uploadIdFuture before checking state: a failure's abort waits on it, even when we return below. + uploadIdFuture.complete(uploadId); + if (state.get() != State.RUNNING) { + discard(firstRequestBody); + discard(asyncRequestBody); + return; + } sendUploadPartRequest(uploadId, firstRequestBody, 1); sendUploadPartRequest(uploadId, asyncRequestBody, 2); - uploadIdFuture.complete(uploadId); // 2 parts already in flight, request the rest of our max in flight int additionalDemand = maxInFlightParts - 2; @@ -191,6 +208,10 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture); } else { uploadIdFuture.whenComplete((r, t) -> { + if (t != null) { + discard(asyncRequestBody); + return; + } sendUploadPartRequest(uploadId, asyncRequestBody, currentPartNum); }); } @@ -212,29 +233,36 @@ private Optional validatePart(AsyncRequestBody asyncRequestB private void sendUploadPartRequest(String uploadId, CloseableAsyncRequestBody asyncRequestBody, int currentPartNum) { + if (state.get() != State.RUNNING) { + discard(asyncRequestBody); + return; + } Long contentLengthCurrentPart = asyncRequestBody.contentLength().get(); this.contentLength.getAndAdd(contentLengthCurrentPart); - multipartUploadHelper - .sendIndividualUploadPartRequest(uploadId, completedParts::add, futures, - uploadPart(asyncRequestBody, currentPartNum), progressListener) - .whenComplete((r, t) -> { - asyncRequestBody.close(); - if (t != null) { - if (failureActionInitiated.compareAndSet(false, true)) { - multipartUploadHelper.failRequestsElegantly(futures, t, uploadId, returnFuture, - putObjectRequest); - } - } else { - int inFlight = asyncRequestBodyInFlight.decrementAndGet(); - if (!isDone && inFlight < maxInFlightParts) { - synchronized (subscriptionLock) { - subscription.request(1); - } + CompletableFuture partFuture = multipartUploadHelper.sendIndividualUploadPartRequest( + uploadId, completedParts::add, futures, uploadPart(asyncRequestBody, currentPartNum), progressListener); + partFuture.whenComplete((r, t) -> { + asyncRequestBody.close(); + if (t != null) { + if (tryMarkFailing()) { + failUpload(t); + } + } else { + int inFlight = asyncRequestBodyInFlight.decrementAndGet(); + if (!isDone && inFlight < maxInFlightParts) { + synchronized (subscriptionLock) { + subscription.request(1); } - completeMultipartUploadIfFinish(); } - }); + completeMultipartUploadIfFinish(); + } + }); + + // A failure that ran before this part's future was registered could not cancel it. + if (state.get() != State.RUNNING) { + partFuture.cancel(true); + } } private Pair uploadPart(AsyncRequestBody asyncRequestBody, int partNum) { @@ -248,9 +276,9 @@ private Pair uploadPart(AsyncRequestBody as @Override public void onError(Throwable t) { log.debug(() -> "Received onError() ", t); - if (failureActionInitiated.compareAndSet(false, true)) { + if (tryMarkFailing()) { isDone = true; - multipartUploadHelper.failRequestsElegantly(futures, t, uploadId, returnFuture, putObjectRequest); + failUpload(t); } } @@ -272,7 +300,7 @@ public void onComplete() { private void completeMultipartUploadIfFinish() { // All atomics including asyncRequestBodyInFlight MUST be re-read here rather than passed in by the caller. - if (isDone && asyncRequestBodyInFlight.get() == 0 && completedMultipartInitiated.compareAndSet(false, true)) { + if (isDone && asyncRequestBodyInFlight.get() == 0 && state.compareAndSet(State.RUNNING, State.COMPLETING)) { CompletedPart[] parts = completedParts.stream() .sorted(Comparator.comparingInt(CompletedPart::partNumber)) .toArray(CompletedPart[]::new); @@ -285,11 +313,41 @@ private void completeMultipartUploadIfFinish() { "The number of UploadParts requests is not equal to the expected number of parts. " + "Expected: %d, Actual: %d", expectedNumParts, parts.length)); - multipartUploadHelper.failRequestsElegantly(futures, exception, uploadId, returnFuture, putObjectRequest); + state.set(State.FAILING); + failUpload(exception); return; } multipartUploadHelper.completeMultipartUpload(returnFuture, uploadId, parts, putObjectRequest, totalLength); } } + + private boolean tryMarkFailing() { + return state.compareAndSet(State.RUNNING, State.FAILING); + } + + /** + * Must only be called by the caller that moved the state to {@link State#FAILING}. The upload id may not be known yet, + * so the abort waits for it. + */ + private void failUpload(Throwable t) { + multipartUploadHelper.failRequestsElegantly(futures, t, null, returnFuture, putObjectRequest); + abortWhenUploadIdKnown(); + } + + private void abortWhenUploadIdKnown() { + uploadIdFuture.thenAccept(id -> genericMultipartHelper.cleanUpParts( + id, SdkPojoConversionUtils.toAbortMultipartUploadRequest(putObjectRequest))); + } + + private static void discard(CloseableAsyncRequestBody asyncRequestBody) { + asyncRequestBody.subscribe(new CancelledSubscriber<>()); + asyncRequestBody.close(); + } + + private enum State { + RUNNING, + COMPLETING, + FAILING + } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java index 1771b508dca2..c70f54ef3d90 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java @@ -19,10 +19,13 @@ import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; import static com.github.tomakehurst.wiremock.client.WireMock.containing; import static com.github.tomakehurst.wiremock.client.WireMock.delete; +import static com.github.tomakehurst.wiremock.client.WireMock.deleteRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.lessThanOrExactly; import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.moreThan; import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.put; import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; @@ -46,6 +49,11 @@ import java.util.Optional; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -62,13 +70,22 @@ import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; +import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; import software.amazon.awssdk.core.async.BufferedSplittableAsyncRequestBody; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.exception.NonRetryableException; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.utils.CancellableOutputStream; @WireMockTest @Timeout(120) @@ -98,15 +115,18 @@ public static Stream retryableErrorTestCase() { @BeforeEach public void setup(WireMockRuntimeInfo wiremock) { - s3AsyncClient = S3AsyncClient.builder() - .region(Region.US_EAST_1) - .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) - .credentialsProvider( - StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret"))) - .multipartEnabled(true) - .multipartConfiguration(b -> b.minimumPartSizeInBytes(10L).apiCallBufferSizeInBytes(20L)) - .httpClientBuilder(NettyNioAsyncHttpClient.builder()) - .build(); + s3AsyncClient = clientBuilder(wiremock).build(); + } + + private static S3AsyncClientBuilder clientBuilder(WireMockRuntimeInfo wiremock) { + return S3AsyncClient.builder() + .region(Region.US_EAST_1) + .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret"))) + .multipartEnabled(true) + .multipartConfiguration(b -> b.minimumPartSizeInBytes(10L).apiCallBufferSizeInBytes(20L)) + .httpClientBuilder(NettyNioAsyncHttpClient.builder()); } private void stubPutObject404Calls() { @@ -133,6 +153,62 @@ void uploadWithUnknownContentLength_onePartFails_shouldCancelUpstream() { assertThatThrownBy(() -> putObjectResponse.join()).hasRootCauseInstanceOf(S3Exception.class); } + @Test + void uploadWithUnknownContentLength_whenCancelledAfterAllPartsAnswered_shouldAbortMultipartUpload( + WireMockRuntimeInfo wiremock) throws Exception { + stubFor(post(anyUrl()).willReturn(aResponse().withStatus(200).withBody(CREATE_MULTIPART_PAYLOAD))); + stubFor(put(anyUrl()).willReturn(aResponse().withStatus(200))); + stubFor(delete(anyUrl()).willReturn(aResponse().withStatus(204))); + + ThreadLocal partAnsweredInThisTask = ThreadLocal.withInitial(() -> false); + CountDownLatch partsCompleted = new CountDownLatch(2); + CountDownLatch abortAnswered = new CountDownLatch(1); + ExecutionInterceptor interceptor = new ExecutionInterceptor() { + @Override + public void afterExecution(Context.AfterExecution context, ExecutionAttributes executionAttributes) { + if (context.request() instanceof UploadPartRequest) { + partAnsweredInThisTask.set(true); + } else if (context.request() instanceof AbortMultipartUploadRequest) { + abortAnswered.countDown(); + } + } + }; + // afterExecution runs before the UploadPart future completes. Count a part only after the completion task that + // ran its afterExecution has returned, so the multipart client has handled the response before the cancel. + ExecutorService completionThreads = Executors.newFixedThreadPool(2); + Executor completionExecutor = task -> completionThreads.execute(() -> { + task.run(); + if (partAnsweredInThisTask.get()) { + partAnsweredInThisTask.set(false); + partsCompleted.countDown(); + } + }); + + try (S3AsyncClient client = clientBuilder(wiremock) + .overrideConfiguration(o -> o.addExecutionInterceptor(interceptor)) + .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + completionExecutor)) + .build()) { + BlockingOutputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingOutputStream(null); + CompletableFuture upload = client.putObject(r -> r.bucket(BUCKET).key(KEY), body); + CancellableOutputStream outputStream = body.outputStream(); + try { + outputStream.write(new byte[20]); + assertThat(partsCompleted.await(10, TimeUnit.SECONDS)).isTrue(); + + upload.cancel(true); + + assertThat(abortAnswered.await(10, TimeUnit.SECONDS)).isTrue(); + verify(1, deleteRequestedFor(anyUrl()).withQueryParam("uploadId", equalTo("string"))); + verify(0, postRequestedFor(anyUrl()).withQueryParam("uploadId", equalTo("string"))); + } finally { + outputStream.cancel(); + } + } finally { + completionThreads.shutdownNow(); + } + } + @Test void uploadWithKnownContentLength_onePartFails_shouldCancelUpstream() { stubPutObject404Calls(); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java new file mode 100644 index 000000000000..2f2cc0cb6305 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java @@ -0,0 +1,284 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.IntConsumer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.CloseableAsyncRequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.multipart.utils.ControlledSubscription; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +@Timeout(10) +class UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest { + private static final long PART_SIZE = 1024; + private static final String UPLOAD_ID = "upload-id"; + private static final RuntimeException BODY_ERROR = new RuntimeException("body failed"); + private static final Consumer NO_EXTRA_CHECK = f -> { }; + private static final Consumer PART_1_CANCELLED = f -> assertThat(f.parts.get(1)).isCancelled(); + + enum FailurePoint { + CMPU_RESPONSE_PENDING_IN_CALLBACK(f -> { + f.deliverParts(2); + f.subscriber.onError(BODY_ERROR); + f.completeCreateMultipartUpload(); + f.completeSentParts(); + }), + BEFORE_PARTS_REGISTERED(f -> { + f.deliverParts(2); + f.onUploadPart = partNumber -> { + if (partNumber == 1) { + f.subscriber.onError(BODY_ERROR); + } + }; + f.completeCreateMultipartUpload(); + f.completeSentParts(); + }, PART_1_CANCELLED), + WHILE_PART_RESPONSE_ARRIVES(f -> { + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.onReturnFutureFailing = f::completeSentParts; + f.subscriber.onError(BODY_ERROR); + }), + PARTS_IN_FLIGHT(f -> { + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.subscriber.onError(BODY_ERROR); + f.completeSentParts(); + }), + PART_FAILS(f -> { + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.parts.get(2).completeExceptionally(S3Exception.builder().message("part failed").build()); + }); + + private final Consumer scenario; + private final Consumer extraCheck; + + FailurePoint(Consumer scenario) { + this(scenario, NO_EXTRA_CHECK); + } + + FailurePoint(Consumer scenario, Consumer extraCheck) { + this.scenario = scenario; + this.extraCheck = extraCheck; + } + } + + enum CancelPoint { + BETWEEN_CMPU_RESPONSE_AND_PARTS(f -> { + f.deliverParts(2); + f.onUploadPart = partNumber -> { + if (partNumber == 1) { + f.returnFuture.cancel(true); + } + }; + f.completeCreateMultipartUpload(); + f.completeSentParts(); + }, PART_1_CANCELLED), + PART_IN_FLIGHT(f -> { + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.completePart(1); + f.returnFuture.cancel(true); + }), + ALL_PARTS_ANSWERED(f -> { + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.completeSentParts(); + f.returnFuture.cancel(true); + }); + + private final Consumer scenario; + private final Consumer extraCheck; + + CancelPoint(Consumer scenario) { + this(scenario, NO_EXTRA_CHECK); + } + + CancelPoint(Consumer scenario, Consumer extraCheck) { + this.scenario = scenario; + this.extraCheck = extraCheck; + } + } + + @ParameterizedTest(name = "{0}") + @EnumSource(FailurePoint.class) + void onError_whenBodyFailsAfterCreateMultipartUpload_shouldAbortOnceAndNeverComplete(FailurePoint failurePoint) { + Fixture f = new Fixture(); + failurePoint.scenario.accept(f); + assertAbortedOnceAndNeverCompleted(f); + failurePoint.extraCheck.accept(f); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(CancelPoint.class) + void cancel_whenMultipartUploadStarted_shouldAbortOnceAndNeverComplete(CancelPoint cancelPoint) { + Fixture f = new Fixture(); + cancelPoint.scenario.accept(f); + assertAbortedOnceAndNeverCompleted(f); + assertThat(f.subscription.isCancelled()).isTrue(); + cancelPoint.extraCheck.accept(f); + } + + @Test + void cancel_whenOnlyFirstPartBuffered_shouldNotCallMultipartApis() { + Fixture f = new Fixture(); + f.deliverParts(1); + f.returnFuture.cancel(true); + + verify(f.s3, never()).createMultipartUpload(any(CreateMultipartUploadRequest.class)); + verify(f.s3, never()).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + verify(f.s3, never()).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + } + + @Test + void onComplete_whenSinglePartUploadFails_shouldLeaveExceptionUnchanged() { + Fixture f = new Fixture(); + CompletableFuture putObject = new CompletableFuture<>(); + when(f.s3.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))).thenReturn(putObject); + S3Exception accessDenied = (S3Exception) S3Exception.builder().message("Access Denied").build(); + + f.deliverParts(1); + f.subscriber.onComplete(); + putObject.completeExceptionally(accessDenied); + + assertThat(f.returnFuture).isCompletedExceptionally(); + assertThat(accessDenied.getSuppressed()).isEmpty(); + } + + private static void assertAbortedOnceAndNeverCompleted(Fixture f) { + assertThat(f.returnFuture).isCompletedExceptionally(); + verify(f.s3, never()).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + ArgumentCaptor abort = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class); + verify(f.s3, times(1)).abortMultipartUpload(abort.capture()); + assertThat(abort.getValue().uploadId()).isEqualTo(UPLOAD_ID); + assertThat(f.partsStartedAfterFailure).hasValue(0); + f.bodies.forEach(body -> verify(body).close()); + } + + private static final class Fixture { + private final S3AsyncClient s3 = mock(S3AsyncClient.class); + private final CompletableFuture createMultipartUploadFuture = + new RespondedCreateMultipartUploadFuture(); + private final Map> parts = new ConcurrentHashMap<>(); + private final List bodies = new ArrayList<>(); + private final AtomicInteger partsStartedAfterFailure = new AtomicInteger(); + private IntConsumer onUploadPart = partNumber -> { }; + private Runnable onReturnFutureFailing = () -> { }; + + private final CompletableFuture returnFuture = new CompletableFuture() { + @Override + public boolean completeExceptionally(Throwable ex) { + onReturnFutureFailing.run(); + return super.completeExceptionally(ex); + } + }; + private final UnknownContentLengthAsyncRequestBodySubscriber subscriber; + private final ControlledSubscription subscription; + + Fixture() { + when(s3.createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn(createMultipartUploadFuture); + when(s3.uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class))).thenAnswer(invocation -> { + if (returnFuture.isDone()) { + partsStartedAfterFailure.incrementAndGet(); + } + int partNumber = invocation.getArgument(0, UploadPartRequest.class).partNumber(); + CompletableFuture part = new CompletableFuture<>(); + parts.put(partNumber, part); + onUploadPart.accept(partNumber); + return part; + }); + when(s3.abortMultipartUpload(any(AbortMultipartUploadRequest.class))) + .thenReturn(completedFuture(AbortMultipartUploadResponse.builder().build())); + when(s3.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) + .thenReturn(completedFuture(CompleteMultipartUploadResponse.builder().build())); + + PutObjectRequest putObjectRequest = PutObjectRequest.builder().bucket("bucket").key("key").build(); + subscriber = new UnknownContentLengthAsyncRequestBodySubscriber( + PART_SIZE, putObjectRequest, returnFuture, + new MultipartUploadHelper(s3, PART_SIZE, Long.MAX_VALUE), + new GenericMultipartHelper<>(s3, SdkPojoConversionUtils::toAbortMultipartUploadRequest, + SdkPojoConversionUtils::toPutObjectResponse), + 4); + subscription = new ControlledSubscription(subscriber); + subscriber.onSubscribe(subscription); + } + + void deliverParts(int count) { + for (int i = 0; i < count; i++) { + CloseableAsyncRequestBody body = mock(CloseableAsyncRequestBody.class); + when(body.contentLength()).thenReturn(Optional.of(PART_SIZE)); + bodies.add(body); + subscription.enqueueBodyAndDeliver(body); + } + } + + void completeCreateMultipartUpload() { + createMultipartUploadFuture.complete(CreateMultipartUploadResponse.builder().uploadId(UPLOAD_ID).build()); + } + + void completePart(int partNumber) { + parts.get(partNumber).complete(UploadPartResponse.builder().eTag("etag-" + partNumber).build()); + } + + void completeSentParts() { + parts.keySet().forEach(this::completePart); + } + } + + /** + * Models a CreateMultipartUpload whose response has already arrived, so a failure forwarded to it is ignored. + */ + private static final class RespondedCreateMultipartUploadFuture extends CompletableFuture { + @Override + public boolean completeExceptionally(Throwable ex) { + return false; + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java new file mode 100644 index 000000000000..88776a4eff37 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java @@ -0,0 +1,114 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.reactivestreams.tck.SubscriberWhiteboxVerification; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.async.CloseableAsyncRequestBody; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +public class UnknownContentLengthAsyncRequestBodySubscriberTckTest + extends SubscriberWhiteboxVerification { + private static final long PART_SIZE = 1024; + + private final MultipartUploadHelper multipartUploadHelper; + private final GenericMultipartHelper genericMultipartHelper; + + @SuppressWarnings("unchecked") + public UnknownContentLengthAsyncRequestBodySubscriberTckTest() { + super(new TestEnvironment()); + multipartUploadHelper = mock(MultipartUploadHelper.class); + genericMultipartHelper = mock(GenericMultipartHelper.class); + when(multipartUploadHelper.createMultipartUpload(any(), any())) + .thenAnswer(i -> CompletableFuture.completedFuture(CreateMultipartUploadResponse.builder() + .uploadId("uploadId") + .build())); + when(multipartUploadHelper.sendIndividualUploadPartRequest(any(), any(), any(), any(), any())) + .thenReturn(CompletableFuture.completedFuture(CompletedPart.builder().build())); + } + + @Override + public Subscriber createSubscriber(WhiteboxSubscriberProbe probe) { + PutObjectRequest putObjectRequest = PutObjectRequest.builder().bucket("bucket").key("key").build(); + + return new UnknownContentLengthAsyncRequestBodySubscriber(PART_SIZE, putObjectRequest, new CompletableFuture<>(), + multipartUploadHelper, genericMultipartHelper, 50) { + @Override + public void onSubscribe(Subscription s) { + super.onSubscribe(s); + probe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long elements) { + s.request(elements); + } + + @Override + public void signalCancel() { + s.cancel(); + } + }); + } + + @Override + public void onNext(CloseableAsyncRequestBody item) { + super.onNext(item); + probe.registerOnNext(item); + } + + @Override + public void onError(Throwable t) { + super.onError(t); + probe.registerOnError(t); + } + + @Override + public void onComplete() { + super.onComplete(); + probe.registerOnComplete(); + } + }; + } + + @Override + public CloseableAsyncRequestBody createElement(int element) { + return new CloseableAsyncRequestBody() { + @Override + public Optional contentLength() { + return Optional.of(PART_SIZE); + } + + @Override + public void subscribe(Subscriber s) { + } + + @Override + public void close() { + } + }; + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/utils/ControlledSubscription.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/utils/ControlledSubscription.java index 2e4ade734aea..9becf076a8ec 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/utils/ControlledSubscription.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/utils/ControlledSubscription.java @@ -33,6 +33,7 @@ public final class ControlledSubscription implements Subscription { private boolean streamComplete; private boolean onCompleteDelivered; private boolean draining; + private volatile boolean cancelled; public ControlledSubscription(Subscriber subscriber) { this.subscriber = subscriber; @@ -46,6 +47,11 @@ public void request(long n) { @Override public void cancel() { + cancelled = true; + } + + public boolean isCancelled() { + return cancelled; } public void enqueueBodyAndDeliver(CloseableAsyncRequestBody body) { From c5cd326e8344261cc2b690c24eb549564915c4c2 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Fri, 20 Nov 2026 19:43:49 -0800 Subject: [PATCH 2/2] Handled Review comments --- .../next-release/bugfix-AmazonS3-d9e250c.json | 2 +- .../multipart/MultipartUploadHelper.java | 10 +- ...ntentLengthAsyncRequestBodySubscriber.java | 10 +- .../UploadWithKnownContentLengthHelper.java | 4 +- ...3MultipartClientPutObjectWiremockTest.java | 105 ++++++++++++++++++ ...yncRequestBodySubscriberLifecycleTest.java | 72 +++++++++--- ...ngthAsyncRequestBodySubscriberTckTest.java | 2 +- ...tLengthAsyncRequestBodySubscriberTest.java | 2 +- 8 files changed, 178 insertions(+), 29 deletions(-) diff --git a/.changes/next-release/bugfix-AmazonS3-d9e250c.json b/.changes/next-release/bugfix-AmazonS3-d9e250c.json index fef3d00a4c93..8fe5f53ba7a2 100644 --- a/.changes/next-release/bugfix-AmazonS3-d9e250c.json +++ b/.changes/next-release/bugfix-AmazonS3-d9e250c.json @@ -2,5 +2,5 @@ "type": "bugfix", "category": "Amazon S3", "contributor": "", - "description": "Fixed `putObject` with an unknown content length on a multipart-enabled `S3AsyncClient` so that a failed request body no longer leaves a partial object in S3 and a cancel between parts aborts the multipart upload." + "description": "Fixed `putObject` with an unknown content length on a multipart-enabled `S3AsyncClient` so that a failed request body or a cancelled `putObject` future aborts the multipart upload, instead of completing a partial object or leaving the upload open." } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index c98e34aae0b2..2c47c6eb0656 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -66,15 +66,9 @@ public MultipartUploadHelper(S3AsyncClient s3AsyncClient, this.multipartUploadThresholdInBytes = multipartUploadThresholdInBytes; } - CompletableFuture createMultipartUpload(PutObjectRequest putObjectRequest, - CompletableFuture returnFuture) { + CompletableFuture createMultipartUpload(PutObjectRequest putObjectRequest) { CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); - CompletableFuture createMultipartUploadFuture = - s3AsyncClient.createMultipartUpload(request); - - // Ensure cancellations are forwarded to the createMultipartUploadFuture future - CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture); - return createMultipartUploadFuture; + return s3AsyncClient.createMultipartUpload(request); } CompletableFuture completeMultipartUpload(CompletableFuture returnFuture, diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java index d94370fdb523..6251da49465d 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriber.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.Optional; import java.util.Queue; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,7 +42,6 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.UploadPartRequest; -import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Pair; import software.amazon.awssdk.utils.Validate; @@ -122,6 +122,9 @@ public void onSubscribe(Subscription s) { if (tryMarkFailing()) { MultipartUploadHelper.cancelingOtherOngoingRequests(futures, t); abortWhenUploadIdKnown(); + } else if (state.get() == State.COMPLETING && t instanceof CancellationException) { + // GenericMultipartHelper already aborts a failed CompleteMultipartUpload, so abort here only on cancel. + abortWhenUploadIdKnown(); } } }); @@ -170,7 +173,7 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { if (createMultipartUploadInitiated.compareAndSet(false, true)) { log.debug(() -> "Starting the upload as multipart upload request"); CompletableFuture createMultipartUploadFuture = multipartUploadHelper - .createMultipartUpload(putObjectRequest, returnFuture); + .createMultipartUpload(putObjectRequest); createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> { if (throwable != null) { @@ -185,7 +188,7 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { } else { uploadId = createMultipartUploadResponse.uploadId(); log.debug(() -> "Initiated a new multipart upload, uploadId: " + uploadId); - // Complete uploadIdFuture before checking state: a failure's abort waits on it, even when we return below. + // Complete uploadIdFuture before checking state. A failure's abort waits on it, even when we return below. uploadIdFuture.complete(uploadId); if (state.get() != State.RUNNING) { discard(firstRequestBody); @@ -205,7 +208,6 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) { } } }); - CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture); } else { uploadIdFuture.whenComplete((r, t) -> { if (t != null) { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java index 9ce0db889a2d..532908d43ccf 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java @@ -106,7 +106,9 @@ private void uploadInParts(PutObjectRequest putObjectRequest, long contentLength private void initiateNewUpload(PutObjectRequest putObjectRequest, long contentLength, AsyncRequestBody asyncRequestBody, CompletableFuture returnFuture) { CompletableFuture createMultipartUploadFuture = - multipartUploadHelper.createMultipartUpload(putObjectRequest, returnFuture); + multipartUploadHelper.createMultipartUpload(putObjectRequest); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture); createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> { if (throwable != null) { diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java index c70f54ef3d90..ff5b110fa94e 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientPutObjectWiremockTest.java @@ -21,6 +21,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.delete; import static com.github.tomakehurst.wiremock.client.WireMock.deleteRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.findAll; import static com.github.tomakehurst.wiremock.client.WireMock.lessThanOrExactly; import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.moreThan; @@ -38,6 +39,7 @@ import com.github.tomakehurst.wiremock.http.Fault; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; import com.github.tomakehurst.wiremock.stubbing.Scenario; import io.reactivex.rxjava3.core.Flowable; import java.io.InputStream; @@ -82,6 +84,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -98,6 +101,10 @@ public class S3MultipartClientPutObjectWiremockTest { + " string\n" + " string\n" + ""; + private static final String ACCESS_DENIED_ERROR_PAYLOAD = "\n" + + " AccessDenied\n" + + " Access Denied\n" + + ""; private S3AsyncClient s3AsyncClient; public static Stream retryableErrorTestCase() { @@ -209,6 +216,104 @@ public void afterExecution(Context.AfterExecution context, ExecutionAttributes e } } + private enum StopTrigger { + CANCEL_FUTURE { + @Override + void stop(CompletableFuture upload, CancellableOutputStream outputStream) { + upload.cancel(true); + } + }, + FAIL_BODY { + @Override + void stop(CompletableFuture upload, CancellableOutputStream outputStream) { + outputStream.cancel(); + } + }; + + abstract void stop(CompletableFuture upload, CancellableOutputStream outputStream); + } + + private static Stream stopBeforeCreateMultipartUploadResponds() { + return Stream.of(StopTrigger.values()).flatMap(trigger -> Stream.of( + Arguments.of(trigger, "CreateMultipartUpload succeeds, upload id known", + aResponse().withStatus(200).withBody(CREATE_MULTIPART_PAYLOAD), 1), + Arguments.of(trigger, "CreateMultipartUpload fails, no upload id", + aResponse().withStatus(403).withBody(ACCESS_DENIED_ERROR_PAYLOAD), 0))); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("stopBeforeCreateMultipartUploadResponds") + void uploadWithUnknownContentLength_whenStoppedBeforeCreateMultipartUploadResponds_shouldAbortOnlyIfUploadIdKnown( + StopTrigger trigger, String scenario, ResponseDefinitionBuilder createMultipartUploadResponse, int expectedAborts, + WireMockRuntimeInfo wiremock) throws Exception { + stubFor(post(anyUrl()).willReturn(createMultipartUploadResponse.withFixedDelay(2000))); + stubFor(put(anyUrl()).willReturn(aResponse().withStatus(200))); + stubFor(delete(anyUrl()).willReturn(aResponse().withStatus(204))); + + ThreadLocal createAnsweredInThisTask = ThreadLocal.withInitial(() -> false); + CountDownLatch createHandled = new CountDownLatch(1); + ExecutionInterceptor interceptor = new ExecutionInterceptor() { + @Override + public void afterExecution(Context.AfterExecution context, ExecutionAttributes executionAttributes) { + if (context.request() instanceof CreateMultipartUploadRequest) { + createAnsweredInThisTask.set(true); + } + } + + @Override + public void onExecutionFailure(Context.FailedExecution context, ExecutionAttributes executionAttributes) { + if (context.request() instanceof CreateMultipartUploadRequest) { + createAnsweredInThisTask.set(true); + } + } + }; + // The interceptor runs before the CreateMultipartUpload future completes. Count down only after the completion + // task has returned, so the multipart client has handled the response before the assertions run. + ExecutorService completionThreads = Executors.newFixedThreadPool(2); + Executor completionExecutor = task -> completionThreads.execute(() -> { + task.run(); + if (createAnsweredInThisTask.get()) { + createAnsweredInThisTask.set(false); + createHandled.countDown(); + } + }); + + try (S3AsyncClient client = clientBuilder(wiremock) + .overrideConfiguration(o -> o.addExecutionInterceptor(interceptor)) + .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + completionExecutor)) + .build()) { + BlockingOutputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingOutputStream(null); + CompletableFuture upload = client.putObject(r -> r.bucket(BUCKET).key(KEY), body); + CancellableOutputStream outputStream = body.outputStream(); + try { + outputStream.write(new byte[20]); + awaitRequests(1, postRequestedFor(urlEqualTo("/Example-Bucket/Example-Object?uploads"))); + + trigger.stop(upload, outputStream); + + assertThat(createHandled.await(10, TimeUnit.SECONDS)).as("CreateMultipartUpload response handled").isTrue(); + assertThat(upload).isCompletedExceptionally(); + awaitRequests(expectedAborts, deleteRequestedFor(anyUrl()).withQueryParam("uploadId", equalTo("string"))); + verify(1, postRequestedFor(urlEqualTo("/Example-Bucket/Example-Object?uploads"))); + verify(0, putRequestedFor(anyUrl()).withQueryParam("partNumber", matching(".*"))); + verify(0, postRequestedFor(anyUrl()).withQueryParam("uploadId", equalTo("string"))); + } finally { + outputStream.cancel(); + } + } finally { + completionThreads.shutdownNow(); + } + } + + private static void awaitRequests(int count, RequestPatternBuilder pattern) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (findAll(pattern).size() < count && System.nanoTime() < deadline) { + Thread.sleep(10); + } + verify(count, pattern); + } + @Test void uploadWithKnownContentLength_onePartFails_shouldCancelUpstream() { stubPutObject404Calls(); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java index 2f2cc0cb6305..f32168644d85 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest.java @@ -63,7 +63,7 @@ class UnknownContentLengthAsyncRequestBodySubscriberLifecycleTest { private static final Consumer PART_1_CANCELLED = f -> assertThat(f.parts.get(1)).isCancelled(); enum FailurePoint { - CMPU_RESPONSE_PENDING_IN_CALLBACK(f -> { + CMPU_IN_FLIGHT(f -> { f.deliverParts(2); f.subscriber.onError(BODY_ERROR); f.completeCreateMultipartUpload(); @@ -111,6 +111,11 @@ enum FailurePoint { } enum CancelPoint { + CMPU_IN_FLIGHT(f -> { + f.deliverParts(2); + f.returnFuture.cancel(true); + f.completeCreateMultipartUpload(); + }), BETWEEN_CMPU_RESPONSE_AND_PARTS(f -> { f.deliverParts(2); f.onUploadPart = partNumber -> { @@ -149,7 +154,7 @@ enum CancelPoint { @ParameterizedTest(name = "{0}") @EnumSource(FailurePoint.class) - void onError_whenBodyFailsAfterCreateMultipartUpload_shouldAbortOnceAndNeverComplete(FailurePoint failurePoint) { + void onError_whenMultipartUploadStarted_shouldAbortOnceAndNeverComplete(FailurePoint failurePoint) { Fixture f = new Fixture(); failurePoint.scenario.accept(f); assertAbortedOnceAndNeverCompleted(f); @@ -177,6 +182,57 @@ void cancel_whenOnlyFirstPartBuffered_shouldNotCallMultipartApis() { verify(f.s3, never()).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); } + @Test + void cancel_whenCreateMultipartUploadFailsAfterCancel_shouldNotAbortAndShouldCloseBodies() { + Fixture f = new Fixture(); + f.deliverParts(2); + f.returnFuture.cancel(true); + assertThat(f.createMultipartUploadFuture).isNotDone(); + + f.createMultipartUploadFuture.completeExceptionally( + S3Exception.builder().message("CreateMultipartUpload failed").build()); + + assertThat(f.returnFuture).isCancelled(); + verify(f.s3, never()).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + verify(f.s3, never()).uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class)); + f.bodies.forEach(body -> verify(body).close()); + } + + @Test + void cancel_whenCompleteMultipartUploadInFlight_shouldAbort() { + Fixture f = new Fixture(); + when(f.s3.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenReturn(new CompletableFuture<>()); + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.completeSentParts(); + f.subscriber.onComplete(); + + f.returnFuture.cancel(true); + + verify(f.s3, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + ArgumentCaptor abort = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class); + verify(f.s3, times(1)).abortMultipartUpload(abort.capture()); + assertThat(abort.getValue().uploadId()).isEqualTo(UPLOAD_ID); + } + + @Test + void onComplete_whenCompleteMultipartUploadFails_shouldAbortOnce() { + Fixture f = new Fixture(); + CompletableFuture complete = new CompletableFuture<>(); + when(f.s3.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenReturn(complete); + f.deliverParts(2); + f.completeCreateMultipartUpload(); + f.completeSentParts(); + f.subscriber.onComplete(); + + complete.completeExceptionally(S3Exception.builder().message("CompleteMultipartUpload failed").build()); + + assertThat(f.returnFuture).isCompletedExceptionally(); + ArgumentCaptor abort = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class); + verify(f.s3, times(1)).abortMultipartUpload(abort.capture()); + assertThat(abort.getValue().uploadId()).isEqualTo(UPLOAD_ID); + } + @Test void onComplete_whenSinglePartUploadFails_shouldLeaveExceptionUnchanged() { Fixture f = new Fixture(); @@ -205,7 +261,7 @@ private static void assertAbortedOnceAndNeverCompleted(Fixture f) { private static final class Fixture { private final S3AsyncClient s3 = mock(S3AsyncClient.class); private final CompletableFuture createMultipartUploadFuture = - new RespondedCreateMultipartUploadFuture(); + new CompletableFuture<>(); private final Map> parts = new ConcurrentHashMap<>(); private final List bodies = new ArrayList<>(); private final AtomicInteger partsStartedAfterFailure = new AtomicInteger(); @@ -271,14 +327,4 @@ void completeSentParts() { parts.keySet().forEach(this::completePart); } } - - /** - * Models a CreateMultipartUpload whose response has already arrived, so a failure forwarded to it is ignored. - */ - private static final class RespondedCreateMultipartUploadFuture extends CompletableFuture { - @Override - public boolean completeExceptionally(Throwable ex) { - return false; - } - } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java index 88776a4eff37..64c86769d793 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTckTest.java @@ -44,7 +44,7 @@ public UnknownContentLengthAsyncRequestBodySubscriberTckTest() { super(new TestEnvironment()); multipartUploadHelper = mock(MultipartUploadHelper.class); genericMultipartHelper = mock(GenericMultipartHelper.class); - when(multipartUploadHelper.createMultipartUpload(any(), any())) + when(multipartUploadHelper.createMultipartUpload(any())) .thenAnswer(i -> CompletableFuture.completedFuture(CreateMultipartUploadResponse.builder() .uploadId("uploadId") .build())); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTest.java index 22b5a7019a2b..62643638c90b 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UnknownContentLengthAsyncRequestBodySubscriberTest.java @@ -226,7 +226,7 @@ private UnknownContentLengthAsyncRequestBodySubscriber createSubscriber(int maxI } private void stubSuccessfulCreateMultipartCall() { - when(multipartUploadHelper.createMultipartUpload(any(), any())) + when(multipartUploadHelper.createMultipartUpload(any())) .thenReturn(CompletableFuture.completedFuture( software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse.builder() .uploadId(UPLOAD_ID)