Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/next-release/bugfix-AmazonS3-d9e250c.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "bugfix",
"category": "Amazon S3",
"contributor": "",
"description": "Fixed `putObject` with an unknown content length on a multipart-enabled `S3AsyncClient` so that a failed request body or a cancelled `putObject` future aborts the multipart upload, instead of completing a partial object or leaving the upload open."
}
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,9 @@ public MultipartUploadHelper(S3AsyncClient s3AsyncClient,
this.multipartUploadThresholdInBytes = multipartUploadThresholdInBytes;
}

CompletableFuture<CreateMultipartUploadResponse> createMultipartUpload(PutObjectRequest putObjectRequest,
CompletableFuture<PutObjectResponse> returnFuture) {
CompletableFuture<CreateMultipartUploadResponse> createMultipartUpload(PutObjectRequest putObjectRequest) {
CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest);
CompletableFuture<CreateMultipartUploadResponse> createMultipartUploadFuture =
s3AsyncClient.createMultipartUpload(request);

// Ensure cancellations are forwarded to the createMultipartUploadFuture future
CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);
return createMultipartUploadFuture;
return s3AsyncClient.createMultipartUpload(request);
}

CompletableFuture<CompleteMultipartUploadResponse> completeMultipartUpload(CompletableFuture<PutObjectResponse> returnFuture,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@
import java.util.Comparator;
import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import software.amazon.awssdk.annotations.SdkInternalApi;
Expand All @@ -40,9 +42,9 @@
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.utils.CompletableFutureUtils;
import software.amazon.awssdk.utils.Logger;
import software.amazon.awssdk.utils.Pair;
import software.amazon.awssdk.utils.Validate;

@SdkInternalApi
public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscriber<CloseableAsyncRequestBody> {
Expand All @@ -58,18 +60,11 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe
*/
private final AtomicBoolean createMultipartUploadInitiated = new AtomicBoolean(false);

/**
* Indicates whether CompleteMultipart has been initiated or not.
*/
private final AtomicBoolean completedMultipartInitiated = new AtomicBoolean(false);

/**
* The number of AsyncRequestBody has been received but yet to be processed
*/
private final AtomicInteger asyncRequestBodyInFlight = new AtomicInteger(0);

private final AtomicBoolean failureActionInitiated = new AtomicBoolean(false);

private final AtomicInteger partNumber = new AtomicInteger(0);
private final AtomicLong contentLength = new AtomicLong(0);

Expand All @@ -89,8 +84,9 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe
private final Object subscriptionLock = new Object();
private Subscription subscription;
private CloseableAsyncRequestBody firstRequestBody;
private String uploadId;
private volatile String uploadId;
private volatile boolean isDone;
private final AtomicReference<State> state = new AtomicReference<>(State.RUNNING);

UnknownContentLengthAsyncRequestBodySubscriber(
long partSizeInBytes,
Expand All @@ -112,17 +108,24 @@ public class UnknownContentLengthAsyncRequestBodySubscriber implements Subscribe

@Override
public void onSubscribe(Subscription s) {
Validate.paramNotNull(s, "subscription");
if (this.subscription != null) {
log.warn(() -> "The subscriber has already been subscribed. Cancelling the incoming subscription");
subscription.cancel();
s.cancel();
return;
}
this.subscription = s;
s.request(1);
returnFuture.whenComplete((r, t) -> {
if (t != null) {
s.cancel();
MultipartUploadHelper.cancelingOtherOngoingRequests(futures, t);
if (tryMarkFailing()) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The known-length subscriber also aborts when CompleteMultipartUpload is in flight:

if (tryMarkFailing() || state.get() == State.COMPLETING) {

Here, a cancel during COMPLETING loses the CAS, so no abort is sent. CompleteMultipartUpload keeps going and can store the object at the key while the caller gets a CancellationException. The abort would race the complete, but it gives the caller the same best-effort outcome as the known-length path. Can these two match? If the difference is intended, a short comment saying why would help.

MultipartUploadHelper.cancelingOtherOngoingRequests(futures, t);
abortWhenUploadIdKnown();
} else if (state.get() == State.COMPLETING && t instanceof CancellationException) {
// GenericMultipartHelper already aborts a failed CompleteMultipartUpload, so abort here only on cancel.
abortWhenUploadIdKnown();
}
}
});
}
Expand All @@ -131,11 +134,14 @@ public void onSubscribe(Subscription s) {
public void onNext(CloseableAsyncRequestBody asyncRequestBody) {
if (asyncRequestBody == null) {
NullPointerException exception = new NullPointerException("asyncRequestBody passed to onNext MUST NOT be null.");
multipartUploadHelper.failRequestsElegantly(futures, exception, uploadId, returnFuture, putObjectRequest);
if (tryMarkFailing()) {
failUpload(exception);
}
throw exception;
}

if (isDone) {
if (state.get() != State.RUNNING || isDone) {
discard(asyncRequestBody);
return;
}

Expand All @@ -145,8 +151,10 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) {

Optional<SdkClientException> sdkClientException = validatePart(asyncRequestBody, currentPartNum);
if (sdkClientException.isPresent()) {
multipartUploadHelper.failRequestsElegantly(
futures, sdkClientException.get(), uploadId, returnFuture, putObjectRequest);
if (tryMarkFailing()) {
failUpload(sdkClientException.get());
}
discard(asyncRequestBody);
subscription.cancel();
return;
}
Expand All @@ -165,19 +173,31 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) {
if (createMultipartUploadInitiated.compareAndSet(false, true)) {
log.debug(() -> "Starting the upload as multipart upload request");
CompletableFuture<CreateMultipartUploadResponse> createMultipartUploadFuture = multipartUploadHelper
.createMultipartUpload(putObjectRequest, returnFuture);
.createMultipartUpload(putObjectRequest);

createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> {
if (throwable != null) {
genericMultipartHelper.handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable);
uploadIdFuture.completeExceptionally(throwable);
if (tryMarkFailing()) {
genericMultipartHelper.handleException(returnFuture, () -> "Failed to initiate multipart upload",
throwable);
}
subscription.cancel();
discard(firstRequestBody);
discard(asyncRequestBody);
} else {
uploadId = createMultipartUploadResponse.uploadId();
log.debug(() -> "Initiated a new multipart upload, uploadId: " + uploadId);
// Complete uploadIdFuture before checking state. A failure's abort waits on it, even when we return below.
uploadIdFuture.complete(uploadId);
if (state.get() != State.RUNNING) {
discard(firstRequestBody);
discard(asyncRequestBody);
return;
}

sendUploadPartRequest(uploadId, firstRequestBody, 1);
sendUploadPartRequest(uploadId, asyncRequestBody, 2);
uploadIdFuture.complete(uploadId);

// 2 parts already in flight, request the rest of our max in flight
int additionalDemand = maxInFlightParts - 2;
Expand All @@ -188,9 +208,12 @@ public void onNext(CloseableAsyncRequestBody asyncRequestBody) {
}
}
});
CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);
} else {
uploadIdFuture.whenComplete((r, t) -> {
if (t != null) {
discard(asyncRequestBody);
return;
}
sendUploadPartRequest(uploadId, asyncRequestBody, currentPartNum);
});
}
Expand All @@ -212,29 +235,36 @@ private Optional<SdkClientException> validatePart(AsyncRequestBody asyncRequestB
private void sendUploadPartRequest(String uploadId,
CloseableAsyncRequestBody asyncRequestBody,
int currentPartNum) {
if (state.get() != State.RUNNING) {
discard(asyncRequestBody);
return;
}
Long contentLengthCurrentPart = asyncRequestBody.contentLength().get();
this.contentLength.getAndAdd(contentLengthCurrentPart);

multipartUploadHelper
.sendIndividualUploadPartRequest(uploadId, completedParts::add, futures,
uploadPart(asyncRequestBody, currentPartNum), progressListener)
.whenComplete((r, t) -> {
asyncRequestBody.close();
if (t != null) {
if (failureActionInitiated.compareAndSet(false, true)) {
multipartUploadHelper.failRequestsElegantly(futures, t, uploadId, returnFuture,
putObjectRequest);
}
} else {
int inFlight = asyncRequestBodyInFlight.decrementAndGet();
if (!isDone && inFlight < maxInFlightParts) {
synchronized (subscriptionLock) {
subscription.request(1);
}
CompletableFuture<CompletedPart> partFuture = multipartUploadHelper.sendIndividualUploadPartRequest(
uploadId, completedParts::add, futures, uploadPart(asyncRequestBody, currentPartNum), progressListener);
partFuture.whenComplete((r, t) -> {
asyncRequestBody.close();
if (t != null) {
if (tryMarkFailing()) {
failUpload(t);
}
} else {
int inFlight = asyncRequestBodyInFlight.decrementAndGet();
if (!isDone && inFlight < maxInFlightParts) {
synchronized (subscriptionLock) {
subscription.request(1);
}
completeMultipartUploadIfFinish();
}
});
completeMultipartUploadIfFinish();
}
});

// A failure that ran before this part's future was registered could not cancel it.
if (state.get() != State.RUNNING) {
partFuture.cancel(true);
}
}

private Pair<UploadPartRequest, AsyncRequestBody> uploadPart(AsyncRequestBody asyncRequestBody, int partNum) {
Expand All @@ -248,9 +278,9 @@ private Pair<UploadPartRequest, AsyncRequestBody> uploadPart(AsyncRequestBody as
@Override
public void onError(Throwable t) {
log.debug(() -> "Received onError() ", t);
if (failureActionInitiated.compareAndSet(false, true)) {
if (tryMarkFailing()) {
isDone = true;
multipartUploadHelper.failRequestsElegantly(futures, t, uploadId, returnFuture, putObjectRequest);
failUpload(t);
}
}

Expand All @@ -272,7 +302,7 @@ public void onComplete() {

private void completeMultipartUploadIfFinish() {
// All atomics including asyncRequestBodyInFlight MUST be re-read here rather than passed in by the caller.
if (isDone && asyncRequestBodyInFlight.get() == 0 && completedMultipartInitiated.compareAndSet(false, true)) {
if (isDone && asyncRequestBodyInFlight.get() == 0 && state.compareAndSet(State.RUNNING, State.COMPLETING)) {
CompletedPart[] parts = completedParts.stream()
.sorted(Comparator.comparingInt(CompletedPart::partNumber))
.toArray(CompletedPart[]::new);
Expand All @@ -285,11 +315,41 @@ private void completeMultipartUploadIfFinish() {
"The number of UploadParts requests is not equal to the expected number of parts. "
+ "Expected: %d, Actual: %d",
expectedNumParts, parts.length));
multipartUploadHelper.failRequestsElegantly(futures, exception, uploadId, returnFuture, putObjectRequest);
state.set(State.FAILING);
failUpload(exception);
return;
}

multipartUploadHelper.completeMultipartUpload(returnFuture, uploadId, parts, putObjectRequest, totalLength);
}
}

private boolean tryMarkFailing() {
return state.compareAndSet(State.RUNNING, State.FAILING);
}

/**
* Must only be called by the caller that moved the state to {@link State#FAILING}. The upload id may not be known yet,
* so the abort waits for it.
*/
private void failUpload(Throwable t) {
multipartUploadHelper.failRequestsElegantly(futures, t, null, returnFuture, putObjectRequest);
abortWhenUploadIdKnown();
}

private void abortWhenUploadIdKnown() {
uploadIdFuture.thenAccept(id -> genericMultipartHelper.cleanUpParts(
id, SdkPojoConversionUtils.toAbortMultipartUploadRequest(putObjectRequest)));
}

private static void discard(CloseableAsyncRequestBody asyncRequestBody) {
asyncRequestBody.subscribe(new CancelledSubscriber<>());
asyncRequestBody.close();
}

private enum State {
RUNNING,
COMPLETING,
FAILING
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,9 @@ private void uploadInParts(PutObjectRequest putObjectRequest, long contentLength
private void initiateNewUpload(PutObjectRequest putObjectRequest, long contentLength, AsyncRequestBody asyncRequestBody,
CompletableFuture<PutObjectResponse> returnFuture) {
CompletableFuture<CreateMultipartUploadResponse> createMultipartUploadFuture =
multipartUploadHelper.createMultipartUpload(putObjectRequest, returnFuture);
multipartUploadHelper.createMultipartUpload(putObjectRequest);

CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);

createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> {
if (throwable != null) {
Expand Down
Loading
Loading