From cf93dcb88d344dac6641aedc75c517a235bd55b1 Mon Sep 17 00:00:00 2001 From: RanVaknin <50976344+RanVaknin@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:47:15 -0800 Subject: [PATCH 1/4] Write downloadDirectory objects through temp files and move them into place --- .../bugfix-S3TransferManager-3eaf213.json | 6 + .../awssdk/transfer/s3/S3TransferManager.java | 12 + .../s3/internal/CrtS3TransferManager.java | 33 +- .../s3/internal/DownloadDirectoryHelper.java | 119 ++++- .../s3/internal/GenericS3TransferManager.java | 33 +- .../internal/DownloadDirectoryHelperTest.java | 409 ++++++++++++++++-- ...DownloadDirectoryTempFileWireMockTest.java | 358 +++++++++++++++ 7 files changed, 893 insertions(+), 77 deletions(-) create mode 100644 .changes/next-release/bugfix-S3TransferManager-3eaf213.json create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java diff --git a/.changes/next-release/bugfix-S3TransferManager-3eaf213.json b/.changes/next-release/bugfix-S3TransferManager-3eaf213.json new file mode 100644 index 000000000000..1f79df4b081e --- /dev/null +++ b/.changes/next-release/bugfix-S3TransferManager-3eaf213.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "S3 Transfer Manager", + "contributor": "", + "description": "Fixed an issue in `downloadDirectory` where keys that resolve to the same local file could produce a file that mixes the bytes of several objects, by writing each object to a temporary file next to its destination and moving it into place once complete, which also leaves an existing file unchanged when its download fails." +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 44827d2fd4da..bab7446d6ee9 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -638,6 +638,18 @@ default DirectoryUpload uploadDirectory(Consumer * object content. * *

+ * Each object is first written to a temporary file in the same directory as its destination, named + * {@code .s3tmp.}, or {@code s3tmp.} when that name would be too long. Once the object is complete, the + * temporary file is moved onto the destination, and only then is the object's download, and its + * {@link software.amazon.awssdk.transfer.s3.progress.TransferListener#transferComplete transferComplete}, completed. The + * move replaces the destination file rather than writing into it: a symbolic link at the destination is replaced instead of + * followed, and the file gets the default permissions of a new file. A download that fails leaves an existing destination + * file unchanged, and so does a cancel that arrives before the file is being moved into place. If the process stops while a + * download is in progress, its temporary file can be left behind. + * When several keys resolve to the same local file, such as {@code a/b} and {@code a//b}, the file holds the complete content + * of one of the objects. + * + *

* The current user must have write access to all directories and files * *

diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java index 65e1f80624bc..b5896b45a665 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java @@ -209,19 +209,20 @@ Download doDownloadWithDirectBufferPool(DownloadRequest returnFuture = new CompletableFuture<>(); - initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture); + initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture, beforeCompletion); return new CrtFileDownload(returnFuture, progressUpdater.progress(), observable, () -> crtDownloadRequest, null); } @@ -268,13 +269,14 @@ FileDownload doResumeDownloadFile(ResumableFileDownload resumableFileDownload) { : ResponseFileOption.CREATE_OR_APPEND; TransferProgressUpdater progressUpdater = new TransferProgressUpdater(newDownloadFileRequest, null); - DownloadFileRequest crtDownloadRequest = crtDownloadFileRequest(newDownloadFileRequest, observable, + DownloadFileRequest crtDownloadRequest = crtDownloadFileRequest(newDownloadFileRequest, + newDownloadFileRequest.destination(), observable, progressUpdater, responseFileOption); newDownloadFileRequestFuture.complete(crtDownloadRequest); log.debug(() -> LogSanitizer.escapeControlCharacters("Sending downloadFileRequest " + crtDownloadRequest)); - initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture); + initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture, NO_OP); progressFuture.complete(progressUpdater.progress()); }).exceptionally(throwable -> { handleException(returnFuture, progressFuture, newDownloadFileRequestFuture, throwable); @@ -303,7 +305,8 @@ private static boolean hasCompletedParts(ResumableFileDownload resumableFileDown private void initiateCrtDownload(DownloadFileRequest downloadRequest, TransferProgressUpdater progressUpdater, - CompletableFuture returnFuture) { + CompletableFuture returnFuture, + Runnable beforeCompletion) { try { progressUpdater.transferInitiated(); AsyncResponseTransformer responseTransformer = @@ -318,26 +321,28 @@ private void initiateCrtDownload(DownloadFileRequest downloadRequest, // Forward download cancellation to CRT future CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); - CompletableFutureUtils.forwardTransformedResultTo(crtFuture, returnFuture, - res -> CompletedFileDownload.builder() - .response(res) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(crtFuture, returnFuture, res -> { + CompletedFileDownload completed = CompletedFileDownload.builder().response(res).build(); + beforeCompletion.run(); + return completed; + }); } catch (Throwable throwable) { returnFuture.completeExceptionally(throwable); } } /** - * Attaches the execution attributes that tell the CRT-based client to write the response body straight to the destination - * file, together with the progress listener and pause observable needed to report progress on and pause that transfer. + * Attaches the execution attributes that tell the CRT-based client to write the response body straight to {@code writePath}, + * together with the progress listener and pause observable needed to report progress on and pause that transfer. */ private DownloadFileRequest crtDownloadFileRequest(DownloadFileRequest downloadRequest, + Path writePath, S3MetaRequestPauseObservable observable, TransferProgressUpdater progressUpdater, ResponseFileOption responseFileOption) { GetObjectRequest getObjectRequest = attachSdkAttribute( downloadRequest.getObjectRequest(), - b -> b.putExecutionAttribute(RESPONSE_FILE_PATH, downloadRequest.destination()) + b -> b.putExecutionAttribute(RESPONSE_FILE_PATH, writePath) .putExecutionAttribute(RESPONSE_FILE_OPTION, responseFileOption)); GetObjectRequest crtGetObjectRequest = attachCrtSdkAttribute( diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java index 7a47de57fa0e..150f5824d1c8 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java @@ -19,10 +19,13 @@ import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.DEFAULT_PREFIX; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Collection; import java.util.Queue; import java.util.UUID; @@ -57,16 +60,18 @@ @SdkInternalApi public class DownloadDirectoryHelper { private static final Logger log = Logger.loggerFor(S3TransferManager.class); + private static final String TEMP_FILE_INFIX = ".s3tmp."; + private static final int MAX_FILE_NAME_BYTES = 255; private final TransferManagerConfiguration transferConfiguration; - private final Function downloadFileFunction; + private final FileDownloader fileDownloader; private final ListObjectsHelper listObjectsHelper; public DownloadDirectoryHelper(TransferManagerConfiguration transferConfiguration, ListObjectsHelper listObjectsHelper, - Function downloadFileFunction) { + FileDownloader fileDownloader) { this.transferConfiguration = transferConfiguration; - this.downloadFileFunction = downloadFileFunction; + this.fileDownloader = fileDownloader; this.listObjectsHelper = listObjectsHelper; } @@ -163,6 +168,11 @@ private void validatePath(Path destinationDirectory, Path targetPath, String key } } + /** + * Downloads the object to a temporary file next to its destination and moves it into place once it is complete. Keys such + * as "a/b" and "a//b", or "A" and "a" on a case-insensitive file system, resolve to the same file, and writing to it + * directly would let their concurrent downloads mix bytes or delete each other's file. + */ private CompletableFuture doDownloadSingleFile( CompletableFuture returnFuture, DownloadDirectoryRequest downloadDirectoryRequest, @@ -183,25 +193,50 @@ private CompletableFuture doDownloadSingleFile( } DownloadFileRequest downloadFileRequest = downloadFileRequest(downloadDirectoryRequest, s3Object, destinationPath); + Path destination = downloadFileRequest.destination(); + Path tempFile; + try { + tempFile = tempFileNextTo(destination); + } catch (RuntimeException e) { + SdkClientException failure = SdkClientException.create("Cannot download key " + s3Object.key() + " to " + + destination + ", it is not a path to a file", e); + failedFileDownloads.add(FailedFileDownload.builder().exception(failure).request(downloadFileRequest).build()); + return CompletableFutureUtils.failedFuture(failure); + } try { log.debug(() -> LogSanitizer.escapeControlCharacters("Sending download request " + downloadFileRequest)); createParentDirectoriesIfNeeded(destinationPath); + // The move runs inside the download, so the download and its listeners complete only once the file is in place. + Runnable moveIntoPlaceOnSuccess = () -> { + SdkClientException failure = moveIntoPlace(returnFuture, tempFile, destination); + if (failure != null) { + throw failure; + } + }; CompletableFuture executionFuture = - downloadFileFunction.apply(downloadFileRequest).completionFuture(); - CompletableFuture future = executionFuture.whenComplete((r, t) -> { - if (t != null) { - failedFileDownloads.add(FailedFileDownload.builder() - .exception(t instanceof CompletionException ? t.getCause() : t) - .request(downloadFileRequest) - .build()); + fileDownloader.download(downloadFileRequest, tempFile, moveIntoPlaceOnSuccess).completionFuture(); + CompletableFuture future = new CompletableFuture<>(); + // Attached to executionFuture rather than to the returned future, so the temporary file is also removed when the + // directory download cancels the returned future. + executionFuture.whenComplete((r, t) -> { + if (t == null) { + future.complete(r); + return; } + deleteQuietly(tempFile); + failedFileDownloads.add(FailedFileDownload.builder() + .exception(t instanceof CompletionException ? t.getCause() : t) + .request(downloadFileRequest) + .build()); + future.completeExceptionally(t); }); CompletableFutureUtils.forwardExceptionTo(future, executionFuture); return future; } catch (Throwable throwable) { + deleteQuietly(tempFile); failedFileDownloads.add(FailedFileDownload.builder() .exception(throwable) .request(downloadFileRequest) @@ -210,6 +245,61 @@ private CompletableFuture doDownloadSingleFile( } } + /** + * Returns {@code .s3tmp.} next to the destination, or {@code s3tmp.} when that name would exceed 255 UTF-8 + * bytes, the file name limit on common file systems. + */ + private static Path tempFileNextTo(Path destination) { + Path fileName = destination.getFileName(); + if (fileName == null) { + throw new IllegalArgumentException(destination + " has no file name"); + } + String suffix = TEMP_FILE_INFIX + UUID.randomUUID(); + String tempName = fileName + suffix; + if (tempName.getBytes(StandardCharsets.UTF_8).length > MAX_FILE_NAME_BYTES) { + tempName = suffix.substring(1); + } + return destination.resolveSibling(tempName); + } + + /** + * Moves a completed download onto its destination, replacing any file already there. Returns the failure to report for the + * key, or null when the key succeeded. + */ + private static SdkClientException moveIntoPlace(CompletableFuture returnFuture, + Path tempFile, + Path destination) { + if (returnFuture.isCompletedExceptionally()) { + deleteQuietly(tempFile); + return SdkClientException.create("Download was cancelled before the file could be moved to " + destination); + } + + if (!Files.exists(tempFile)) { + return SdkClientException.create("The download completed without writing " + tempFile + ", so " + destination + + " was not updated"); + } + + try { + try { + Files.move(tempFile, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tempFile, destination, StandardCopyOption.REPLACE_EXISTING); + } + return null; + } catch (IOException | RuntimeException e) { + deleteQuietly(tempFile); + return SdkClientException.create("Failed to move the downloaded file to " + destination, e); + } + } + + private static void deleteQuietly(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException | RuntimeException e) { + log.debug(() -> LogSanitizer.escapeControlCharacters("Failed to delete temporary file " + path), e); + } + } + /** * Reports a key that can't be mapped to a path inside the destination directory as a single failed download, without @@ -283,4 +373,13 @@ private static void createParentDirectoriesIfNeeded(Path destinationPath) { throw SdkClientException.create("Failed to create parent directories for " + destinationPath, e); } } + + /** + * Starts the download of one object into {@code writePath}. {@code beforeCompletion} runs once the object is written and + * before the download completes; if it throws, the download fails with that exception. + */ + @FunctionalInterface + interface FileDownloader { + FileDownload download(DownloadFileRequest request, Path writePath, Runnable beforeCompletion); + } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 1f7033417f1c..51d321ffa4f6 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -100,6 +100,7 @@ @SdkInternalApi class GenericS3TransferManager implements S3TransferManager { + static final Runnable NO_OP = () -> { }; private static final Logger log = Logger.loggerFor(S3TransferManager.class); private static final PauseResumeHelper PAUSE_RESUME_HELPER = new PauseResumeHelper(); private final S3AsyncClient s3AsyncClient; @@ -119,7 +120,7 @@ class GenericS3TransferManager implements S3TransferManager { ListObjectsHelper listObjectsHelper = new ListObjectsHelper(s3AsyncClient::listObjectsV2); downloadDirectoryHelper = new DownloadDirectoryHelper(transferConfiguration, listObjectsHelper, - this::downloadFile); + this::downloadFileTo); this.isDefaultS3AsyncClient = isDefaultS3AsyncClient; } @@ -453,6 +454,17 @@ public final FileDownload downloadFile(DownloadFileRequest downloadRequest) { * May be overridden by subclasses to provide customized behavior */ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { + return downloadFileTo(downloadRequest, downloadRequest.destination(), NO_OP); + } + + /** + * Downloads the object into {@code writePath} while reporting {@code downloadRequest}, including its destination, to the + * request's transfer listeners. {@code beforeCompletion} runs after the object is written and before the download, and its + * listeners, complete; if it throws, the download fails with that exception. The returned download cannot be paused and + * resumed when {@code writePath} differs from the destination. Subclasses override this rather than {@link #doDownloadFile}. + */ + FileDownload downloadFileTo(DownloadFileRequest downloadRequest, Path writePath, Runnable beforeCompletion) { + Validate.paramNotNull(writePath, "writePath"); GetObjectRequest getObjectRequestWithAttributes = attachSdkAttribute( downloadRequest.getObjectRequest(), b -> b.putExecutionAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT, new MultipartDownloadResumeContext())); @@ -460,12 +472,11 @@ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { downloadRequest.copy(downloadFileRequest -> downloadFileRequest.getObjectRequest(getObjectRequestWithAttributes)); AsyncResponseTransformer responseTransformer = - AsyncResponseTransformer.toFile(downloadFileRequestWithAttributes.destination(), - FileTransformerConfiguration.defaultCreateOrReplaceExisting()); + AsyncResponseTransformer.toFile(writePath, FileTransformerConfiguration.defaultCreateOrReplaceExisting()); CompletableFuture returnFuture = new CompletableFuture<>(); TransferProgressUpdater progressUpdater = initiateDownload( - downloadFileRequestWithAttributes, responseTransformer, returnFuture); + downloadFileRequestWithAttributes, responseTransformer, returnFuture, beforeCompletion); return new DefaultFileDownload(returnFuture, progressUpdater.progress(), () -> downloadFileRequestWithAttributes, null); } @@ -473,7 +484,8 @@ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { private TransferProgressUpdater initiateDownload( DownloadFileRequest downloadRequest, AsyncResponseTransformer responseTransformer, - CompletableFuture returnFuture) { + CompletableFuture returnFuture, + Runnable beforeCompletion) { TransferProgressUpdater progressUpdater = new TransferProgressUpdater(downloadRequest, null); try { progressUpdater.transferInitiated(); @@ -491,10 +503,11 @@ private TransferProgressUpdater initiateDownload( // Forward download cancellation to future CompletableFutureUtils.forwardExceptionTo(returnFuture, future); - CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, - res -> CompletedFileDownload.builder() - .response(res) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, res -> { + CompletedFileDownload completed = CompletedFileDownload.builder().response(res).build(); + beforeCompletion.run(); + return completed; + }); } catch (Throwable throwable) { returnFuture.completeExceptionally(throwable); } @@ -551,7 +564,7 @@ FileDownload doResumeDownloadFile(ResumableFileDownload resumableFileDownload) { TransferProgressUpdater progressUpdater = initiateDownload(newDownloadFileRequest, requestPair.right(), - returnFuture); + returnFuture, NO_OP); progressFuture.complete(progressUpdater.progress()); }).exceptionally(throwable -> { handleException(returnFuture, progressFuture, newDownloadFileRequestFuture, throwable); diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java index 7ead9e8f3613..5fc6900df07e 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java @@ -28,26 +28,31 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; import java.time.Duration; import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import java.util.concurrent.locks.LockSupport; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.logging.log4j.Level; @@ -60,6 +65,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.Answer; import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.model.EncodingType; @@ -88,7 +94,7 @@ public class DownloadDirectoryHelperTest { private static final String DIRECTORY_NAME = "test"; private FileSystem fs; private Path directory; - private Function singleDownloadFunction; + private DownloadDirectoryHelper.FileDownloader singleDownloadFunction; private DownloadDirectoryHelper downloadDirectoryHelper; private ListObjectsHelper listObjectsHelper; @@ -97,7 +103,7 @@ public void methodSetup() { fs = Jimfs.newFileSystem(); directory = fs.getPath("test"); listObjectsHelper = mock(ListObjectsHelper.class); - singleDownloadFunction = mock(Function.class); + singleDownloadFunction = mock(DownloadDirectoryHelper.FileDownloader.class); downloadDirectoryHelper = new DownloadDirectoryHelper(TransferManagerConfiguration.builder().build(), listObjectsHelper, singleDownloadFunction); @@ -122,7 +128,7 @@ void downloadDirectory_allDownloadsSucceed_failedDownloadsShouldBeEmpty() throws FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -133,7 +139,7 @@ void downloadDirectory_allDownloadsSucceed_failedDownloadsShouldBeEmpty() throws CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).element(0).satisfies(d -> assertThat(d.getObjectRequest().key()).isEqualTo( @@ -151,7 +157,7 @@ void downloadDirectory_containsFolderObjects_shouldSkip() throws Exception { FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -162,7 +168,7 @@ void downloadDirectory_containsFolderObjects_shouldSkip() throws Exception { CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); List allValues = argumentCaptor.getAllValues(); @@ -194,7 +200,7 @@ void invalidKey_shouldReportFailedDownloadAndContinue(String testingString) thro assertThat(completed.failedTransfers()).hasSize(1); assertRefused(completed.failedTransfers().get(0), destinationDirectory, key); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); } @Test @@ -212,7 +218,7 @@ void downloadDirectory_destinationDoesNotExistAndOnlyKeyRefused_shouldReportFail assertThat(completed.failedTransfers()).hasSize(1); assertRefused(completed.failedTransfers().get(0), directory, "../escape.txt"); assertThat(Files.exists(directory)).isFalse(); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); } @Test @@ -259,7 +265,7 @@ void downloadDirectory_manyRefusedKeysInOneListingPage_shouldReportEachWithoutSt assertThat(completed.failedTransfers()).allSatisfy(f -> assertThat(f.exception()) .isInstanceOf(SdkClientException.class) .hasMessageContaining("Cannot download key")); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); assertThat(Files.exists(directory)).isFalse(); } @@ -267,7 +273,7 @@ void downloadDirectory_manyRefusedKeysInOneListingPage_shouldReportEachWithoutSt void downloadDirectory_refusedKeyAmongBenignKeys_shouldDownloadOthersAndReportOneFailure() throws Exception { stubSuccessfulListObjects(listObjectsHelper, "a.txt", "b.txt", "c/../../escape.txt", "d.txt"); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); DownloadDirectoryHelper helper = new DownloadDirectoryHelper(TransferManagerConfiguration.builder().transferDirectoryMaxConcurrency(1).build(), listObjectsHelper, @@ -295,7 +301,7 @@ void downloadDirectory_keyInvalidOnWindowsFileSystem_shouldReportFailedDownloadA stubSuccessfulListObjects(listObjectsHelper, "ok.txt", invalidKey); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -321,7 +327,7 @@ void downloadDirectory_keysWithControlCharacters_debugLogsAreOneEscapedLineEach( String refusedKey = "../escape\r\nFORGED level=ERROR"; stubSuccessfulListObjects(listObjectsHelper, sentKey, refusedKey); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); CompletedDirectoryDownload completed; List messages; @@ -378,7 +384,7 @@ void downloadDirectory_cancel_shouldCancelAllFutures() throws Exception { AtomicInteger callCount = new AtomicInteger(0); CountDownLatch bothStarted = new CountDownLatch(2); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenAnswer(invocation -> { + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { bothStarted.countDown(); return callCount.incrementAndGet() == 1 ? fileDownload : fileDownload2; }); @@ -409,7 +415,7 @@ void downloadDirectory_partialSuccess_shouldProvideFailedDownload() throws Excep SdkClientException exception = SdkClientException.create("failed"); FileDownload fileDownload2 = newFailedDownload(exception); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -431,7 +437,7 @@ void downloadDirectory_withFilter_shouldHonorFilter() throws Exception { FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -443,7 +449,7 @@ void downloadDirectory_withFilter_shouldHonorFilter() throws Exception { CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(1)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(1)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).element(0).satisfies(d -> assertThat(d.getObjectRequest().key()).isEqualTo( @@ -457,8 +463,8 @@ void downloadDirectory_withDownloadRequestTransformer_shouldTransform() throws E FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); - Path newDestination = Paths.get("/new/path"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); + Path newDestination = directory.getFileSystem().getPath("/new/path"); GetObjectRequest newGetObjectRequest = GetObjectRequest.builder().build(); List newTransferListener = Arrays.asList(LoggingTransferListener.create()); @@ -475,7 +481,7 @@ void downloadDirectory_withDownloadRequestTransformer_shouldTransform() throws E CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).allSatisfy(d -> { @@ -495,7 +501,7 @@ void downloadDirectory_withListObjectsRequestTransformer_shouldTransform() throw EncodingType newEncodingType = EncodingType.URL; int newMaxKeys = 10; - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -525,8 +531,8 @@ void downloadDirectory_shouldRecursivelyDownload(FileSystem jimfs) { stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -563,8 +569,8 @@ void downloadDirectory_withPrefix_shouldStripPrefixInDestinationPath(FileSystem stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -579,16 +585,13 @@ void downloadDirectory_withPrefix_shouldStripPrefixInDestinationPath(FileSystem assertThat(actualRequests.size()).isEqualTo(keys.length); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); String jimfsSeparator = jimfs.getSeparator(); List expectedPaths = Arrays.asList("image.jpg", "title.jpg", "ghi/xyz.txt").stream() .map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace("/",jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } @ParameterizedTest @@ -600,8 +603,8 @@ void downloadDirectory_containsObjectWithPrefixInIt_shouldInclude(FileSystem jim stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -615,16 +618,13 @@ void downloadDirectory_containsObjectWithPrefixInIt_shouldInclude(FileSystem jim assertThat(actualRequests.size()).isEqualTo(keys.length); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); String jimfsSeparator = jimfs.getSeparator(); List expectedPaths = Arrays.asList("def/image.jpg", "def/title.jpg", "abcd").stream() .map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace("/",jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } @ParameterizedTest @@ -636,7 +636,7 @@ void downloadDirectory_withDelimiter_shouldHonor(FileSystem jimfs) { stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -671,7 +671,7 @@ void downloadDirectory_withDownloadRequestTransformer_transformerThrows_failsDow FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); RuntimeException exception = new RuntimeException("boom"); @@ -696,7 +696,7 @@ void downloadDirectory_withListObjectsRequestTransformer_transformerThrows_fails FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); RuntimeException exception = new RuntimeException("boom"); @@ -730,12 +730,9 @@ private static DefaultFileDownload completedDownload() { private static void verifyDestinationPathForSingleDownload(FileSystem jimfs, String delimiter, String[] keys, List actualRequests) { String jimfsSeparator = jimfs.getSeparator(); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); List expectedPaths = Arrays.stream(keys).map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace(delimiter, jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } private FileDownload newSuccessfulDownload() { @@ -747,7 +744,7 @@ private FileDownload newSuccessfulDownload() { return fileDownload; } - private FileDownload newFailedDownload(SdkClientException exception) { + private FileDownload newFailedDownload(RuntimeException exception) { CompletableFuture failedFuture = new CompletableFuture<>(); FileDownload fileDownload2 = newDownload(failedFuture); failedFuture.completeExceptionally(exception); @@ -791,4 +788,330 @@ private FileDownload newDownload(CompletableFuture future ".")).getObjectRequest(GetObjectRequest.builder().build()).build(), null); } + + @Test + void downloadDirectory_keysResolvingToSameFile_shouldLeaveOneCompleteObject() throws Exception { + String longBody = "C2:x//y.txt"; + String shortBody = "C2:x/y.txt"; + stubSuccessfulListObjects(listObjectsHelper, "x//y.txt", "x/y.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "x//y.txt", "x/y.txt"); + Path destination = directory.resolve("x/y.txt"); + assertThat(downloads.values()).allSatisfy(d -> { + assertThat(d.request.destination()).isEqualTo(destination); + assertTempFileFor(d.writePath, destination); + }); + assertThat(downloads.get("x//y.txt").writePath).isNotEqualTo(downloads.get("x/y.txt").writePath); + downloads.get("x//y.txt").succeed(longBody); + downloads.get("x/y.txt").succeed(shortBody); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).isEmpty(); + assertThat(readString(directory.resolve("x/y.txt"))).isEqualTo(shortBody); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_keyFailsAfterCollidingKeySucceeded_shouldKeepSucceededObject() throws Exception { + String body = "C2:x//y.txt"; + stubSuccessfulListObjects(listObjectsHelper, "x//y.txt", "x/y.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "x//y.txt", "x/y.txt"); + downloads.get("x//y.txt").succeed(body); + downloads.get("x/y.txt").fail(SdkClientException.create("connection reset")); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).hasSize(1); + FailedFileDownload failed = completed.failedTransfers().iterator().next(); + assertThat(failed.request().getObjectRequest().key()).isEqualTo("x/y.txt"); + assertThat(failed.request().destination()).isEqualTo(directory.resolve("x/y.txt")); + assertThat(failed.exception()).isInstanceOf(SdkClientException.class).hasMessageContaining("connection reset"); + assertThat(readString(directory.resolve("x/y.txt"))).isEqualTo(body); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_downloadFails_shouldKeepExistingFile() throws Exception { + Files.createDirectories(directory); + Files.write(directory.resolve("a.txt"), "existing".getBytes(StandardCharsets.UTF_8)); + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + downloads.get("a.txt").fail(SdkClientException.create("failed")); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).hasSize(1); + assertThat(readString(directory.resolve("a.txt"))).isEqualTo("existing"); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_downloadSucceeds_shouldReplaceExistingFile() throws Exception { + Files.createDirectories(directory); + Files.write(directory.resolve("a.txt"), "a longer existing body".getBytes(StandardCharsets.UTF_8)); + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + downloads.get("a.txt").succeed("new"); + + assertThat(directoryDownload.completionFuture().get(5, TimeUnit.SECONDS).failedTransfers()).isEmpty(); + assertThat(readString(directory.resolve("a.txt"))).isEqualTo("new"); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_cancelledDuringDownload_shouldRemoveTempFileAndNotCreateDestination() throws Exception { + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + directoryDownload.completionFuture().cancel(true); + + assertThatThrownBy(() -> downloads.get("a.txt").future.get(1, TimeUnit.SECONDS)) + .isInstanceOf(CancellationException.class); + assertThat(Files.exists(directory.resolve("a.txt"))).isFalse(); + assertNoTempFiles(directory); + } + + @ParameterizedTest + @ValueSource(strings = {"a", "\u00e9", "\u6f22", "\ud83d\ude00"}) + void downloadDirectory_nameTooLongForSuffix_shouldUseSuffixOnlyWithin255Bytes(String character) throws Exception { + int bytesPerCharacter = character.getBytes(StandardCharsets.UTF_8).length; + String longName = IntStream.range(0, 255 / bytesPerCharacter).mapToObj(i -> character).collect(Collectors.joining()); + + Path writePath = writePathFor(longName); + + String tempName = writePath.getFileName().toString(); + assertThat(tempName).matches("s3tmp\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + assertThat(writePath.getParent()).isEqualTo(directory); + } + + @Test + void downloadDirectory_nameThatFitsWithSuffix_shouldKeepName() throws Exception { + String name = IntStream.range(0, 212).mapToObj(i -> "a").collect(Collectors.joining()); + + Path writePath = writePathFor(name); + + String tempName = writePath.getFileName().toString(); + assertThat(tempName).startsWith(name + ".s3tmp."); + assertThat(tempName.getBytes(StandardCharsets.UTF_8)).hasSize(255); + } + + @Test + void downloadDirectory_nameOneByteTooLongForSuffix_shouldUseSuffixOnly() throws Exception { + String name = IntStream.range(0, 213).mapToObj(i -> "a").collect(Collectors.joining()); + + Path writePath = writePathFor(name); + + assertThat(writePath.getFileName().toString()).startsWith("s3tmp."); + } + + private Path writePathFor(String key) throws Exception { + stubSuccessfulListObjects(listObjectsHelper, key); + ArgumentCaptor writePathCaptor = ArgumentCaptor.forClass(Path.class); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), writePathCaptor.capture(), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); + downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()) + .completionFuture().get(5, TimeUnit.SECONDS); + return writePathCaptor.getValue(); + } + + @Test + void downloadDirectory_destinationHasNoFileName_shouldReportThatKeyAndDownloadOthers() throws Exception { + stubSuccessfulListObjects(listObjectsHelper, "good.txt", "bad.txt"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); + Path root = directory.toAbsolutePath().getRoot(); + + CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder() + .destination(directory) + .bucket("bucket") + .downloadFileRequestTransformer(f -> { + if ("bad.txt".equals(f.build().getObjectRequest().key())) { + f.destination(root); + } + }) + .build()).completionFuture().get(5, TimeUnit.SECONDS); + + assertThat(completed.failedTransfers()).hasSize(1); + FailedFileDownload failed = completed.failedTransfers().iterator().next(); + assertThat(failed.request().getObjectRequest().key()).isEqualTo("bad.txt"); + assertThat(failed.exception()).isInstanceOf(SdkClientException.class).hasMessageContaining("not a path to a file"); + verify(singleDownloadFunction, times(1)).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); + } + + /** + * Answers like a real download: the write path exists once the download is started, and a download that has already + * succeeded has run its before-completion step. Returns the downloads in order, then keeps returning the last one. + */ + private Answer writingWritePath(FileDownload... downloads) { + AtomicInteger calls = new AtomicInteger(); + return invocation -> { + Path writePath = invocation.getArgument(1); + if (writePath.getParent() != null) { + Files.createDirectories(writePath.getParent()); + } + if (!Files.exists(writePath)) { + Files.createFile(writePath); + } + FileDownload download = downloads[Math.min(calls.getAndIncrement(), downloads.length - 1)]; + CompletableFuture completion = download.completionFuture(); + if (completion.isDone() && !completion.isCompletedExceptionally()) { + try { + Runnable beforeCompletion = invocation.getArgument(2); + beforeCompletion.run(); + } catch (RuntimeException e) { + return newFailedDownload(e); + } + } + return download; + }; + } + + @Test + void downloadDirectory_downloadSucceedsWithoutWritingFile_shouldReportKeyAndKeepExistingFile() throws Exception { + Files.createDirectories(directory); + Files.write(directory.resolve("a.txt"), "existing".getBytes(StandardCharsets.UTF_8)); + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) + .thenAnswer(invocation -> { + try { + ((Runnable) invocation.getArgument(2)).run(); + } catch (RuntimeException e) { + return newFailedDownload(e); + } + return completedDownload(); + }); + + CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()).completionFuture() + .get(5, TimeUnit.SECONDS); + + assertThat(completed.failedTransfers()).hasSize(1); + assertThat(completed.failedTransfers().iterator().next().exception()) + .isInstanceOf(SdkClientException.class).hasMessageContaining("completed without writing"); + assertThat(readString(directory.resolve("a.txt"))).isEqualTo("existing"); + } + + @Test + void downloadDirectory_destinationIsSymbolicLink_shouldReplaceLinkAndLeaveItsTargetUnchanged() throws Exception { + Files.createDirectories(directory); + Path linkTarget = fs.getPath("elsewhere.txt"); + Files.write(linkTarget, "link target".getBytes(StandardCharsets.UTF_8)); + Files.createSymbolicLink(directory.resolve("a.txt"), linkTarget.toAbsolutePath()); + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + downloads.get("a.txt").succeed("new"); + + assertThat(directoryDownload.completionFuture().get(5, TimeUnit.SECONDS).failedTransfers()).isEmpty(); + assertThat(Files.isSymbolicLink(directory.resolve("a.txt"))).isFalse(); + assertThat(readString(directory.resolve("a.txt"))).isEqualTo("new"); + assertThat(readString(linkTarget)).isEqualTo("link target"); + } + + private Map stubFileWritingDownloads() { + Map downloads = new ConcurrentHashMap<>(); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { + DownloadFileRequest request = invocation.getArgument(0); + FileWritingDownload download = new FileWritingDownload(request, invocation.getArgument(1), invocation.getArgument(2)); + downloads.put(request.getObjectRequest().key(), download); + return newDownload(download.future); + }); + return downloads; + } + + private static void awaitStarted(Map downloads, String... keys) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!downloads.keySet().containsAll(Arrays.asList(keys))) { + assertThat(System.nanoTime()).as("downloads started: %s", downloads.keySet()).isLessThan(deadline); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(5)); + } + } + + private static String readString(Path path) throws IOException { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } + + private static void assertNoTempFiles(Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (java.util.stream.Stream paths = Files.walk(directory)) { + assertThat(paths.filter(p -> p.getFileName().toString().contains("s3tmp."))).isEmpty(); + } + } + + private static void assertDestinations(List requests, List expectedPaths) { + assertThat(requests.stream().map(r -> r.destination().toString()).collect(Collectors.toList())) + .isEqualTo(expectedPaths); + } + + private static void assertTempFileFor(Path actual, Path destination) { + assertThat(actual.getParent()).isEqualTo(destination.getParent()); + assertThat(actual.getFileName().toString()).matches(Pattern.quote(destination.getFileName().toString()) + + "\\.s3tmp\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + } + + /** + * Writes to the write path the way the SDK's file writers do: the file is created and truncated when the download starts, + * the body is written from offset 0 without truncating when it arrives, and a partial file is left in place when the + * download fails. + */ + private static final class FileWritingDownload { + private final DownloadFileRequest request; + private final Path writePath; + private final Runnable beforeCompletion; + private final CompletableFuture future = new CompletableFuture<>(); + + FileWritingDownload(DownloadFileRequest request, Path writePath, Runnable beforeCompletion) throws IOException { + this.request = request; + this.writePath = writePath; + this.beforeCompletion = beforeCompletion; + Files.write(writePath, new byte[0]); + } + + void succeed(String body) throws IOException { + try (FileChannel channel = FileChannel.open(writePath, StandardOpenOption.WRITE)) { + channel.write(ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8)), 0); + } + try { + beforeCompletion.run(); + } catch (RuntimeException e) { + future.completeExceptionally(e); + return; + } + future.complete(CompletedFileDownload.builder().response(GetObjectResponse.builder().build()).build()); + } + + void fail(SdkClientException exception) throws IOException { + try (FileChannel channel = FileChannel.open(writePath, StandardOpenOption.WRITE)) { + channel.write(ByteBuffer.wrap("partial".getBytes(StandardCharsets.UTF_8)), 0); + } + future.completeExceptionally(exception); + } + } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java new file mode 100644 index 000000000000..611faa60b76d --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java @@ -0,0 +1,358 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.head; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.testutils.FileUtils; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +@WireMockTest +public class DownloadDirectoryTempFileWireMockTest { + private static final String BUCKET = "bucket"; + + private Path destination; + private S3AsyncClient s3; + private S3TransferManager tm; + + @BeforeEach + void setUp() throws IOException { + destination = Files.createTempDirectory("tm-temp-file"); + } + + @AfterEach + void tearDown() { + if (tm != null) { + tm.close(); + } + if (s3 != null) { + s3.close(); + } + FileUtils.cleanUpTestDirectory(destination); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_withPerFileListener_listenerSeesRealDestinationAndNoTempFileRemains(String clientType, + WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubListObjects("a.txt", "sub/b.txt"); + stubObject("a.txt", "content-a"); + stubObject("sub/b.txt", "content-b"); + + List initiated = new CopyOnWriteArrayList<>(); + List completed = new CopyOnWriteArrayList<>(); + List contentAtCompletion = new CopyOnWriteArrayList<>(); + TransferListener listener = new TransferListener() { + @Override + public void transferInitiated(Context.TransferInitiated context) { + initiated.add(((DownloadFileRequest) context.request()).destination()); + } + + @Override + public void transferComplete(Context.TransferComplete context) { + Path completedDestination = ((DownloadFileRequest) context.request()).destination(); + contentAtCompletion.add(readOrMissing(completedDestination)); + completed.add(completedDestination); + } + }; + + CompletedDirectoryDownload result = + tm.downloadDirectory(r -> r.bucket(BUCKET) + .destination(destination) + .downloadFileRequestTransformer(f -> f.addTransferListener(listener))) + .completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(result.failedTransfers()).isEmpty(); + Path a = destination.resolve("a.txt"); + Path b = destination.resolve("sub").resolve("b.txt"); + assertThat(initiated).containsExactlyInAnyOrder(a, b); + // The multipart client does not deliver transferComplete against these stubs, for downloadFile as well. + if (!"java-multipart".equals(clientType)) { + // transferComplete can be delivered after the directory download's future completes, but only once the file + // is in place. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (completed.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(completed).containsExactlyInAnyOrder(a, b); + assertThat(contentAtCompletion).containsExactlyInAnyOrder("content-a", "content-b"); + } + assertThat(a).hasContent("content-a"); + assertThat(b).hasContent("content-b"); + try (Stream files = Files.walk(destination)) { + assertThat(files.filter(Files::isRegularFile).map(destination::relativize).map(Path::toString) + .collect(Collectors.toList())) + .containsExactlyInAnyOrder("a.txt", "sub" + destination.getFileSystem().getSeparator() + "b.txt"); + } + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_keysResolvingToSameFile_fileHoldsOneCompleteObject(String clientType, WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + String longBody = "C2:x//y.txt"; + String shortBody = "C2:x/y.txt"; + stubListObjects("x//y.txt", "x/y.txt"); + // Both responses send their headers at once and their bodies later, the longer one first, so every client has + // opened its file for both keys before either body is written. + stubObject("x//y.txt", longBody.getBytes(StandardCharsets.UTF_8), 300); + stubObject("x/y.txt", shortBody.getBytes(StandardCharsets.UTF_8), 900); + + CompletedDirectoryDownload result = tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)) + .completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(result.failedTransfers()).isEmpty(); + assertThat(readOrMissing(destination.resolve("x").resolve("y.txt"))).isIn(longBody, shortBody); + assertThat(regularFiles()).containsExactly("x" + destination.getFileSystem().getSeparator() + "y.txt"); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_cancelledMidDownload_leavesNoFiles(String clientType, WireMockRuntimeInfo wm) throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubListObjects("big.bin"); + stubObject("big.bin", new byte[2 * 1024 * 1024], 10_000); + + CompletableFuture download = + tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)).completionFuture(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (regularFiles().isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(regularFiles()).as("the download started writing").isNotEmpty(); + + download.cancel(true); + // Gives an in-flight write the chance to land after the cancel. + Thread.sleep(1_000); + + assertThat(regularFiles()).isEmpty(); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadFileTo_beforeCompletionRunsBeforeDownloadAndListenerComplete(String clientType, WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubObject("a.txt", "content-a"); + Path writePath = destination.resolve("a.txt.tmp"); + AtomicBoolean ran = new AtomicBoolean(); + List ranAtTransferComplete = new CopyOnWriteArrayList<>(); + TransferListener listener = new TransferListener() { + @Override + public void transferComplete(Context.TransferComplete context) { + ranAtTransferComplete.add(ran.get()); + } + }; + DownloadFileRequest request = DownloadFileRequest.builder() + .getObjectRequest(g -> g.bucket(BUCKET).key("a.txt")) + .destination(destination.resolve("a.txt")) + .addTransferListener(listener) + .build(); + + ((GenericS3TransferManager) tm).downloadFileTo(request, writePath, () -> { + assertThat(writePath).hasContent("content-a"); + ran.set(true); + }).completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(ran).isTrue(); + // The multipart client does not deliver transferComplete against these stubs, for downloadFile as well. + if (!"java-multipart".equals(clientType)) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ranAtTransferComplete.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(ranAtTransferComplete).containsExactly(true); + } + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadFileTo_beforeCompletionThrows_downloadAndListenerFail(String clientType, WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubObject("a.txt", "content-a"); + RuntimeException moveFailure = new RuntimeException("move failed"); + List failures = new CopyOnWriteArrayList<>(); + AtomicBoolean completed = new AtomicBoolean(); + TransferListener listener = new TransferListener() { + @Override + public void transferComplete(Context.TransferComplete context) { + completed.set(true); + } + + @Override + public void transferFailed(Context.TransferFailed context) { + failures.add(context.exception()); + } + }; + DownloadFileRequest request = DownloadFileRequest.builder() + .getObjectRequest(g -> g.bucket(BUCKET).key("a.txt")) + .destination(destination.resolve("a.txt")) + .addTransferListener(listener) + .build(); + + CompletableFuture download = + ((GenericS3TransferManager) tm).downloadFileTo(request, destination.resolve("a.txt.tmp"), () -> { + throw moveFailure; + }).completionFuture(); + + assertThatThrownBy(() -> download.get(30, TimeUnit.SECONDS)).hasCause(moveFailure); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (failures.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(failures).containsExactly(moveFailure); + assertThat(completed).isFalse(); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_emptyObject_createsEmptyFile(String clientType, WireMockRuntimeInfo wm) throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubListObjects("empty.bin"); + String objectPath = "/" + BUCKET + "/empty.bin"; + stubFor(head(urlPathEqualTo(objectPath)) + .willReturn(aResponse().withStatus(200).withHeader("Content-Length", "0").withHeader("ETag", "\"e\""))); + // S3 rejects a ranged GET of an empty object, and the CRT-based client retries such a request without the range. + stubFor(get(urlPathEqualTo(objectPath)).withHeader("Range", matching(".+")).atPriority(1) + .willReturn(aResponse().withStatus(416) + .withHeader("Content-Type", "application/xml") + .withHeader("Content-Range", "bytes */0") + .withBody("InvalidRange0"))); + stubFor(get(urlPathEqualTo(objectPath)).atPriority(2) + .willReturn(aResponse().withStatus(200).withHeader("Content-Length", "0").withHeader("ETag", "\"e\""))); + + CompletedDirectoryDownload result = tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)) + .completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(result.failedTransfers()).isEmpty(); + assertThat(destination.resolve("empty.bin")).isEmptyFile(); + assertThat(regularFiles()).containsExactly("empty.bin"); + } + + private List regularFiles() throws IOException { + try (Stream files = Files.walk(destination)) { + return files.filter(Files::isRegularFile).map(destination::relativize).map(Path::toString) + .collect(Collectors.toList()); + } + } + + private static String readOrMissing(Path path) { + try { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } + + private static S3AsyncClient s3Client(String clientType, URI endpoint) { + StaticCredentialsProvider credentials = + StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")); + if ("crt".equals(clientType)) { + return S3AsyncClient.crtBuilder() + .region(Region.US_EAST_1) + .endpointOverride(endpoint) + .forcePathStyle(true) + .credentialsProvider(credentials) + .build(); + } + return S3AsyncClient.builder() + .region(Region.US_EAST_1) + .endpointOverride(endpoint) + .forcePathStyle(true) + .multipartEnabled("java-multipart".equals(clientType)) + .credentialsProvider(credentials) + .build(); + } + + private static void stubListObjects(String... keys) { + StringBuilder contents = new StringBuilder(); + for (String key : keys) { + contents.append("").append(key).append("9"); + } + String body = "" + + "" + + "" + BUCKET + "" + keys.length + "" + + "1000false" + + contents + + ""; + stubFor(get(urlPathMatching("/" + BUCKET + "/?")) + .withQueryParam("list-type", equalTo("2")) + .willReturn(aResponse().withStatus(200).withBody(body))); + } + + private static void stubObject(String key, String content) { + stubObject(key, content.getBytes(StandardCharsets.UTF_8), 0); + } + + private static void stubObject(String key, byte[] bytes, int bodyDelayMillis) { + stubFor(head(urlPathEqualTo("/" + BUCKET + "/" + key)) + .willReturn(aResponse().withStatus(200) + .withHeader("Content-Length", String.valueOf(bytes.length)) + .withHeader("ETag", "\"" + key.replace('/', '-') + "\""))); + stubFor(get(urlPathEqualTo("/" + BUCKET + "/" + key)) + .willReturn(aResponse().withStatus(200) + .withHeader("ETag", "\"" + key.replace('/', '-') + "\"") + .withHeader("Content-Length", String.valueOf(bytes.length)) + .withHeader("Content-Range", "bytes 0-" + (bytes.length - 1) + "/" + bytes.length) + .withBody(bytes) + .withChunkedDribbleDelay(bodyDelayMillis > 0 ? 2 : 1, bodyDelayMillis))); + } +} From c04f4108bdd30eb50d6a36fe541860ffaa8ad501 Mon Sep 17 00:00:00 2001 From: RanVaknin <50976344+RanVaknin@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:09:29 -0800 Subject: [PATCH 2/4] change randomization to fit other sdks approach --- .../amazon/awssdk/transfer/s3/S3TransferManager.java | 10 +++++----- .../transfer/s3/internal/DownloadDirectoryHelper.java | 9 ++++++--- .../s3/internal/DownloadDirectoryHelperTest.java | 8 ++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index bab7446d6ee9..5a403b0eddc9 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -639,15 +639,15 @@ default DirectoryUpload uploadDirectory(Consumer * *

* Each object is first written to a temporary file in the same directory as its destination, named - * {@code .s3tmp.}, or {@code s3tmp.} when that name would be too long. Once the object is complete, the - * temporary file is moved onto the destination, and only then is the object's download, and its + * {@code .s3tmp.}, or {@code s3tmp.} when that name would be too long, where {@code } is 16 + * random hex digits. Once the object is complete, the temporary file is moved onto the destination, and only then is the + * object's download, and its * {@link software.amazon.awssdk.transfer.s3.progress.TransferListener#transferComplete transferComplete}, completed. The * move replaces the destination file rather than writing into it: a symbolic link at the destination is replaced instead of * followed, and the file gets the default permissions of a new file. A download that fails leaves an existing destination * file unchanged, and so does a cancel that arrives before the file is being moved into place. If the process stops while a - * download is in progress, its temporary file can be left behind. - * When several keys resolve to the same local file, such as {@code a/b} and {@code a//b}, the file holds the complete content - * of one of the objects. + * download is in progress, its temporary file can be left behind. When several keys resolve to the same local file, such as + * {@code a/b} and {@code a//b}, the file holds the complete content of one of the objects. * *

* The current user must have write access to all directories and files diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java index 150f5824d1c8..41e804fd0b5c 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java @@ -26,6 +26,7 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.SecureRandom; import java.util.Collection; import java.util.Queue; import java.util.UUID; @@ -61,6 +62,7 @@ public class DownloadDirectoryHelper { private static final Logger log = Logger.loggerFor(S3TransferManager.class); private static final String TEMP_FILE_INFIX = ".s3tmp."; + private static final SecureRandom TEMP_FILE_RANDOM = new SecureRandom(); private static final int MAX_FILE_NAME_BYTES = 255; private final TransferManagerConfiguration transferConfiguration; private final FileDownloader fileDownloader; @@ -246,15 +248,16 @@ private CompletableFuture doDownloadSingleFile( } /** - * Returns {@code .s3tmp.} next to the destination, or {@code s3tmp.} when that name would exceed 255 UTF-8 - * bytes, the file name limit on common file systems. + * Returns {@code .s3tmp.<16 hex digits>} next to the destination, or {@code s3tmp.<16 hex digits>} when that name would + * exceed 255 UTF-8 bytes, the file name limit on common file systems. The digits come from a {@link SecureRandom}, so the + * name can't be predicted and planted in advance. */ private static Path tempFileNextTo(Path destination) { Path fileName = destination.getFileName(); if (fileName == null) { throw new IllegalArgumentException(destination + " has no file name"); } - String suffix = TEMP_FILE_INFIX + UUID.randomUUID(); + String suffix = TEMP_FILE_INFIX + String.format("%016x", TEMP_FILE_RANDOM.nextLong()); String tempName = fileName + suffix; if (tempName.getBytes(StandardCharsets.UTF_8).length > MAX_FILE_NAME_BYTES) { tempName = suffix.substring(1); diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java index 5fc6900df07e..fe36d4d403be 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java @@ -901,13 +901,13 @@ void downloadDirectory_nameTooLongForSuffix_shouldUseSuffixOnlyWithin255Bytes(St Path writePath = writePathFor(longName); String tempName = writePath.getFileName().toString(); - assertThat(tempName).matches("s3tmp\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + assertThat(tempName).matches("s3tmp\\.[0-9a-f]{16}"); assertThat(writePath.getParent()).isEqualTo(directory); } @Test void downloadDirectory_nameThatFitsWithSuffix_shouldKeepName() throws Exception { - String name = IntStream.range(0, 212).mapToObj(i -> "a").collect(Collectors.joining()); + String name = IntStream.range(0, 232).mapToObj(i -> "a").collect(Collectors.joining()); Path writePath = writePathFor(name); @@ -918,7 +918,7 @@ void downloadDirectory_nameThatFitsWithSuffix_shouldKeepName() throws Exception @Test void downloadDirectory_nameOneByteTooLongForSuffix_shouldUseSuffixOnly() throws Exception { - String name = IntStream.range(0, 213).mapToObj(i -> "a").collect(Collectors.joining()); + String name = IntStream.range(0, 233).mapToObj(i -> "a").collect(Collectors.joining()); Path writePath = writePathFor(name); @@ -1073,7 +1073,7 @@ private static void assertDestinations(List requests, List< private static void assertTempFileFor(Path actual, Path destination) { assertThat(actual.getParent()).isEqualTo(destination.getParent()); assertThat(actual.getFileName().toString()).matches(Pattern.quote(destination.getFileName().toString()) - + "\\.s3tmp\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + + "\\.s3tmp\\.[0-9a-f]{16}"); } /** From 86f0a96dbc8a34cdc7486e8352e2e041eaec9f6d Mon Sep 17 00:00:00 2001 From: RanVaknin <50976344+RanVaknin@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:27:11 -0800 Subject: [PATCH 3/4] reject keys that resolve to the destination directory itself --- .../s3/internal/DownloadDirectoryHelper.java | 8 ++- .../internal/DownloadDirectoryHelperTest.java | 62 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java index 41e804fd0b5c..384cba075f8f 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java @@ -164,10 +164,16 @@ private Path determineDestinationPath(DownloadDirectoryRequest downloadDirectory } private void validatePath(Path destinationDirectory, Path targetPath, String key) { - if (!targetPath.toAbsolutePath().normalize().startsWith(destinationDirectory.toAbsolutePath().normalize())) { + Path normalizedDirectory = destinationDirectory.toAbsolutePath().normalize(); + Path normalizedTarget = targetPath.toAbsolutePath().normalize(); + if (!normalizedTarget.startsWith(normalizedDirectory)) { throw SdkClientException.create("Cannot download key " + key + ", its relative path resolves outside the parent directory."); } + if (normalizedTarget.equals(normalizedDirectory)) { + throw SdkClientException.create("Cannot download key " + key + + ", its relative path resolves to the destination directory itself."); + } } /** diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java index fe36d4d403be..a698c3d6bdd6 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java @@ -50,6 +50,7 @@ import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import java.util.concurrent.locks.LockSupport; import java.util.function.Consumer; @@ -1033,6 +1034,67 @@ void downloadDirectory_destinationIsSymbolicLink_shouldReplaceLinkAndLeaveItsTar assertThat(readString(linkTarget)).isEqualTo("link target"); } + @Test + void downloadDirectory_keyResolvesToDestinationDirectory_shouldReportKeyAndWriteNothing() throws Exception { + Files.createDirectories(directory); + stubSuccessfulListObjects(listObjectsHelper, "p/", "p/a.txt"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); + + CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder() + .destination(directory) + .bucket("bucket") + .listObjectsV2RequestTransformer(l -> l.prefix("p/")) + .build()).completionFuture().get(5, TimeUnit.SECONDS); + + assertThat(completed.failedTransfers()).hasSize(1); + FailedFileDownload failed = completed.failedTransfers().iterator().next(); + assertThat(failed.request().getObjectRequest().key()).isEqualTo("p/"); + assertThat(failed.exception()).isInstanceOf(SdkClientException.class) + .hasMessageContaining("resolves to the destination directory itself"); + verify(singleDownloadFunction, times(1)).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); + assertThat(Files.isDirectory(directory)).isTrue(); + assertThat(readString(directory.resolve("a.txt"))).isEmpty(); + assertNoTempFiles(directory.toAbsolutePath().getParent()); + } + + @Test + void downloadDirectory_cancelledBeforeMove_shouldDeleteTempFileAndNotCreateDestination() throws Exception { + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + CompletableFuture> directoryFuture = new CompletableFuture<>(); + AtomicReference writtenTempFile = new AtomicReference<>(); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) + .thenAnswer(invocation -> { + Path writePath = invocation.getArgument(1); + Files.createDirectories(writePath.getParent()); + Files.write(writePath, "body".getBytes(StandardCharsets.UTF_8)); + writtenTempFile.set(writePath); + // The directory download is cancelled after the object is written and before it is moved into place. + directoryFuture.get(5, TimeUnit.SECONDS).cancel(true); + try { + ((Runnable) invocation.getArgument(2)).run(); + } catch (RuntimeException e) { + return newFailedDownload(e); + } + return completedDownload(); + }); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + directoryFuture.complete(directoryDownload.completionFuture()); + + assertThatThrownBy(() -> directoryDownload.completionFuture().get(5, TimeUnit.SECONDS)) + .isInstanceOf(CancellationException.class); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while ((writtenTempFile.get() == null || Files.exists(writtenTempFile.get())) && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(writtenTempFile.get()).isNotNull(); + assertThat(Files.exists(writtenTempFile.get())).isFalse(); + assertThat(Files.exists(directory.resolve("a.txt"))).isFalse(); + } + private Map stubFileWritingDownloads() { Map downloads = new ConcurrentHashMap<>(); when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { From 29c7004352d0368da56be339d1cf9d7f8ac204d6 Mon Sep 17 00:00:00 2001 From: RanVaknin <50976344+RanVaknin@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:07:47 -0800 Subject: [PATCH 4/4] Refactor --- .../s3/internal/DownloadDirectoryHelper.java | 22 +-- .../s3/internal/GenericS3TransferManager.java | 5 +- .../internal/DownloadDirectoryHelperTest.java | 138 ------------------ ...DownloadDirectoryTempFileWireMockTest.java | 128 ---------------- 4 files changed, 5 insertions(+), 288 deletions(-) diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java index 384cba075f8f..1f2022a0eea1 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -254,9 +253,7 @@ private CompletableFuture doDownloadSingleFile( } /** - * Returns {@code .s3tmp.<16 hex digits>} next to the destination, or {@code s3tmp.<16 hex digits>} when that name would - * exceed 255 UTF-8 bytes, the file name limit on common file systems. The digits come from a {@link SecureRandom}, so the - * name can't be predicted and planted in advance. + * Returns {@code .s3tmp.<16 hex>} next to the destination, or {@code s3tmp.<16 hex>} if that exceeds 255 bytes. */ private static Path tempFileNextTo(Path destination) { Path fileName = destination.getFileName(); @@ -272,8 +269,7 @@ private static Path tempFileNextTo(Path destination) { } /** - * Moves a completed download onto its destination, replacing any file already there. Returns the failure to report for the - * key, or null when the key succeeded. + * Moves a completed download onto its destination, returning the failure to report for the key, or null on success. */ private static SdkClientException moveIntoPlace(CompletableFuture returnFuture, Path tempFile, @@ -283,17 +279,8 @@ private static SdkClientException moveIntoPlace(CompletableFuture downloads = stubFileWritingDownloads(); - - DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( - DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); - - awaitStarted(downloads, "a.txt"); - downloads.get("a.txt").succeed("new"); - - assertThat(directoryDownload.completionFuture().get(5, TimeUnit.SECONDS).failedTransfers()).isEmpty(); - assertThat(readString(directory.resolve("a.txt"))).isEqualTo("new"); - assertNoTempFiles(directory); - } - @Test void downloadDirectory_cancelledDuringDownload_shouldRemoveTempFileAndNotCreateDestination() throws Exception { stubSuccessfulListObjects(listObjectsHelper, "a.txt"); @@ -893,19 +874,6 @@ void downloadDirectory_cancelledDuringDownload_shouldRemoveTempFileAndNotCreateD assertNoTempFiles(directory); } - @ParameterizedTest - @ValueSource(strings = {"a", "\u00e9", "\u6f22", "\ud83d\ude00"}) - void downloadDirectory_nameTooLongForSuffix_shouldUseSuffixOnlyWithin255Bytes(String character) throws Exception { - int bytesPerCharacter = character.getBytes(StandardCharsets.UTF_8).length; - String longName = IntStream.range(0, 255 / bytesPerCharacter).mapToObj(i -> character).collect(Collectors.joining()); - - Path writePath = writePathFor(longName); - - String tempName = writePath.getFileName().toString(); - assertThat(tempName).matches("s3tmp\\.[0-9a-f]{16}"); - assertThat(writePath.getParent()).isEqualTo(directory); - } - @Test void downloadDirectory_nameThatFitsWithSuffix_shouldKeepName() throws Exception { String name = IntStream.range(0, 232).mapToObj(i -> "a").collect(Collectors.joining()); @@ -936,30 +904,6 @@ private Path writePathFor(String key) throws Exception { return writePathCaptor.getValue(); } - @Test - void downloadDirectory_destinationHasNoFileName_shouldReportThatKeyAndDownloadOthers() throws Exception { - stubSuccessfulListObjects(listObjectsHelper, "good.txt", "bad.txt"); - when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); - Path root = directory.toAbsolutePath().getRoot(); - - CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( - DownloadDirectoryRequest.builder() - .destination(directory) - .bucket("bucket") - .downloadFileRequestTransformer(f -> { - if ("bad.txt".equals(f.build().getObjectRequest().key())) { - f.destination(root); - } - }) - .build()).completionFuture().get(5, TimeUnit.SECONDS); - - assertThat(completed.failedTransfers()).hasSize(1); - FailedFileDownload failed = completed.failedTransfers().iterator().next(); - assertThat(failed.request().getObjectRequest().key()).isEqualTo("bad.txt"); - assertThat(failed.exception()).isInstanceOf(SdkClientException.class).hasMessageContaining("not a path to a file"); - verify(singleDownloadFunction, times(1)).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); - } - /** * Answers like a real download: the write path exists once the download is started, and a download that has already * succeeded has run its before-completion step. Returns the downloads in order, then keeps returning the last one. @@ -988,52 +932,6 @@ private Answer writingWritePath(FileDownload... downloads) { }; } - @Test - void downloadDirectory_downloadSucceedsWithoutWritingFile_shouldReportKeyAndKeepExistingFile() throws Exception { - Files.createDirectories(directory); - Files.write(directory.resolve("a.txt"), "existing".getBytes(StandardCharsets.UTF_8)); - stubSuccessfulListObjects(listObjectsHelper, "a.txt"); - when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) - .thenAnswer(invocation -> { - try { - ((Runnable) invocation.getArgument(2)).run(); - } catch (RuntimeException e) { - return newFailedDownload(e); - } - return completedDownload(); - }); - - CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( - DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()).completionFuture() - .get(5, TimeUnit.SECONDS); - - assertThat(completed.failedTransfers()).hasSize(1); - assertThat(completed.failedTransfers().iterator().next().exception()) - .isInstanceOf(SdkClientException.class).hasMessageContaining("completed without writing"); - assertThat(readString(directory.resolve("a.txt"))).isEqualTo("existing"); - } - - @Test - void downloadDirectory_destinationIsSymbolicLink_shouldReplaceLinkAndLeaveItsTargetUnchanged() throws Exception { - Files.createDirectories(directory); - Path linkTarget = fs.getPath("elsewhere.txt"); - Files.write(linkTarget, "link target".getBytes(StandardCharsets.UTF_8)); - Files.createSymbolicLink(directory.resolve("a.txt"), linkTarget.toAbsolutePath()); - stubSuccessfulListObjects(listObjectsHelper, "a.txt"); - Map downloads = stubFileWritingDownloads(); - - DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( - DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); - - awaitStarted(downloads, "a.txt"); - downloads.get("a.txt").succeed("new"); - - assertThat(directoryDownload.completionFuture().get(5, TimeUnit.SECONDS).failedTransfers()).isEmpty(); - assertThat(Files.isSymbolicLink(directory.resolve("a.txt"))).isFalse(); - assertThat(readString(directory.resolve("a.txt"))).isEqualTo("new"); - assertThat(readString(linkTarget)).isEqualTo("link target"); - } - @Test void downloadDirectory_keyResolvesToDestinationDirectory_shouldReportKeyAndWriteNothing() throws Exception { Files.createDirectories(directory); @@ -1059,42 +957,6 @@ void downloadDirectory_keyResolvesToDestinationDirectory_shouldReportKeyAndWrite assertNoTempFiles(directory.toAbsolutePath().getParent()); } - @Test - void downloadDirectory_cancelledBeforeMove_shouldDeleteTempFileAndNotCreateDestination() throws Exception { - stubSuccessfulListObjects(listObjectsHelper, "a.txt"); - CompletableFuture> directoryFuture = new CompletableFuture<>(); - AtomicReference writtenTempFile = new AtomicReference<>(); - when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) - .thenAnswer(invocation -> { - Path writePath = invocation.getArgument(1); - Files.createDirectories(writePath.getParent()); - Files.write(writePath, "body".getBytes(StandardCharsets.UTF_8)); - writtenTempFile.set(writePath); - // The directory download is cancelled after the object is written and before it is moved into place. - directoryFuture.get(5, TimeUnit.SECONDS).cancel(true); - try { - ((Runnable) invocation.getArgument(2)).run(); - } catch (RuntimeException e) { - return newFailedDownload(e); - } - return completedDownload(); - }); - - DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( - DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); - directoryFuture.complete(directoryDownload.completionFuture()); - - assertThatThrownBy(() -> directoryDownload.completionFuture().get(5, TimeUnit.SECONDS)) - .isInstanceOf(CancellationException.class); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while ((writtenTempFile.get() == null || Files.exists(writtenTempFile.get())) && System.nanoTime() < deadline) { - Thread.sleep(10); - } - assertThat(writtenTempFile.get()).isNotNull(); - assertThat(Files.exists(writtenTempFile.get())).isFalse(); - assertThat(Files.exists(directory.resolve("a.txt"))).isFalse(); - } - private Map stubFileWritingDownloads() { Map downloads = new ConcurrentHashMap<>(); when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java index 611faa60b76d..96915b865ec8 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java @@ -19,12 +19,10 @@ import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.head; -import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -51,7 +49,6 @@ import software.amazon.awssdk.testutils.FileUtils; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload; -import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.progress.TransferListener; @@ -79,64 +76,6 @@ void tearDown() { FileUtils.cleanUpTestDirectory(destination); } - @ParameterizedTest(name = "client={0}") - @ValueSource(strings = {"java", "java-multipart", "crt"}) - void downloadDirectory_withPerFileListener_listenerSeesRealDestinationAndNoTempFileRemains(String clientType, - WireMockRuntimeInfo wm) - throws Exception { - s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); - tm = S3TransferManager.builder().s3Client(s3).build(); - stubListObjects("a.txt", "sub/b.txt"); - stubObject("a.txt", "content-a"); - stubObject("sub/b.txt", "content-b"); - - List initiated = new CopyOnWriteArrayList<>(); - List completed = new CopyOnWriteArrayList<>(); - List contentAtCompletion = new CopyOnWriteArrayList<>(); - TransferListener listener = new TransferListener() { - @Override - public void transferInitiated(Context.TransferInitiated context) { - initiated.add(((DownloadFileRequest) context.request()).destination()); - } - - @Override - public void transferComplete(Context.TransferComplete context) { - Path completedDestination = ((DownloadFileRequest) context.request()).destination(); - contentAtCompletion.add(readOrMissing(completedDestination)); - completed.add(completedDestination); - } - }; - - CompletedDirectoryDownload result = - tm.downloadDirectory(r -> r.bucket(BUCKET) - .destination(destination) - .downloadFileRequestTransformer(f -> f.addTransferListener(listener))) - .completionFuture().get(30, TimeUnit.SECONDS); - - assertThat(result.failedTransfers()).isEmpty(); - Path a = destination.resolve("a.txt"); - Path b = destination.resolve("sub").resolve("b.txt"); - assertThat(initiated).containsExactlyInAnyOrder(a, b); - // The multipart client does not deliver transferComplete against these stubs, for downloadFile as well. - if (!"java-multipart".equals(clientType)) { - // transferComplete can be delivered after the directory download's future completes, but only once the file - // is in place. - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (completed.size() < 2 && System.nanoTime() < deadline) { - Thread.sleep(10); - } - assertThat(completed).containsExactlyInAnyOrder(a, b); - assertThat(contentAtCompletion).containsExactlyInAnyOrder("content-a", "content-b"); - } - assertThat(a).hasContent("content-a"); - assertThat(b).hasContent("content-b"); - try (Stream files = Files.walk(destination)) { - assertThat(files.filter(Files::isRegularFile).map(destination::relativize).map(Path::toString) - .collect(Collectors.toList())) - .containsExactlyInAnyOrder("a.txt", "sub" + destination.getFileSystem().getSeparator() + "b.txt"); - } - } - @ParameterizedTest(name = "client={0}") @ValueSource(strings = {"java", "java-multipart", "crt"}) void downloadDirectory_keysResolvingToSameFile_fileHoldsOneCompleteObject(String clientType, WireMockRuntimeInfo wm) @@ -220,73 +159,6 @@ public void transferComplete(Context.TransferComplete context) { } } - @ParameterizedTest(name = "client={0}") - @ValueSource(strings = {"java", "java-multipart", "crt"}) - void downloadFileTo_beforeCompletionThrows_downloadAndListenerFail(String clientType, WireMockRuntimeInfo wm) - throws Exception { - s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); - tm = S3TransferManager.builder().s3Client(s3).build(); - stubObject("a.txt", "content-a"); - RuntimeException moveFailure = new RuntimeException("move failed"); - List failures = new CopyOnWriteArrayList<>(); - AtomicBoolean completed = new AtomicBoolean(); - TransferListener listener = new TransferListener() { - @Override - public void transferComplete(Context.TransferComplete context) { - completed.set(true); - } - - @Override - public void transferFailed(Context.TransferFailed context) { - failures.add(context.exception()); - } - }; - DownloadFileRequest request = DownloadFileRequest.builder() - .getObjectRequest(g -> g.bucket(BUCKET).key("a.txt")) - .destination(destination.resolve("a.txt")) - .addTransferListener(listener) - .build(); - - CompletableFuture download = - ((GenericS3TransferManager) tm).downloadFileTo(request, destination.resolve("a.txt.tmp"), () -> { - throw moveFailure; - }).completionFuture(); - - assertThatThrownBy(() -> download.get(30, TimeUnit.SECONDS)).hasCause(moveFailure); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (failures.isEmpty() && System.nanoTime() < deadline) { - Thread.sleep(10); - } - assertThat(failures).containsExactly(moveFailure); - assertThat(completed).isFalse(); - } - - @ParameterizedTest(name = "client={0}") - @ValueSource(strings = {"java", "java-multipart", "crt"}) - void downloadDirectory_emptyObject_createsEmptyFile(String clientType, WireMockRuntimeInfo wm) throws Exception { - s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); - tm = S3TransferManager.builder().s3Client(s3).build(); - stubListObjects("empty.bin"); - String objectPath = "/" + BUCKET + "/empty.bin"; - stubFor(head(urlPathEqualTo(objectPath)) - .willReturn(aResponse().withStatus(200).withHeader("Content-Length", "0").withHeader("ETag", "\"e\""))); - // S3 rejects a ranged GET of an empty object, and the CRT-based client retries such a request without the range. - stubFor(get(urlPathEqualTo(objectPath)).withHeader("Range", matching(".+")).atPriority(1) - .willReturn(aResponse().withStatus(416) - .withHeader("Content-Type", "application/xml") - .withHeader("Content-Range", "bytes */0") - .withBody("InvalidRange0"))); - stubFor(get(urlPathEqualTo(objectPath)).atPriority(2) - .willReturn(aResponse().withStatus(200).withHeader("Content-Length", "0").withHeader("ETag", "\"e\""))); - - CompletedDirectoryDownload result = tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)) - .completionFuture().get(30, TimeUnit.SECONDS); - - assertThat(result.failedTransfers()).isEmpty(); - assertThat(destination.resolve("empty.bin")).isEmptyFile(); - assertThat(regularFiles()).containsExactly("empty.bin"); - } - private List regularFiles() throws IOException { try (Stream files = Files.walk(destination)) { return files.filter(Files::isRegularFile).map(destination::relativize).map(Path::toString)