diff --git a/.changes/next-release/bugfix-S3TransferManager-3eaf213.json b/.changes/next-release/bugfix-S3TransferManager-3eaf213.json new file mode 100644 index 000000000000..1f79df4b081e --- /dev/null +++ b/.changes/next-release/bugfix-S3TransferManager-3eaf213.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "S3 Transfer Manager", + "contributor": "", + "description": "Fixed an issue in `downloadDirectory` where keys that resolve to the same local file could produce a file that mixes the bytes of several objects, by writing each object to a temporary file next to its destination and moving it into place once complete, which also leaves an existing file unchanged when its download fails." +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 44827d2fd4da..5a403b0eddc9 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -638,6 +638,18 @@ default DirectoryUpload uploadDirectory(Consumer * object content. * *

+ * Each object is first written to a temporary file in the same directory as its destination, named + * {@code .s3tmp.}, or {@code s3tmp.} when that name would be too long, where {@code } is 16 + * random hex digits. Once the object is complete, the temporary file is moved onto the destination, and only then is the + * object's download, and its + * {@link software.amazon.awssdk.transfer.s3.progress.TransferListener#transferComplete transferComplete}, completed. The + * move replaces the destination file rather than writing into it: a symbolic link at the destination is replaced instead of + * followed, and the file gets the default permissions of a new file. A download that fails leaves an existing destination + * file unchanged, and so does a cancel that arrives before the file is being moved into place. If the process stops while a + * download is in progress, its temporary file can be left behind. When several keys resolve to the same local file, such as + * {@code a/b} and {@code a//b}, the file holds the complete content of one of the objects. + * + *

* The current user must have write access to all directories and files * *

diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java index 65e1f80624bc..b5896b45a665 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtS3TransferManager.java @@ -209,19 +209,20 @@ Download doDownloadWithDirectBufferPool(DownloadRequest returnFuture = new CompletableFuture<>(); - initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture); + initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture, beforeCompletion); return new CrtFileDownload(returnFuture, progressUpdater.progress(), observable, () -> crtDownloadRequest, null); } @@ -268,13 +269,14 @@ FileDownload doResumeDownloadFile(ResumableFileDownload resumableFileDownload) { : ResponseFileOption.CREATE_OR_APPEND; TransferProgressUpdater progressUpdater = new TransferProgressUpdater(newDownloadFileRequest, null); - DownloadFileRequest crtDownloadRequest = crtDownloadFileRequest(newDownloadFileRequest, observable, + DownloadFileRequest crtDownloadRequest = crtDownloadFileRequest(newDownloadFileRequest, + newDownloadFileRequest.destination(), observable, progressUpdater, responseFileOption); newDownloadFileRequestFuture.complete(crtDownloadRequest); log.debug(() -> LogSanitizer.escapeControlCharacters("Sending downloadFileRequest " + crtDownloadRequest)); - initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture); + initiateCrtDownload(crtDownloadRequest, progressUpdater, returnFuture, NO_OP); progressFuture.complete(progressUpdater.progress()); }).exceptionally(throwable -> { handleException(returnFuture, progressFuture, newDownloadFileRequestFuture, throwable); @@ -303,7 +305,8 @@ private static boolean hasCompletedParts(ResumableFileDownload resumableFileDown private void initiateCrtDownload(DownloadFileRequest downloadRequest, TransferProgressUpdater progressUpdater, - CompletableFuture returnFuture) { + CompletableFuture returnFuture, + Runnable beforeCompletion) { try { progressUpdater.transferInitiated(); AsyncResponseTransformer responseTransformer = @@ -318,26 +321,28 @@ private void initiateCrtDownload(DownloadFileRequest downloadRequest, // Forward download cancellation to CRT future CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); - CompletableFutureUtils.forwardTransformedResultTo(crtFuture, returnFuture, - res -> CompletedFileDownload.builder() - .response(res) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(crtFuture, returnFuture, res -> { + CompletedFileDownload completed = CompletedFileDownload.builder().response(res).build(); + beforeCompletion.run(); + return completed; + }); } catch (Throwable throwable) { returnFuture.completeExceptionally(throwable); } } /** - * Attaches the execution attributes that tell the CRT-based client to write the response body straight to the destination - * file, together with the progress listener and pause observable needed to report progress on and pause that transfer. + * Attaches the execution attributes that tell the CRT-based client to write the response body straight to {@code writePath}, + * together with the progress listener and pause observable needed to report progress on and pause that transfer. */ private DownloadFileRequest crtDownloadFileRequest(DownloadFileRequest downloadRequest, + Path writePath, S3MetaRequestPauseObservable observable, TransferProgressUpdater progressUpdater, ResponseFileOption responseFileOption) { GetObjectRequest getObjectRequest = attachSdkAttribute( downloadRequest.getObjectRequest(), - b -> b.putExecutionAttribute(RESPONSE_FILE_PATH, downloadRequest.destination()) + b -> b.putExecutionAttribute(RESPONSE_FILE_PATH, writePath) .putExecutionAttribute(RESPONSE_FILE_OPTION, responseFileOption)); GetObjectRequest crtGetObjectRequest = attachCrtSdkAttribute( diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java index 7a47de57fa0e..1f2022a0eea1 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelper.java @@ -19,10 +19,13 @@ import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.DEFAULT_PREFIX; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.SecureRandom; import java.util.Collection; import java.util.Queue; import java.util.UUID; @@ -57,16 +60,19 @@ @SdkInternalApi public class DownloadDirectoryHelper { private static final Logger log = Logger.loggerFor(S3TransferManager.class); + private static final String TEMP_FILE_INFIX = ".s3tmp."; + private static final SecureRandom TEMP_FILE_RANDOM = new SecureRandom(); + private static final int MAX_FILE_NAME_BYTES = 255; private final TransferManagerConfiguration transferConfiguration; - private final Function downloadFileFunction; + private final FileDownloader fileDownloader; private final ListObjectsHelper listObjectsHelper; public DownloadDirectoryHelper(TransferManagerConfiguration transferConfiguration, ListObjectsHelper listObjectsHelper, - Function downloadFileFunction) { + FileDownloader fileDownloader) { this.transferConfiguration = transferConfiguration; - this.downloadFileFunction = downloadFileFunction; + this.fileDownloader = fileDownloader; this.listObjectsHelper = listObjectsHelper; } @@ -157,12 +163,23 @@ private Path determineDestinationPath(DownloadDirectoryRequest downloadDirectory } private void validatePath(Path destinationDirectory, Path targetPath, String key) { - if (!targetPath.toAbsolutePath().normalize().startsWith(destinationDirectory.toAbsolutePath().normalize())) { + Path normalizedDirectory = destinationDirectory.toAbsolutePath().normalize(); + Path normalizedTarget = targetPath.toAbsolutePath().normalize(); + if (!normalizedTarget.startsWith(normalizedDirectory)) { throw SdkClientException.create("Cannot download key " + key + ", its relative path resolves outside the parent directory."); } + if (normalizedTarget.equals(normalizedDirectory)) { + throw SdkClientException.create("Cannot download key " + key + + ", its relative path resolves to the destination directory itself."); + } } + /** + * Downloads the object to a temporary file next to its destination and moves it into place once it is complete. Keys such + * as "a/b" and "a//b", or "A" and "a" on a case-insensitive file system, resolve to the same file, and writing to it + * directly would let their concurrent downloads mix bytes or delete each other's file. + */ private CompletableFuture doDownloadSingleFile( CompletableFuture returnFuture, DownloadDirectoryRequest downloadDirectoryRequest, @@ -183,25 +200,50 @@ private CompletableFuture doDownloadSingleFile( } DownloadFileRequest downloadFileRequest = downloadFileRequest(downloadDirectoryRequest, s3Object, destinationPath); + Path destination = downloadFileRequest.destination(); + Path tempFile; + try { + tempFile = tempFileNextTo(destination); + } catch (RuntimeException e) { + SdkClientException failure = SdkClientException.create("Cannot download key " + s3Object.key() + " to " + + destination + ", it is not a path to a file", e); + failedFileDownloads.add(FailedFileDownload.builder().exception(failure).request(downloadFileRequest).build()); + return CompletableFutureUtils.failedFuture(failure); + } try { log.debug(() -> LogSanitizer.escapeControlCharacters("Sending download request " + downloadFileRequest)); createParentDirectoriesIfNeeded(destinationPath); + // The move runs inside the download, so the download and its listeners complete only once the file is in place. + Runnable moveIntoPlaceOnSuccess = () -> { + SdkClientException failure = moveIntoPlace(returnFuture, tempFile, destination); + if (failure != null) { + throw failure; + } + }; CompletableFuture executionFuture = - downloadFileFunction.apply(downloadFileRequest).completionFuture(); - CompletableFuture future = executionFuture.whenComplete((r, t) -> { - if (t != null) { - failedFileDownloads.add(FailedFileDownload.builder() - .exception(t instanceof CompletionException ? t.getCause() : t) - .request(downloadFileRequest) - .build()); + fileDownloader.download(downloadFileRequest, tempFile, moveIntoPlaceOnSuccess).completionFuture(); + CompletableFuture future = new CompletableFuture<>(); + // Attached to executionFuture rather than to the returned future, so the temporary file is also removed when the + // directory download cancels the returned future. + executionFuture.whenComplete((r, t) -> { + if (t == null) { + future.complete(r); + return; } + deleteQuietly(tempFile); + failedFileDownloads.add(FailedFileDownload.builder() + .exception(t instanceof CompletionException ? t.getCause() : t) + .request(downloadFileRequest) + .build()); + future.completeExceptionally(t); }); CompletableFutureUtils.forwardExceptionTo(future, executionFuture); return future; } catch (Throwable throwable) { + deleteQuietly(tempFile); failedFileDownloads.add(FailedFileDownload.builder() .exception(throwable) .request(downloadFileRequest) @@ -210,6 +252,50 @@ private CompletableFuture doDownloadSingleFile( } } + /** + * Returns {@code .s3tmp.<16 hex>} next to the destination, or {@code s3tmp.<16 hex>} if that exceeds 255 bytes. + */ + private static Path tempFileNextTo(Path destination) { + Path fileName = destination.getFileName(); + if (fileName == null) { + throw new IllegalArgumentException(destination + " has no file name"); + } + String suffix = TEMP_FILE_INFIX + String.format("%016x", TEMP_FILE_RANDOM.nextLong()); + String tempName = fileName + suffix; + if (tempName.getBytes(StandardCharsets.UTF_8).length > MAX_FILE_NAME_BYTES) { + tempName = suffix.substring(1); + } + return destination.resolveSibling(tempName); + } + + /** + * Moves a completed download onto its destination, returning the failure to report for the key, or null on success. + */ + private static SdkClientException moveIntoPlace(CompletableFuture returnFuture, + Path tempFile, + Path destination) { + if (returnFuture.isCompletedExceptionally()) { + deleteQuietly(tempFile); + return SdkClientException.create("Download was cancelled before the file could be moved to " + destination); + } + + try { + Files.move(tempFile, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + return null; + } catch (IOException | RuntimeException e) { + deleteQuietly(tempFile); + return SdkClientException.create("Failed to move the downloaded file to " + destination, e); + } + } + + private static void deleteQuietly(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException | RuntimeException e) { + log.debug(() -> LogSanitizer.escapeControlCharacters("Failed to delete temporary file " + path), e); + } + } + /** * Reports a key that can't be mapped to a path inside the destination directory as a single failed download, without @@ -283,4 +369,12 @@ private static void createParentDirectoriesIfNeeded(Path destinationPath) { throw SdkClientException.create("Failed to create parent directories for " + destinationPath, e); } } + + /** + * Downloads one object into {@code writePath}, running {@code beforeCompletion} before the download completes. + */ + @FunctionalInterface + interface FileDownloader { + FileDownload download(DownloadFileRequest request, Path writePath, Runnable beforeCompletion); + } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 1f7033417f1c..552eae454cef 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -100,6 +100,7 @@ @SdkInternalApi class GenericS3TransferManager implements S3TransferManager { + static final Runnable NO_OP = () -> { }; private static final Logger log = Logger.loggerFor(S3TransferManager.class); private static final PauseResumeHelper PAUSE_RESUME_HELPER = new PauseResumeHelper(); private final S3AsyncClient s3AsyncClient; @@ -119,7 +120,7 @@ class GenericS3TransferManager implements S3TransferManager { ListObjectsHelper listObjectsHelper = new ListObjectsHelper(s3AsyncClient::listObjectsV2); downloadDirectoryHelper = new DownloadDirectoryHelper(transferConfiguration, listObjectsHelper, - this::downloadFile); + this::downloadFileTo); this.isDefaultS3AsyncClient = isDefaultS3AsyncClient; } @@ -453,6 +454,14 @@ public final FileDownload downloadFile(DownloadFileRequest downloadRequest) { * May be overridden by subclasses to provide customized behavior */ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { + return downloadFileTo(downloadRequest, downloadRequest.destination(), NO_OP); + } + + /** + * Downloads the object into {@code writePath}, running {@code beforeCompletion} before the download and listeners complete. + */ + FileDownload downloadFileTo(DownloadFileRequest downloadRequest, Path writePath, Runnable beforeCompletion) { + Validate.paramNotNull(writePath, "writePath"); GetObjectRequest getObjectRequestWithAttributes = attachSdkAttribute( downloadRequest.getObjectRequest(), b -> b.putExecutionAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT, new MultipartDownloadResumeContext())); @@ -460,12 +469,11 @@ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { downloadRequest.copy(downloadFileRequest -> downloadFileRequest.getObjectRequest(getObjectRequestWithAttributes)); AsyncResponseTransformer responseTransformer = - AsyncResponseTransformer.toFile(downloadFileRequestWithAttributes.destination(), - FileTransformerConfiguration.defaultCreateOrReplaceExisting()); + AsyncResponseTransformer.toFile(writePath, FileTransformerConfiguration.defaultCreateOrReplaceExisting()); CompletableFuture returnFuture = new CompletableFuture<>(); TransferProgressUpdater progressUpdater = initiateDownload( - downloadFileRequestWithAttributes, responseTransformer, returnFuture); + downloadFileRequestWithAttributes, responseTransformer, returnFuture, beforeCompletion); return new DefaultFileDownload(returnFuture, progressUpdater.progress(), () -> downloadFileRequestWithAttributes, null); } @@ -473,7 +481,8 @@ FileDownload doDownloadFile(DownloadFileRequest downloadRequest) { private TransferProgressUpdater initiateDownload( DownloadFileRequest downloadRequest, AsyncResponseTransformer responseTransformer, - CompletableFuture returnFuture) { + CompletableFuture returnFuture, + Runnable beforeCompletion) { TransferProgressUpdater progressUpdater = new TransferProgressUpdater(downloadRequest, null); try { progressUpdater.transferInitiated(); @@ -491,10 +500,11 @@ private TransferProgressUpdater initiateDownload( // Forward download cancellation to future CompletableFutureUtils.forwardExceptionTo(returnFuture, future); - CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, - res -> CompletedFileDownload.builder() - .response(res) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, res -> { + CompletedFileDownload completed = CompletedFileDownload.builder().response(res).build(); + beforeCompletion.run(); + return completed; + }); } catch (Throwable throwable) { returnFuture.completeExceptionally(throwable); } @@ -551,7 +561,7 @@ FileDownload doResumeDownloadFile(ResumableFileDownload resumableFileDownload) { TransferProgressUpdater progressUpdater = initiateDownload(newDownloadFileRequest, requestPair.right(), - returnFuture); + returnFuture, NO_OP); progressFuture.complete(progressUpdater.progress()); }).exceptionally(throwable -> { handleException(returnFuture, progressFuture, newDownloadFileRequestFuture, throwable); diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java index 7ead9e8f3613..39f1ecaf0dda 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryHelperTest.java @@ -28,26 +28,31 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; import java.time.Duration; import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import java.util.concurrent.locks.LockSupport; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.logging.log4j.Level; @@ -60,6 +65,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.Answer; import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.model.EncodingType; @@ -88,7 +94,7 @@ public class DownloadDirectoryHelperTest { private static final String DIRECTORY_NAME = "test"; private FileSystem fs; private Path directory; - private Function singleDownloadFunction; + private DownloadDirectoryHelper.FileDownloader singleDownloadFunction; private DownloadDirectoryHelper downloadDirectoryHelper; private ListObjectsHelper listObjectsHelper; @@ -97,7 +103,7 @@ public void methodSetup() { fs = Jimfs.newFileSystem(); directory = fs.getPath("test"); listObjectsHelper = mock(ListObjectsHelper.class); - singleDownloadFunction = mock(Function.class); + singleDownloadFunction = mock(DownloadDirectoryHelper.FileDownloader.class); downloadDirectoryHelper = new DownloadDirectoryHelper(TransferManagerConfiguration.builder().build(), listObjectsHelper, singleDownloadFunction); @@ -122,7 +128,7 @@ void downloadDirectory_allDownloadsSucceed_failedDownloadsShouldBeEmpty() throws FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -133,7 +139,7 @@ void downloadDirectory_allDownloadsSucceed_failedDownloadsShouldBeEmpty() throws CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).element(0).satisfies(d -> assertThat(d.getObjectRequest().key()).isEqualTo( @@ -151,7 +157,7 @@ void downloadDirectory_containsFolderObjects_shouldSkip() throws Exception { FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -162,7 +168,7 @@ void downloadDirectory_containsFolderObjects_shouldSkip() throws Exception { CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); List allValues = argumentCaptor.getAllValues(); @@ -194,7 +200,7 @@ void invalidKey_shouldReportFailedDownloadAndContinue(String testingString) thro assertThat(completed.failedTransfers()).hasSize(1); assertRefused(completed.failedTransfers().get(0), destinationDirectory, key); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); } @Test @@ -212,7 +218,7 @@ void downloadDirectory_destinationDoesNotExistAndOnlyKeyRefused_shouldReportFail assertThat(completed.failedTransfers()).hasSize(1); assertRefused(completed.failedTransfers().get(0), directory, "../escape.txt"); assertThat(Files.exists(directory)).isFalse(); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); } @Test @@ -259,7 +265,7 @@ void downloadDirectory_manyRefusedKeysInOneListingPage_shouldReportEachWithoutSt assertThat(completed.failedTransfers()).allSatisfy(f -> assertThat(f.exception()) .isInstanceOf(SdkClientException.class) .hasMessageContaining("Cannot download key")); - verify(singleDownloadFunction, never()).apply(any(DownloadFileRequest.class)); + verify(singleDownloadFunction, never()).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); assertThat(Files.exists(directory)).isFalse(); } @@ -267,7 +273,7 @@ void downloadDirectory_manyRefusedKeysInOneListingPage_shouldReportEachWithoutSt void downloadDirectory_refusedKeyAmongBenignKeys_shouldDownloadOthersAndReportOneFailure() throws Exception { stubSuccessfulListObjects(listObjectsHelper, "a.txt", "b.txt", "c/../../escape.txt", "d.txt"); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); DownloadDirectoryHelper helper = new DownloadDirectoryHelper(TransferManagerConfiguration.builder().transferDirectoryMaxConcurrency(1).build(), listObjectsHelper, @@ -295,7 +301,7 @@ void downloadDirectory_keyInvalidOnWindowsFileSystem_shouldReportFailedDownloadA stubSuccessfulListObjects(listObjectsHelper, "ok.txt", invalidKey); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -321,7 +327,7 @@ void downloadDirectory_keysWithControlCharacters_debugLogsAreOneEscapedLineEach( String refusedKey = "../escape\r\nFORGED level=ERROR"; stubSuccessfulListObjects(listObjectsHelper, sentKey, refusedKey); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); CompletedDirectoryDownload completed; List messages; @@ -378,7 +384,7 @@ void downloadDirectory_cancel_shouldCancelAllFutures() throws Exception { AtomicInteger callCount = new AtomicInteger(0); CountDownLatch bothStarted = new CountDownLatch(2); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenAnswer(invocation -> { + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { bothStarted.countDown(); return callCount.incrementAndGet() == 1 ? fileDownload : fileDownload2; }); @@ -409,7 +415,7 @@ void downloadDirectory_partialSuccess_shouldProvideFailedDownload() throws Excep SdkClientException exception = SdkClientException.create("failed"); FileDownload fileDownload2 = newFailedDownload(exception); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -431,7 +437,7 @@ void downloadDirectory_withFilter_shouldHonorFilter() throws Exception { FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() @@ -443,7 +449,7 @@ void downloadDirectory_withFilter_shouldHonorFilter() throws Exception { CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(1)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(1)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).element(0).satisfies(d -> assertThat(d.getObjectRequest().key()).isEqualTo( @@ -457,8 +463,8 @@ void downloadDirectory_withDownloadRequestTransformer_shouldTransform() throws E FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); - Path newDestination = Paths.get("/new/path"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); + Path newDestination = directory.getFileSystem().getPath("/new/path"); GetObjectRequest newGetObjectRequest = GetObjectRequest.builder().build(); List newTransferListener = Arrays.asList(LoggingTransferListener.create()); @@ -475,7 +481,7 @@ void downloadDirectory_withDownloadRequestTransformer_shouldTransform() throws E CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - verify(singleDownloadFunction, times(2)).apply(argumentCaptor.capture()); + verify(singleDownloadFunction, times(2)).download(argumentCaptor.capture(), any(Path.class), any(Runnable.class)); assertThat(completedDirectoryDownload.failedTransfers()).isEmpty(); assertThat(argumentCaptor.getAllValues()).allSatisfy(d -> { @@ -495,7 +501,7 @@ void downloadDirectory_withListObjectsRequestTransformer_shouldTransform() throw EncodingType newEncodingType = EncodingType.URL; int newMaxKeys = 10; - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -525,8 +531,8 @@ void downloadDirectory_shouldRecursivelyDownload(FileSystem jimfs) { stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -563,8 +569,8 @@ void downloadDirectory_withPrefix_shouldStripPrefixInDestinationPath(FileSystem stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -579,16 +585,13 @@ void downloadDirectory_withPrefix_shouldStripPrefixInDestinationPath(FileSystem assertThat(actualRequests.size()).isEqualTo(keys.length); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); String jimfsSeparator = jimfs.getSeparator(); List expectedPaths = Arrays.asList("image.jpg", "title.jpg", "ghi/xyz.txt").stream() .map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace("/",jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } @ParameterizedTest @@ -600,8 +603,8 @@ void downloadDirectory_containsObjectWithPrefixInIt_shouldInclude(FileSystem jim stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())) - .thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -615,16 +618,13 @@ void downloadDirectory_containsObjectWithPrefixInIt_shouldInclude(FileSystem jim assertThat(actualRequests.size()).isEqualTo(keys.length); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); String jimfsSeparator = jimfs.getSeparator(); List expectedPaths = Arrays.asList("def/image.jpg", "def/title.jpg", "abcd").stream() .map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace("/",jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } @ParameterizedTest @@ -636,7 +636,7 @@ void downloadDirectory_withDelimiter_shouldHonor(FileSystem jimfs) { stubSuccessfulListObjects(listObjectsHelper, keys); ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(DownloadFileRequest.class); - when(singleDownloadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedDownload()); + when(singleDownloadFunction.download(requestArgumentCaptor.capture(), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(completedDownload())); DirectoryDownload downloadDirectory = downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder() .destination(directory) @@ -671,7 +671,7 @@ void downloadDirectory_withDownloadRequestTransformer_transformerThrows_failsDow FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); RuntimeException exception = new RuntimeException("boom"); @@ -696,7 +696,7 @@ void downloadDirectory_withListObjectsRequestTransformer_transformerThrows_fails FileDownload fileDownload = newSuccessfulDownload(); FileDownload fileDownload2 = newSuccessfulDownload(); - when(singleDownloadFunction.apply(any(DownloadFileRequest.class))).thenReturn(fileDownload, fileDownload2); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(writingWritePath(fileDownload, fileDownload2)); RuntimeException exception = new RuntimeException("boom"); @@ -730,12 +730,9 @@ private static DefaultFileDownload completedDownload() { private static void verifyDestinationPathForSingleDownload(FileSystem jimfs, String delimiter, String[] keys, List actualRequests) { String jimfsSeparator = jimfs.getSeparator(); - List destinations = - actualRequests.stream().map(u -> u.destination().toString()) - .collect(Collectors.toList()); List expectedPaths = Arrays.stream(keys).map(k -> DIRECTORY_NAME + jimfsSeparator + k.replace(delimiter, jimfsSeparator)).collect(Collectors.toList()); - assertThat(destinations).isEqualTo(expectedPaths); + assertDestinations(actualRequests, expectedPaths); } private FileDownload newSuccessfulDownload() { @@ -747,7 +744,7 @@ private FileDownload newSuccessfulDownload() { return fileDownload; } - private FileDownload newFailedDownload(SdkClientException exception) { + private FileDownload newFailedDownload(RuntimeException exception) { CompletableFuture failedFuture = new CompletableFuture<>(); FileDownload fileDownload2 = newDownload(failedFuture); failedFuture.completeExceptionally(exception); @@ -791,4 +788,254 @@ private FileDownload newDownload(CompletableFuture future ".")).getObjectRequest(GetObjectRequest.builder().build()).build(), null); } + + @Test + void downloadDirectory_keysResolvingToSameFile_shouldLeaveOneCompleteObject() throws Exception { + String longBody = "C2:x//y.txt"; + String shortBody = "C2:x/y.txt"; + stubSuccessfulListObjects(listObjectsHelper, "x//y.txt", "x/y.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "x//y.txt", "x/y.txt"); + Path destination = directory.resolve("x/y.txt"); + assertThat(downloads.values()).allSatisfy(d -> { + assertThat(d.request.destination()).isEqualTo(destination); + assertTempFileFor(d.writePath, destination); + }); + assertThat(downloads.get("x//y.txt").writePath).isNotEqualTo(downloads.get("x/y.txt").writePath); + downloads.get("x//y.txt").succeed(longBody); + downloads.get("x/y.txt").succeed(shortBody); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).isEmpty(); + assertThat(readString(directory.resolve("x/y.txt"))).isEqualTo(shortBody); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_keyFailsAfterCollidingKeySucceeded_shouldKeepSucceededObject() throws Exception { + String body = "C2:x//y.txt"; + stubSuccessfulListObjects(listObjectsHelper, "x//y.txt", "x/y.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "x//y.txt", "x/y.txt"); + downloads.get("x//y.txt").succeed(body); + downloads.get("x/y.txt").fail(SdkClientException.create("connection reset")); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).hasSize(1); + FailedFileDownload failed = completed.failedTransfers().iterator().next(); + assertThat(failed.request().getObjectRequest().key()).isEqualTo("x/y.txt"); + assertThat(failed.request().destination()).isEqualTo(directory.resolve("x/y.txt")); + assertThat(failed.exception()).isInstanceOf(SdkClientException.class).hasMessageContaining("connection reset"); + assertThat(readString(directory.resolve("x/y.txt"))).isEqualTo(body); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_downloadFails_shouldKeepExistingFile() throws Exception { + Files.createDirectories(directory); + Files.write(directory.resolve("a.txt"), "existing".getBytes(StandardCharsets.UTF_8)); + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + downloads.get("a.txt").fail(SdkClientException.create("failed")); + + CompletedDirectoryDownload completed = directoryDownload.completionFuture().get(5, TimeUnit.SECONDS); + assertThat(completed.failedTransfers()).hasSize(1); + assertThat(readString(directory.resolve("a.txt"))).isEqualTo("existing"); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_cancelledDuringDownload_shouldRemoveTempFileAndNotCreateDestination() throws Exception { + stubSuccessfulListObjects(listObjectsHelper, "a.txt"); + Map downloads = stubFileWritingDownloads(); + + DirectoryDownload directoryDownload = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()); + + awaitStarted(downloads, "a.txt"); + directoryDownload.completionFuture().cancel(true); + + assertThatThrownBy(() -> downloads.get("a.txt").future.get(1, TimeUnit.SECONDS)) + .isInstanceOf(CancellationException.class); + assertThat(Files.exists(directory.resolve("a.txt"))).isFalse(); + assertNoTempFiles(directory); + } + + @Test + void downloadDirectory_nameThatFitsWithSuffix_shouldKeepName() throws Exception { + String name = IntStream.range(0, 232).mapToObj(i -> "a").collect(Collectors.joining()); + + Path writePath = writePathFor(name); + + String tempName = writePath.getFileName().toString(); + assertThat(tempName).startsWith(name + ".s3tmp."); + assertThat(tempName.getBytes(StandardCharsets.UTF_8)).hasSize(255); + } + + @Test + void downloadDirectory_nameOneByteTooLongForSuffix_shouldUseSuffixOnly() throws Exception { + String name = IntStream.range(0, 233).mapToObj(i -> "a").collect(Collectors.joining()); + + Path writePath = writePathFor(name); + + assertThat(writePath.getFileName().toString()).startsWith("s3tmp."); + } + + private Path writePathFor(String key) throws Exception { + stubSuccessfulListObjects(listObjectsHelper, key); + ArgumentCaptor writePathCaptor = ArgumentCaptor.forClass(Path.class); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), writePathCaptor.capture(), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); + downloadDirectoryHelper.downloadDirectory(DownloadDirectoryRequest.builder().destination(directory).bucket("bucket").build()) + .completionFuture().get(5, TimeUnit.SECONDS); + return writePathCaptor.getValue(); + } + + /** + * Answers like a real download: the write path exists once the download is started, and a download that has already + * succeeded has run its before-completion step. Returns the downloads in order, then keeps returning the last one. + */ + private Answer writingWritePath(FileDownload... downloads) { + AtomicInteger calls = new AtomicInteger(); + return invocation -> { + Path writePath = invocation.getArgument(1); + if (writePath.getParent() != null) { + Files.createDirectories(writePath.getParent()); + } + if (!Files.exists(writePath)) { + Files.createFile(writePath); + } + FileDownload download = downloads[Math.min(calls.getAndIncrement(), downloads.length - 1)]; + CompletableFuture completion = download.completionFuture(); + if (completion.isDone() && !completion.isCompletedExceptionally()) { + try { + Runnable beforeCompletion = invocation.getArgument(2); + beforeCompletion.run(); + } catch (RuntimeException e) { + return newFailedDownload(e); + } + } + return download; + }; + } + + @Test + void downloadDirectory_keyResolvesToDestinationDirectory_shouldReportKeyAndWriteNothing() throws Exception { + Files.createDirectories(directory); + stubSuccessfulListObjects(listObjectsHelper, "p/", "p/a.txt"); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))) + .thenAnswer(writingWritePath(completedDownload())); + + CompletedDirectoryDownload completed = downloadDirectoryHelper.downloadDirectory( + DownloadDirectoryRequest.builder() + .destination(directory) + .bucket("bucket") + .listObjectsV2RequestTransformer(l -> l.prefix("p/")) + .build()).completionFuture().get(5, TimeUnit.SECONDS); + + assertThat(completed.failedTransfers()).hasSize(1); + FailedFileDownload failed = completed.failedTransfers().iterator().next(); + assertThat(failed.request().getObjectRequest().key()).isEqualTo("p/"); + assertThat(failed.exception()).isInstanceOf(SdkClientException.class) + .hasMessageContaining("resolves to the destination directory itself"); + verify(singleDownloadFunction, times(1)).download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class)); + assertThat(Files.isDirectory(directory)).isTrue(); + assertThat(readString(directory.resolve("a.txt"))).isEmpty(); + assertNoTempFiles(directory.toAbsolutePath().getParent()); + } + + private Map stubFileWritingDownloads() { + Map downloads = new ConcurrentHashMap<>(); + when(singleDownloadFunction.download(any(DownloadFileRequest.class), any(Path.class), any(Runnable.class))).thenAnswer(invocation -> { + DownloadFileRequest request = invocation.getArgument(0); + FileWritingDownload download = new FileWritingDownload(request, invocation.getArgument(1), invocation.getArgument(2)); + downloads.put(request.getObjectRequest().key(), download); + return newDownload(download.future); + }); + return downloads; + } + + private static void awaitStarted(Map downloads, String... keys) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!downloads.keySet().containsAll(Arrays.asList(keys))) { + assertThat(System.nanoTime()).as("downloads started: %s", downloads.keySet()).isLessThan(deadline); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(5)); + } + } + + private static String readString(Path path) throws IOException { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } + + private static void assertNoTempFiles(Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (java.util.stream.Stream paths = Files.walk(directory)) { + assertThat(paths.filter(p -> p.getFileName().toString().contains("s3tmp."))).isEmpty(); + } + } + + private static void assertDestinations(List requests, List expectedPaths) { + assertThat(requests.stream().map(r -> r.destination().toString()).collect(Collectors.toList())) + .isEqualTo(expectedPaths); + } + + private static void assertTempFileFor(Path actual, Path destination) { + assertThat(actual.getParent()).isEqualTo(destination.getParent()); + assertThat(actual.getFileName().toString()).matches(Pattern.quote(destination.getFileName().toString()) + + "\\.s3tmp\\.[0-9a-f]{16}"); + } + + /** + * Writes to the write path the way the SDK's file writers do: the file is created and truncated when the download starts, + * the body is written from offset 0 without truncating when it arrives, and a partial file is left in place when the + * download fails. + */ + private static final class FileWritingDownload { + private final DownloadFileRequest request; + private final Path writePath; + private final Runnable beforeCompletion; + private final CompletableFuture future = new CompletableFuture<>(); + + FileWritingDownload(DownloadFileRequest request, Path writePath, Runnable beforeCompletion) throws IOException { + this.request = request; + this.writePath = writePath; + this.beforeCompletion = beforeCompletion; + Files.write(writePath, new byte[0]); + } + + void succeed(String body) throws IOException { + try (FileChannel channel = FileChannel.open(writePath, StandardOpenOption.WRITE)) { + channel.write(ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8)), 0); + } + try { + beforeCompletion.run(); + } catch (RuntimeException e) { + future.completeExceptionally(e); + return; + } + future.complete(CompletedFileDownload.builder().response(GetObjectResponse.builder().build()).build()); + } + + void fail(SdkClientException exception) throws IOException { + try (FileChannel channel = FileChannel.open(writePath, StandardOpenOption.WRITE)) { + channel.write(ByteBuffer.wrap("partial".getBytes(StandardCharsets.UTF_8)), 0); + } + future.completeExceptionally(exception); + } + } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java new file mode 100644 index 000000000000..96915b865ec8 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DownloadDirectoryTempFileWireMockTest.java @@ -0,0 +1,230 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.head; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.testutils.FileUtils; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload; +import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +@WireMockTest +public class DownloadDirectoryTempFileWireMockTest { + private static final String BUCKET = "bucket"; + + private Path destination; + private S3AsyncClient s3; + private S3TransferManager tm; + + @BeforeEach + void setUp() throws IOException { + destination = Files.createTempDirectory("tm-temp-file"); + } + + @AfterEach + void tearDown() { + if (tm != null) { + tm.close(); + } + if (s3 != null) { + s3.close(); + } + FileUtils.cleanUpTestDirectory(destination); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_keysResolvingToSameFile_fileHoldsOneCompleteObject(String clientType, WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + String longBody = "C2:x//y.txt"; + String shortBody = "C2:x/y.txt"; + stubListObjects("x//y.txt", "x/y.txt"); + // Both responses send their headers at once and their bodies later, the longer one first, so every client has + // opened its file for both keys before either body is written. + stubObject("x//y.txt", longBody.getBytes(StandardCharsets.UTF_8), 300); + stubObject("x/y.txt", shortBody.getBytes(StandardCharsets.UTF_8), 900); + + CompletedDirectoryDownload result = tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)) + .completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(result.failedTransfers()).isEmpty(); + assertThat(readOrMissing(destination.resolve("x").resolve("y.txt"))).isIn(longBody, shortBody); + assertThat(regularFiles()).containsExactly("x" + destination.getFileSystem().getSeparator() + "y.txt"); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadDirectory_cancelledMidDownload_leavesNoFiles(String clientType, WireMockRuntimeInfo wm) throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubListObjects("big.bin"); + stubObject("big.bin", new byte[2 * 1024 * 1024], 10_000); + + CompletableFuture download = + tm.downloadDirectory(r -> r.bucket(BUCKET).destination(destination)).completionFuture(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (regularFiles().isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(regularFiles()).as("the download started writing").isNotEmpty(); + + download.cancel(true); + // Gives an in-flight write the chance to land after the cancel. + Thread.sleep(1_000); + + assertThat(regularFiles()).isEmpty(); + } + + @ParameterizedTest(name = "client={0}") + @ValueSource(strings = {"java", "java-multipart", "crt"}) + void downloadFileTo_beforeCompletionRunsBeforeDownloadAndListenerComplete(String clientType, WireMockRuntimeInfo wm) + throws Exception { + s3 = s3Client(clientType, URI.create(wm.getHttpBaseUrl())); + tm = S3TransferManager.builder().s3Client(s3).build(); + stubObject("a.txt", "content-a"); + Path writePath = destination.resolve("a.txt.tmp"); + AtomicBoolean ran = new AtomicBoolean(); + List ranAtTransferComplete = new CopyOnWriteArrayList<>(); + TransferListener listener = new TransferListener() { + @Override + public void transferComplete(Context.TransferComplete context) { + ranAtTransferComplete.add(ran.get()); + } + }; + DownloadFileRequest request = DownloadFileRequest.builder() + .getObjectRequest(g -> g.bucket(BUCKET).key("a.txt")) + .destination(destination.resolve("a.txt")) + .addTransferListener(listener) + .build(); + + ((GenericS3TransferManager) tm).downloadFileTo(request, writePath, () -> { + assertThat(writePath).hasContent("content-a"); + ran.set(true); + }).completionFuture().get(30, TimeUnit.SECONDS); + + assertThat(ran).isTrue(); + // The multipart client does not deliver transferComplete against these stubs, for downloadFile as well. + if (!"java-multipart".equals(clientType)) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ranAtTransferComplete.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(ranAtTransferComplete).containsExactly(true); + } + } + + private List regularFiles() throws IOException { + try (Stream files = Files.walk(destination)) { + return files.filter(Files::isRegularFile).map(destination::relativize).map(Path::toString) + .collect(Collectors.toList()); + } + } + + private static String readOrMissing(Path path) { + try { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } + + private static S3AsyncClient s3Client(String clientType, URI endpoint) { + StaticCredentialsProvider credentials = + StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")); + if ("crt".equals(clientType)) { + return S3AsyncClient.crtBuilder() + .region(Region.US_EAST_1) + .endpointOverride(endpoint) + .forcePathStyle(true) + .credentialsProvider(credentials) + .build(); + } + return S3AsyncClient.builder() + .region(Region.US_EAST_1) + .endpointOverride(endpoint) + .forcePathStyle(true) + .multipartEnabled("java-multipart".equals(clientType)) + .credentialsProvider(credentials) + .build(); + } + + private static void stubListObjects(String... keys) { + StringBuilder contents = new StringBuilder(); + for (String key : keys) { + contents.append("").append(key).append("9"); + } + String body = "" + + "" + + "" + BUCKET + "" + keys.length + "" + + "1000false" + + contents + + ""; + stubFor(get(urlPathMatching("/" + BUCKET + "/?")) + .withQueryParam("list-type", equalTo("2")) + .willReturn(aResponse().withStatus(200).withBody(body))); + } + + private static void stubObject(String key, String content) { + stubObject(key, content.getBytes(StandardCharsets.UTF_8), 0); + } + + private static void stubObject(String key, byte[] bytes, int bodyDelayMillis) { + stubFor(head(urlPathEqualTo("/" + BUCKET + "/" + key)) + .willReturn(aResponse().withStatus(200) + .withHeader("Content-Length", String.valueOf(bytes.length)) + .withHeader("ETag", "\"" + key.replace('/', '-') + "\""))); + stubFor(get(urlPathEqualTo("/" + BUCKET + "/" + key)) + .willReturn(aResponse().withStatus(200) + .withHeader("ETag", "\"" + key.replace('/', '-') + "\"") + .withHeader("Content-Length", String.valueOf(bytes.length)) + .withHeader("Content-Range", "bytes 0-" + (bytes.length - 1) + "/" + bytes.length) + .withBody(bytes) + .withChunkedDribbleDelay(bodyDelayMillis > 0 ? 2 : 1, bodyDelayMillis))); + } +}