diff --git a/.changes/next-release/bugfix-AWSSDKforJavav2-f0e1099.json b/.changes/next-release/bugfix-AWSSDKforJavav2-f0e1099.json new file mode 100644 index 000000000000..c4f63acf2138 --- /dev/null +++ b/.changes/next-release/bugfix-AWSSDKforJavav2-f0e1099.json @@ -0,0 +1,5 @@ +{ + "category": "AWS SDK for Java v2", + "type": "bugfix", + "description": "Fix a bug where events in an event stream were being signed with the request date, and not with the current system time." +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java index 87b387c22dbe..cf85fcbebee2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java @@ -118,6 +118,26 @@ static AsyncResponseTransformer toFile(Path pa return new FileAsyncResponseTransformer<>(path); } + /** + * Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. + * + * @param path Path to file to write to. + * @param position The value for the data between the current end of the file and the starting position is + * undefined. + * @param isNewFile Whether this is a new file. If this is {@code true} and the file already exists, the + * transformer will complete with an exception. + * @param deleteOnFailure Whether the file on disk should be deleted in the event of a failure when writing the + * stream. + * @param Pojo Response type. + * @return AsyncResponseTransformer instance. + */ + static AsyncResponseTransformer toFile(Path path, + long position, + boolean isNewFile, + boolean deleteOnFailure) { + return new FileAsyncResponseTransformer<>(path, position, isNewFile, deleteOnFailure); + } + /** * Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error, * the SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformer.java index 6b4daea2e1bb..a534a90acc80 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformer.java @@ -31,6 +31,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.utils.Validate; /** * {@link AsyncResponseTransformer} that writes the data to the specified file. @@ -40,16 +41,29 @@ @SdkInternalApi public final class FileAsyncResponseTransformer implements AsyncResponseTransformer { private final Path path; + private final long offset; + private final boolean isNewFile; + private final boolean deleteOnFailure; private volatile AsynchronousFileChannel fileChannel; private volatile CompletableFuture cf; private volatile ResponseT response; public FileAsyncResponseTransformer(Path path) { + this(path, 0L, true, true); + } + + public FileAsyncResponseTransformer(Path path, long offset, boolean isNewFile, boolean deleteOnFailure) { this.path = path; + this.offset = Validate.isNotNegative(offset, "offset"); + this.isNewFile = isNewFile; + this.deleteOnFailure = deleteOnFailure; } private AsynchronousFileChannel createChannel(Path path) throws IOException { - return AsynchronousFileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + if (isNewFile) { + return AsynchronousFileChannel.open(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + } + return AsynchronousFileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE); } @Override @@ -72,7 +86,7 @@ public void onResponse(ResponseT response) { public void onStream(SdkPublisher publisher) { // onStream may be called multiple times so reset the file channel every time this.fileChannel = invokeSafely(() -> createChannel(path)); - publisher.subscribe(new FileSubscriber(this.fileChannel, path, cf)); + publisher.subscribe(new FileSubscriber(offset, this.fileChannel, path, cf)); } @Override @@ -80,7 +94,9 @@ public void exceptionOccurred(Throwable throwable) { try { invokeSafely(fileChannel::close); } finally { - invokeSafely(() -> Files.deleteIfExists(path)); + if (deleteOnFailure) { + invokeSafely(() -> Files.deleteIfExists(path)); + } } cf.completeExceptionally(throwable); } @@ -89,7 +105,7 @@ public void exceptionOccurred(Throwable throwable) { * {@link Subscriber} implementation that writes chunks to a file. */ static class FileSubscriber implements Subscriber { - private final AtomicLong position = new AtomicLong(); + private final AtomicLong position; private final AsynchronousFileChannel fileChannel; private final Path path; @@ -99,12 +115,17 @@ static class FileSubscriber implements Subscriber { private volatile boolean closeOnLastWrite = false; private Subscription subscription; - FileSubscriber(AsynchronousFileChannel fileChannel, Path path, CompletableFuture future) { + FileSubscriber(long position, AsynchronousFileChannel fileChannel, Path path, CompletableFuture future) { + this.position = new AtomicLong(position); this.fileChannel = fileChannel; this.path = path; this.future = future; } + FileSubscriber(AsynchronousFileChannel fileChannel, Path path, CompletableFuture future) { + this(0, fileChannel, path, future); + } + @Override public void onSubscribe(Subscription s) { if (this.subscription != null) { diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java new file mode 100644 index 000000000000..d1499a2faa4d --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java @@ -0,0 +1,212 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.async; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CompletableFuture; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; + +/** + * Tests for {@link FileAsyncResponseTransformer}. + */ +public class FileAsyncResponseTransformerTest { + private FileSystem testFs; + + @Before + public void setup() { + testFs = Jimfs.newFileSystem(Configuration.forCurrentPlatform()); + } + + @After + public void teardown() throws IOException { + testFs.close(); + } + + @Test + public void defaultCreatesNewWritableFile() { + byte[] content = "test".getBytes(StandardCharsets.UTF_8); + Path testFile = testFs.getPath("testFile"); + AsyncResponseTransformer transformer = AsyncResponseTransformer.toFile(testFile); + CompletableFuture transformFuture = transformer.prepare(); + transformer.onResponse("some response"); + transformer.onStream(AsyncRequestBody.fromBytes(content)); + transformFuture.join(); + + assertFileContentsEquals(testFile, "test"); + } + + @Test + public void honorsPosition() throws IOException { + byte[] content = "test".getBytes(StandardCharsets.UTF_8); + Path testFile = testFs.getPath("testFile"); + AsyncResponseTransformer transformer = AsyncResponseTransformer.toFile(testFile, content.length, true, true); + CompletableFuture transformFuture = transformer.prepare(); + transformer.onResponse("some response"); + transformer.onStream(AsyncRequestBody.fromBytes(content)); + transformFuture.join(); + + assertThat(Files.size(testFile)).isEqualTo(content.length * 2); + } + + @Test + public void honorsNewFileFlags_False() throws IOException { + Path exists = testFs.getPath("exists"); + createFileWithContents(exists, "Hello".getBytes(StandardCharsets.UTF_8)); + + honorsNewFileFlagTest(exists, 5, false, "Test", "HelloTest"); + } + + @Test + public void honorsNewFileFlag_True_FileNotExists() { + Path notExists = testFs.getPath("notExists"); + honorsNewFileFlagTest(notExists, 0, true, "Test", "Test"); + } + + @Test + public void honorsNewFileFlag_True_FileExists() throws IOException { + Path exists = testFs.getPath("exists"); + createFileWithContents(exists, "Hello".getBytes(StandardCharsets.UTF_8)); + assertThatThrownBy(() -> honorsNewFileFlagTest(exists, 5, true, "Test", null)) + .hasCauseInstanceOf(FileAlreadyExistsException.class); + } + + @Test + public void honorsDeleteOnFailure_True_NoExistingFile() { + Path notExists = testFs.getPath("notExists"); + honorsDeleteOnFailureTest(notExists, true, true); + } + + @Test + public void honorsDeleteOnFailure_True_ExistingFile() throws IOException { + Path exists = testFs.getPath("exists"); + createFileWithContents(exists, "Hello".getBytes(StandardCharsets.UTF_8)); + honorsDeleteOnFailureTest(exists, false, true); + } + + @Test + public void honorsDeleteOnFailure_False_NonExistingFile() { + Path notExists = testFs.getPath("notExists"); + honorsDeleteOnFailureTest(notExists, true, false); + } + + @Test + public void honorsDeleteOnFailure_False_ExistingFile() throws IOException { + Path exists = testFs.getPath("exists"); + createFileWithContents(exists, "Hello".getBytes(StandardCharsets.UTF_8)); + honorsDeleteOnFailureTest(exists, false, false); + } + + private void honorsNewFileFlagTest(Path file, long position, boolean isNewFile, String streamContents, String expectedContents) { + AsyncResponseTransformer transformer = AsyncResponseTransformer.toFile(file, position, isNewFile, true); + CompletableFuture transformFuture = transformer.prepare(); + transformer.onResponse("some response"); + transformer.onStream(AsyncRequestBody.fromString(streamContents)); + transformFuture.join(); + + if (expectedContents != null) { + assertFileContentsEquals(file, expectedContents); + } + } + + private void honorsDeleteOnFailureTest(Path file, boolean isNewFile, boolean deleteOnFailure) { + AsyncResponseTransformer transformer = AsyncResponseTransformer.toFile(file, 0, isNewFile, deleteOnFailure); + CompletableFuture transformFuture = transformer.prepare(); + IOException error = new IOException("Something went wrong"); + transformer.onResponse("some response"); + transformer.onStream(new ErrorPublisher<>(error)); + transformer.exceptionOccurred(error); + assertThatThrownBy(transformFuture::join).hasCause(error); + if (deleteOnFailure) { + assertThat(Files.exists(file)).isFalse(); + } else { + assertThat(Files.exists(file)).isTrue(); + } + } + + private static void createFileWithContents(Path file, byte[] contents) throws IOException { + OutputStream os = Files.newOutputStream(file, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + os.write(contents); + os.close(); + } + + private static void assertFileContentsEquals(Path file, String expected) { + StringBuilder sb = new StringBuilder(); + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(file))); + String s; + while ((s = reader.readLine()) != null) { + sb.append(s); + } + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + assertThat(sb.toString()).isEqualTo(expected); + } + + private static final class ErrorPublisher implements SdkPublisher { + private final Throwable error; + + private ErrorPublisher(Throwable error) { + this.error = error; + } + + @Override + public void subscribe(Subscriber subscriber) { + subscriber.onSubscribe(new ErrorSubscription(subscriber, error)); + } + } + + private static final class ErrorSubscription implements Subscription { + private final Subscriber subscriber; + private final Throwable error; + + public ErrorSubscription(Subscriber subscriber, Throwable error) { + this.subscriber = subscriber; + this.error = error; + } + + @Override + public void request(long l) { + subscriber.onError(error); + } + + @Override + public void cancel() { + + } + } +} diff --git a/docs/design/services/s3/transfermanager/prototype.java b/docs/design/services/s3/transfermanager/prototype.java index a9a3aa629e19..edc55e7f94b2 100644 --- a/docs/design/services/s3/transfermanager/prototype.java +++ b/docs/design/services/s3/transfermanager/prototype.java @@ -221,6 +221,16 @@ interface Builder { */ Builder maxDownloadBytesPerSecond(Long maxDownloadBytesPerSecond); + /** + * The multipart download configuration. + */ + Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration); + + /** + * The multipart upload configuration. + */ + Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration); + /** * Add a progress listener to the currently configured list of * listeners. @@ -445,11 +455,6 @@ public interface SinglePartDownloadContext { * The original download request given to the Transfer Manager. */ DownloadObjectRequest downloadRequest(); - - /** - * The request sent to S3 for this object. This is empty if downloading a presigned URL. - */ - GetObjectRequest objectRequest(); } /** diff --git a/pom.xml b/pom.xml index 0e6707930ffd..66c713b15c55 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ aws-sdk-java core services + services-custom/s3-transfermanager bom bom-internal codegen diff --git a/services-custom/s3-transfermanager/pom.xml b/services-custom/s3-transfermanager/pom.xml new file mode 100644 index 000000000000..c5eab08134e3 --- /dev/null +++ b/services-custom/s3-transfermanager/pom.xml @@ -0,0 +1,122 @@ + + + + + 4.0.0 + + software.amazon.awssdk + aws-sdk-java-pom + 2.5.32-SNAPSHOT + ../../pom.xml + + s3-transfermanager + preview-SNAPSHOT + AWS Java SDK :: S3 :: Transfer Manager + + The S3 Transfer Manager allows customers to easily and optimally + transfer objects and directories to and from S3. + + https://aws.amazon.com/sdkforjava + + + 1.8 + + + + + + software.amazon.awssdk + bom-internal + 2.5.32-SNAPSHOT + pom + import + + + + + + + software.amazon.awssdk + s3 + 2.5.32-SNAPSHOT + + + software.amazon.awssdk + sdk-core + 2.5.32-SNAPSHOT + + + software.amazon.awssdk + aws-core + 2.5.32-SNAPSHOT + + + software.amazon.awssdk + regions + 2.5.32-SNAPSHOT + + + software.amazon.awssdk + utils + 2.5.32-SNAPSHOT + + + software.amazon.awssdk + annotations + 2.5.32-SNAPSHOT + + + + software.amazon.awssdk + service-test-utils + 2.5.32-SNAPSHOT + test + + + software.amazon.awssdk + test-utils + 2.5.32-SNAPSHOT + test + + + junit + junit + test + + + org.assertj + assertj-core + test + + + org.mockito + mockito-core + test + + + log4j + log4j + test + + + org.slf4j + slf4j-log4j12 + test + + + diff --git a/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/DownloadIntegrationTest.java b/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/DownloadIntegrationTest.java new file mode 100644 index 000000000000..0f26db2bc01e --- /dev/null +++ b/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/DownloadIntegrationTest.java @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import static org.assertj.core.api.Assertions.assertThat; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import software.amazon.awssdk.custom.s3.transfer.util.SizeConstant; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.testutils.service.S3BucketUtils; +import software.amazon.awssdk.utils.BinaryUtils; + +/** + * Integration test for TransferManager downloads. + */ +public class DownloadIntegrationTest extends S3TransferManagerIntegrationTestBase { + private static final String BUCKET = S3BucketUtils.temporaryBucketName(DownloadIntegrationTest.class); + private static final String KEY_8KiB = "8kb_test_file.dat"; + private static final String KEY_16MiB = "16mb_test_file.dat"; + private static final Path TMP_DIR = Paths.get(System.getProperty("java.io.tmpdir")); + private static final MessageDigest MD5_DIGEST; + + static { + try { + MD5_DIGEST = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Could not instantiate MD5 digest"); + } + } + + private static S3TransferManager transferManager; + + private static Path testFile8KiB; + private static Path testFile16MiB; + + private static String testFile8KiBDigest; + private static String testFile16MiBDigest; + + @BeforeClass + public static void setup() throws Exception { + S3TransferManagerIntegrationTestBase.setUp(); + + transferManager = S3TransferManager.builder() + .s3client(s3Async) + .multipartDownloadConfiguration(MultipartDownloadConfiguration.defaultConfig() + .toBuilder() + .multipartDownloadThreshold(16 * SizeConstant.MiB - 1) + .build()) + .build(); + + testFile8KiB = new RandomTempFile(8 * SizeConstant.KiB).toPath(); + testFile16MiB = new RandomTempFile(16 * SizeConstant.MiB).toPath(); + + testFile8KiBDigest = computeMd5(testFile8KiB); + testFile16MiBDigest = computeMd5(testFile16MiB); + + createBucket(BUCKET); + putFile(KEY_8KiB, testFile8KiB); + putFile(KEY_16MiB, testFile16MiB); + } + + @AfterClass + public static void teardown() { + deleteBucketAndAllContents(BUCKET); + tryDeleteFiles(testFile8KiB, testFile16MiB); + } + + @Test + public void singlePartDownload() throws IOException { + downloadTest(KEY_8KiB, testFile8KiBDigest); + } + + @Test + public void multipartDownload() throws IOException { + downloadTest(KEY_16MiB, testFile16MiBDigest); + } + + private static void putFile(String key, Path file) { + s3.putObject(r -> r.bucket(BUCKET).key(key), file); + } + + private static void downloadTest(String key, String expectedMd5) throws IOException { + Path tempFile = createTempPath(); + try { + transferManager.download(BUCKET, key, tempFile).completionFuture().join(); + String downloadedFileMd5 = computeMd5(tempFile); + assertThat(downloadedFileMd5).isEqualTo(expectedMd5); + } finally { + Files.deleteIfExists(tempFile); + } + } + + private static Path createTempPath() { + return TMP_DIR.resolve(DownloadIntegrationTest.class.getSimpleName() + "-" + System.currentTimeMillis()); + } + + private static String computeMd5(Path file) { + try (InputStream is = Files.newInputStream(file, StandardOpenOption.READ)) { + MD5_DIGEST.reset(); + byte[] buff = new byte[4096]; + int read; + while ((read = is.read(buff)) != -1) { + MD5_DIGEST.update(buff, 0, read); + } + return BinaryUtils.toBase64(MD5_DIGEST.digest()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static void tryDeleteFiles(Path... files) { + for (Path file : files) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + System.err.println("Could not delete file " + file); + } + } + } +} diff --git a/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManagerIntegrationTestBase.java b/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManagerIntegrationTestBase.java new file mode 100644 index 000000000000..39ae21393889 --- /dev/null +++ b/services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManagerIntegrationTestBase.java @@ -0,0 +1,86 @@ +package software.amazon.awssdk.custom.s3.transfer; + +import org.junit.BeforeClass; +import software.amazon.awssdk.custom.s3.transfer.utils.S3TestUtils; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.testutils.service.AwsTestBase; + +/** + * Base class for TransferManager integration tests. + */ +public class S3TransferManagerIntegrationTestBase extends AwsTestBase { + protected static final Region DEFAULT_REGION = Region.US_WEST_2; + /** + * The S3 client for all tests to use. + */ + protected static S3Client s3; + + protected static S3AsyncClient s3Async; + + /** + * Loads the AWS account info for the integration tests and creates an S3 + * client for tests to use. + */ + @BeforeClass + public static void setUp() { + s3 = s3ClientBuilder().build(); + s3Async = s3AsyncClientBuilder().build(); + } + + protected static S3ClientBuilder s3ClientBuilder() { + return S3Client.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static S3AsyncClientBuilder s3AsyncClientBuilder() { + return S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static void createBucket(String bucketName) { + createBucket(bucketName, 0); + } + + private static void createBucket(String bucketName, int retryCount) { + try { + s3.createBucket( + CreateBucketRequest.builder() + .bucket(bucketName) + .createBucketConfiguration( + CreateBucketConfiguration.builder() + .locationConstraint(BucketLocationConstraint.US_WEST_2) + .build()) + .build()); + } catch (S3Exception e) { + System.err.println("Error attempting to create bucket: " + bucketName); + if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) { + System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName); + } else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) { + System.err.println("Printing all buckets for debug:"); + s3.listBuckets().buckets().forEach(System.err::println); + if (retryCount < 2) { + System.err.println("Retrying..."); + createBucket(bucketName, retryCount + 1); + } else { + throw e; + } + } else { + throw e; + } + } + } + + protected static void deleteBucketAndAllContents(String bucketName) { + S3TestUtils.deleteBucketAndAllContents(s3, bucketName); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestDownloadObjectSpecification.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestDownloadObjectSpecification.java new file mode 100644 index 000000000000..999473664ddf --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestDownloadObjectSpecification.java @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.Validate; + +/** + * Implementation of {@link DownloadObjectSpecification} for {@link GetObjectRequest}. + */ +@SdkInternalApi +final class ApiRequestDownloadObjectSpecification extends DownloadObjectSpecification { + private final GetObjectRequest apiRequest; + + ApiRequestDownloadObjectSpecification(GetObjectRequest apiRequest) { + this.apiRequest = Validate.notNull(apiRequest, "apiRequest must not be null"); + } + + @Override + public boolean isApiRequest() { + return true; + } + + @Override + public GetObjectRequest asApiRequest() { + return apiRequest; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ApiRequestDownloadObjectSpecification that = (ApiRequestDownloadObjectSpecification) o; + return apiRequest.equals(that.apiRequest); + } + + @Override + public int hashCode() { + return apiRequest.hashCode(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestUploadObjectSpecification.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestUploadObjectSpecification.java new file mode 100644 index 000000000000..0a9b8665d595 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/ApiRequestUploadObjectSpecification.java @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.Validate; + +/** + * Implementation for {@link UploadObjectSpecification} for {@link PutObjectRequest}. + */ +@SdkInternalApi +final class ApiRequestUploadObjectSpecification extends UploadObjectSpecification { + private final PutObjectRequest apiRequest; + + ApiRequestUploadObjectSpecification(PutObjectRequest apiRequest) { + this.apiRequest = Validate.notNull(apiRequest, "apiRequest must not be null"); + } + + @Override + public boolean isApiRequest() { + return true; + } + + @Override + public PutObjectRequest asApiRequest() { + return apiRequest; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ApiRequestUploadObjectSpecification that = (ApiRequestUploadObjectSpecification) o; + return apiRequest.equals(that.apiRequest); + } + + @Override + public int hashCode() { + return apiRequest.hashCode(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownload.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownload.java new file mode 100644 index 000000000000..99dc4b73ac8e --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownload.java @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed download transfer. + */ +@SdkPublicApi +public interface CompletedDownload extends CompletedTransfer { +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownloadDirectory.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownloadDirectory.java new file mode 100644 index 000000000000..f220fd3d4de8 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedDownloadDirectory.java @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed download directory transfer. + */ +@SdkPublicApi +public interface CompletedDownloadDirectory extends CompletedTransfer { +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedTransfer.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedTransfer.java new file mode 100644 index 000000000000..0fbe1399ef64 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedTransfer.java @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed transfer. + */ +@SdkPublicApi +public interface CompletedTransfer { + /** + * The metrics for this transfer. + */ + TransferMetrics metrics(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUpload.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUpload.java new file mode 100644 index 000000000000..bed15409f700 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUpload.java @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed upload transfer. + */ +@SdkPublicApi +public interface CompletedUpload extends CompletedTransfer { +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUploadDirectory.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUploadDirectory.java new file mode 100644 index 000000000000..7986ce099c91 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/CompletedUploadDirectory.java @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed upload directory transfer. + */ +@SdkPublicApi +public interface CompletedUploadDirectory extends CompletedTransfer { +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Download.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Download.java new file mode 100644 index 000000000000..a926861f4ab3 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Download.java @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A download transfer of a single object from S3. + */ +@SdkPublicApi +public interface Download extends Transfer { + @Override + CompletableFuture completionFuture(); + + DownloadState pause(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadDirectory.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadDirectory.java new file mode 100644 index 000000000000..029eb3b7a158 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadDirectory.java @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A download transfer of a directory from S3. + */ +@SdkPublicApi +public interface DownloadDirectory extends Transfer { + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadObjectSpecification.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadObjectSpecification.java new file mode 100644 index 000000000000..33715a1d6007 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadObjectSpecification.java @@ -0,0 +1,70 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.net.URL; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +/** + * Union of the various ways to specify how to download an object from S3. + */ +@SdkPublicApi +public abstract class DownloadObjectSpecification { + + DownloadObjectSpecification() { + } + + /** + * @return {@code true} if this is a presigned URL, {@code false} otherwise. + */ + public boolean isPresignedUrl() { + return false; + } + + /** + * @return {@code true} if this is an API request, {@code false} otherwise. + */ + public boolean isApiRequest() { + return false; + } + + /** + * @return This specification as a presigned URL. + * @throws IllegalStateException If this is not a presigned URL. + */ + public URL asPresignedUrl() { + throw new IllegalStateException("Not a presigned URL"); + } + + /** + * @return This specification as an API request. + * @throws IllegalStateException If this is not an API request. + */ + public GetObjectRequest asApiRequest() { + throw new IllegalStateException("Not an API Request"); + } + + /** + * Create a specification from a {@link GetObjectRequest}. + * + * @param getObjectRequest The request. + * @return The new API request specification. + */ + public static DownloadObjectSpecification fromApiRequest(GetObjectRequest getObjectRequest) { + return new ApiRequestDownloadObjectSpecification(getObjectRequest); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java new file mode 100644 index 000000000000..90daf557a568 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java @@ -0,0 +1,180 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.Collection; +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.Validate; + +/** + * Request object to download an object from S3 using the Transfer Manager. + */ +@SdkPublicApi +public final class DownloadRequest extends TransferObjectRequest { + private final DownloadObjectSpecification downloadSpecification; + private final Long size; + + private DownloadRequest(BuilderImpl builder) { + super(builder); + this.downloadSpecification = Validate.notNull(builder.downloadSpecification, "downloadSpecification must not be null"); + this.size = builder.size; + } + + /** + * @return The download specification. + */ + public DownloadObjectSpecification downloadSpecification() { + return downloadSpecification; + } + + /** + * @return The known size of the object to be downloaded. If multipart + * downloads are enabled, this allows the Transfer Manager to omit a call + * to S3 to get the object size. + */ + public Long size() { + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DownloadRequest that = (DownloadRequest) o; + return downloadSpecification.equals(that.downloadSpecification) && + Objects.equals(size, that.size); + } + + @Override + public int hashCode() { + int hashCode = 1; + hashCode = 31 * hashCode + Objects.hashCode(downloadSpecification); + hashCode = 31 * hashCode + Objects.hashCode(size); + return hashCode; + } + + /** + * @return A builder for this request. + */ + public static Builder builder() { + return new BuilderImpl(); + } + + public static DownloadRequest forBucketAndKey(String bucket, String key) { + return DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest( + GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build() + )) + .build(); + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public interface Builder extends TransferObjectRequest.Builder { + /** + * Set the download specification. + * + * @param downloadSpecification The specification. + * @return This object for method chaining. + */ + Builder downloadSpecification(DownloadObjectSpecification downloadSpecification); + + /** + * Optionally set the known size of the object to be downloaded. If + * multipart downloads are enabled, this allows the Transfer Manager to + * omit a call to S3 to get the object size. + * + * @param size The object size. + * @return This object for method chaining. + */ + Builder size(Long size); + + @Override + Builder overrideConfiguration(TransferOverrideConfiguration config); + + @Override + Builder progressListeners(Collection progressListeners); + + @Override + Builder addProgressListener(TransferProgressListener progressListener); + + /** + * @return The built request. + */ + DownloadRequest build(); + } + + private static final class BuilderImpl extends TransferObjectRequest.BuilderImpl implements Builder { + private DownloadObjectSpecification downloadSpecification; + private Long size; + + private BuilderImpl(DownloadRequest other) { + super(other); + this.downloadSpecification = other.downloadSpecification; + this.size = other.size; + } + + private BuilderImpl() { + } + + @Override + public Builder downloadSpecification(DownloadObjectSpecification downloadSpecification) { + this.downloadSpecification = downloadSpecification; + return this; + } + + @Override + public Builder size(Long size) { + this.size = size; + return this; + } + + @Override + public Builder overrideConfiguration(TransferOverrideConfiguration config) { + super.overrideConfiguration(config); + return this; + } + + @Override + public Builder progressListeners(Collection progressListeners) { + super.progressListeners(progressListeners); + return this; + } + + @Override + public Builder addProgressListener(TransferProgressListener progressListener) { + super.addProgressListener(progressListener); + return this; + } + + @Override + public DownloadRequest build() { + return new DownloadRequest(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadState.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadState.java new file mode 100644 index 000000000000..71695bd3beb0 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadState.java @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.io.InputStream; +import java.io.OutputStream; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * The state of a paused download. The download can be resumed using {@link S3TransferManager#resumeDownload(DownloadState)}. + */ +@SdkPublicApi +public class DownloadState { + /** + * Persist this state to the given stream. + * + * @param os The stream to write this state to. + */ + public void persistTo(OutputStream os) { + throw new UnsupportedOperationException(); + } + + /** + * Load a persisted state. + * + * @param is The stream to read the state from. + * @return The loaded state. + */ + public static DownloadState loadFrom(InputStream is) { + throw new UnsupportedOperationException(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartDownloadConfiguration.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartDownloadConfiguration.java new file mode 100644 index 000000000000..ed38935a6c6e --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartDownloadConfiguration.java @@ -0,0 +1,226 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.custom.s3.transfer.util.SizeConstant; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * The configuration object for multipart downloads in {@link S3TransferManager}. + */ +@SdkPublicApi +public final class MultipartDownloadConfiguration implements + ToCopyableBuilder { + /** + * The default value for whether multipart downloads are enabled. + */ + public static final boolean DEFAULT_ENABLE_MULTIPART_DOWNLOADS = true; + + /** + * The default value for the multipart download threshold. + */ + public static final long DEFAULT_MULTIPART_DOWNLOAD_THRESHOLD = 16 * SizeConstant.MiB; + + /** + * The default value for the maximum download part count. + */ + public static final int DEFAULT_MAX_DOWNLOAD_PART_COUNT = 10_000; + + /** + * The default for value for minimum download part size. + */ + public static final long DEFAULT_MIN_DOWNLOAD_PART_SIZE = 5 * SizeConstant.MiB; + + private static final MultipartDownloadConfiguration DEFAULT = MultipartDownloadConfiguration.builder() + .enableMultipartDownloads(DEFAULT_ENABLE_MULTIPART_DOWNLOADS) + .multipartDownloadThreshold(DEFAULT_MULTIPART_DOWNLOAD_THRESHOLD) + .maxDownloadPartCount(DEFAULT_MAX_DOWNLOAD_PART_COUNT) + .minDownloadPartSize(DEFAULT_MIN_DOWNLOAD_PART_SIZE) + .build(); + + private final Boolean enableMultipartDownloads; + private final Long multipartDownloadThreshold; + private final Integer maxDownloadPartCount; + private final Long minDownloadPartSize; + + private MultipartDownloadConfiguration(BuilderImpl builder) { + this.enableMultipartDownloads = resolveEnableMultipartDownloads(builder.enableMultipartDownloads); + this.multipartDownloadThreshold = resolveMultipartDownloadThreshold(builder.multipartDownloadThreshold); + this.maxDownloadPartCount = resolveMaxDownloadPartCount(builder.maxDownloadPartCount); + this.minDownloadPartSize = resolveMinDownloadPartSize(builder.minDownloadPartSize); + validateConfig(); + } + + /** + * @return Whether multipart downloads are enabled. + */ + public Boolean enableMultipartDownloads() { + return enableMultipartDownloads; + } + + /** + * @return The minimum size for an object for it to be downloaded in + * multiple parts. + */ + public Long multipartDownloadThreshold() { + return multipartDownloadThreshold; + } + + /** + * @return The maximum number of parts to download a single object. + */ + public Integer maxDownloadPartCount() { + return maxDownloadPartCount; + } + + /** + * @return The minimum size for a part of an object to download. + */ + public Long minDownloadPartSize() { + return minDownloadPartSize; + } + + /** + * @return A new {@link Builder} with the current values of this + * configuration. + */ + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public static Builder builder() { + return new BuilderImpl(); + } + + /** + * @return An instance of this class using the default values. + */ + public static MultipartDownloadConfiguration defaultConfig() { + return DEFAULT; + } + + private Boolean resolveEnableMultipartDownloads(Boolean configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_ENABLE_MULTIPART_DOWNLOADS); + } + + private Long resolveMultipartDownloadThreshold(Long configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MULTIPART_DOWNLOAD_THRESHOLD); + } + + private Integer resolveMaxDownloadPartCount(Integer configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MAX_DOWNLOAD_PART_COUNT); + } + + private Long resolveMinDownloadPartSize(Long configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MIN_DOWNLOAD_PART_SIZE); + } + + private void validateConfig() { + Validate.isPositive(multipartDownloadThreshold, "multipartDownloadThreshold"); + Validate.isPositive(maxDownloadPartCount, "maxDownloadPartCount"); + Validate.isTrue(maxDownloadPartCount <= DEFAULT_MAX_DOWNLOAD_PART_COUNT, + "maxDownloadPartCount must be at most %d", DEFAULT_MAX_DOWNLOAD_PART_COUNT); + Validate.isPositive(minDownloadPartSize, "minDownloadPartSize"); + Validate.isTrue(minDownloadPartSize <= multipartDownloadThreshold, + "minDownloadPartSize must not be greater than multipartDownloadThreshold"); + } + + public interface Builder extends CopyableBuilder { + /** + * Set whether multipart downloads are enabled. + * + * @param enableMultipartDownloads Whether multipart downloads are + * enabled. + * @return This object for method chaining. + */ + Builder enableMultipartDownloads(Boolean enableMultipartDownloads); + + /** + * Set the minimum size for an object for it to be downloaded in + * multiple parts. + * + * @param multipartDownloadThreshold The threshold. + * @return This object for method chaining. + */ + Builder multipartDownloadThreshold(Long multipartDownloadThreshold); + + /** + * Set the maximum number of parts to download a single object. + * + * @param maxDownloadPartCount The maximum part count. + * @return This object for method chaining. + */ + Builder maxDownloadPartCount(Integer maxDownloadPartCount); + + /** + * Set the minimum size for a part of an object to download. + * + * @param minDownloadPartSize The minimum part size. + * @return This object for method chaining. + */ + Builder minDownloadPartSize(Long minDownloadPartSize); + } + + private static final class BuilderImpl implements Builder { + private Boolean enableMultipartDownloads; + private Long multipartDownloadThreshold; + private Integer maxDownloadPartCount; + private Long minDownloadPartSize; + + private BuilderImpl() { + } + + private BuilderImpl(MultipartDownloadConfiguration original) { + enableMultipartDownloads = original.enableMultipartDownloads; + multipartDownloadThreshold = original.multipartDownloadThreshold; + maxDownloadPartCount = original.maxDownloadPartCount; + minDownloadPartSize = original.minDownloadPartSize; + } + + @Override + public Builder enableMultipartDownloads(Boolean enableMultipartDownloads) { + this.enableMultipartDownloads = enableMultipartDownloads; + return this; + } + + @Override + public Builder multipartDownloadThreshold(Long multipartDownloadThreshold) { + this.multipartDownloadThreshold = multipartDownloadThreshold; + return this; + } + + @Override + public Builder maxDownloadPartCount(Integer maxDownloadPartCount) { + this.maxDownloadPartCount = maxDownloadPartCount; + return this; + } + + @Override + public Builder minDownloadPartSize(Long minDownloadPartSize) { + this.minDownloadPartSize = minDownloadPartSize; + return this; + } + + @Override + public MultipartDownloadConfiguration build() { + return new MultipartDownloadConfiguration(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartUploadConfiguration.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartUploadConfiguration.java new file mode 100644 index 000000000000..5337db715cee --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/MultipartUploadConfiguration.java @@ -0,0 +1,228 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.custom.s3.transfer.util.SizeConstant; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * The configuration object for multipart uplaods in {@link S3TransferManager}. + */ +@SdkPublicApi +public final class MultipartUploadConfiguration implements + ToCopyableBuilder { + /** + * The default value for whether multipart uploads are enabled. + */ + public static final boolean DEFAULT_ENABLE_MULTIPART_UPLOAD = true; + + /** + * The default value for the multipart upload threshold. + */ + public static final long DEFAULT_MULTIPART_UPLOAD_THRESHOLD = 16 * SizeConstant.MiB; + + /** + * The default value for maximum upload part count. + */ + public static final int DEFAULT_MAX_UPLOAD_PART_COUNT = 10_000; + + /** + * The default for value for minimum upload part size. + */ + public static final long DEFAULT_MIN_UPLOAD_PART_SIZE = 5 * SizeConstant.MiB; + + public static final long MULTIPART_MIN_PART_SIZE = 5 * SizeConstant.MiB; + + + private static final MultipartUploadConfiguration DEFAULT = MultipartUploadConfiguration.builder() + .enableMultipartUploads(DEFAULT_ENABLE_MULTIPART_UPLOAD) + .multipartUploadThreshold(DEFAULT_MULTIPART_UPLOAD_THRESHOLD) + .maxUploadPartCount(DEFAULT_MAX_UPLOAD_PART_COUNT) + .minUploadPartSize(DEFAULT_MIN_UPLOAD_PART_SIZE) + .build(); + + private final Boolean enableMultipartUploads; + private final Long multipartUploadThreshold; + private final Integer maxUploadPartCount; + private final Long minUploadPartSize; + + private MultipartUploadConfiguration(BuilderImpl builder) { + this.enableMultipartUploads = resolveEnableMultipartUploads(builder.enableMultipartUploads); + this.multipartUploadThreshold = resolveMultipartUploadThreshold(builder.multipartUploadThreshold); + this.maxUploadPartCount = resolveMaxUploadPartCount(builder.maxUploadPartCount); + this.minUploadPartSize = resolveMinUploadPartSize(builder.minUploadPartSize); + validateConfig(); + } + + /** + * @return Whether multipart uploads are enabled. + */ + public Boolean enableMultipartUploads() { + return enableMultipartUploads; + } + + /** + * @return The minimum size for an object for it to be uploaded in multiple + * parts. + */ + public Long multipartUploadThreshold() { + return multipartUploadThreshold; + } + + /** + * @return The maximum number of parts to upload an object in. + */ + public Integer maxUploadPartCount() { + return maxUploadPartCount; + } + + /** + * @return The minimum size for a part of an object to upload. + */ + public Long minimumUploadPartSize() { + return minUploadPartSize; + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public static Builder builder() { + return new BuilderImpl(); + } + + /** + * @return An instance of this class using the default values. + */ + public static MultipartUploadConfiguration defaultConfig() { + return DEFAULT; + } + + private static Boolean resolveEnableMultipartUploads(Boolean configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_ENABLE_MULTIPART_UPLOAD); + } + + private static Long resolveMultipartUploadThreshold(Long configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MULTIPART_UPLOAD_THRESHOLD); + } + + private static Integer resolveMaxUploadPartCount(Integer configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MAX_UPLOAD_PART_COUNT); + } + + private static Long resolveMinUploadPartSize(Long configured) { + return Optional.ofNullable(configured).orElse(DEFAULT_MIN_UPLOAD_PART_SIZE); + } + + private void validateConfig() { + Validate.isPositive(multipartUploadThreshold, "multipartUploadThreshold"); + Validate.isPositive(minUploadPartSize, "minUploadPartSize"); + Validate.isTrue(minUploadPartSize >= MULTIPART_MIN_PART_SIZE, + "minUploadPartSize must be at least %d", MULTIPART_MIN_PART_SIZE); + Validate.isPositive(maxUploadPartCount, "maxUploadPartCount"); + Validate.isTrue(maxUploadPartCount <= DEFAULT_MAX_UPLOAD_PART_COUNT, + "maxUploadPartCount must be at most %d", DEFAULT_MAX_UPLOAD_PART_COUNT); + + } + + public interface Builder extends CopyableBuilder { + /** + * Set whether multipart uploads are enabled. + * + * @param enableMultipartUploads Whether multipart uploads are enabled. + * @return This object for method chaining. + */ + Builder enableMultipartUploads(Boolean enableMultipartUploads); + + /** + * Set the minimum size for an object for it to be uploaded in + * multiple parts. + * + * @param multipartUploadThreshold The threshold. + * @return This object for method chaining. + */ + Builder multipartUploadThreshold(Long multipartUploadThreshold); + + /** + * Se the maximum number of parts to upload an object in. This must be + * {@code <=} 10,000, the maximum parts allowed by S3 for a multipart + * object. + * + * @param maxUploadPartCount The max part count. + * @return This object for method chaining. + */ + Builder maxUploadPartCount(Integer maxUploadPartCount); + + /** + * Set the minimum size for a part of an object to upload. + * + * @param minimumUploadPartSize The minimum part size. + * @return This object for method chaining. + */ + Builder minUploadPartSize(Long minimumUploadPartSize); + } + + private static final class BuilderImpl implements Builder { + private Boolean enableMultipartUploads; + private Long multipartUploadThreshold; + private Integer maxUploadPartCount; + private Long minUploadPartSize; + + private BuilderImpl(MultipartUploadConfiguration other) { + this.enableMultipartUploads = other.enableMultipartUploads; + this.multipartUploadThreshold = other.multipartUploadThreshold; + this.maxUploadPartCount = other.maxUploadPartCount; + this.minUploadPartSize = other.minUploadPartSize; + } + + private BuilderImpl() { + } + + @Override + public Builder enableMultipartUploads(Boolean enableMultipartUploads) { + this.enableMultipartUploads = enableMultipartUploads; + return this; + } + + @Override + public Builder multipartUploadThreshold(Long multipartUploadThreshold) { + this.multipartUploadThreshold = multipartUploadThreshold; + return this; + } + + @Override + public Builder maxUploadPartCount(Integer maxUploadPartCount) { + this.maxUploadPartCount = maxUploadPartCount; + return this; + } + + @Override + public Builder minUploadPartSize(Long minUploadPartSize) { + this.minUploadPartSize = minUploadPartSize; + return this; + } + + @Override + public MultipartUploadConfiguration build() { + return new MultipartUploadConfiguration(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManager.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManager.java new file mode 100644 index 000000000000..9405770b53fa --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/S3TransferManager.java @@ -0,0 +1,263 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.net.URL; +import java.nio.file.Path; +import java.util.Collection; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.custom.s3.transfer.internal.DefaultS3TransferManager; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.SdkAutoCloseable; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * The S3 Transfer Manager is a library that allows users to easily and + * optimally upload and downloads to and from S3. + *

+ * The list of features includes: + *

    + *
  • Parallel uploads and downloads
    • + *
  • Bandwidth limiting
    • + *
  • Pause and resume of transfers
    • + *
+ *

+ * Usage Example: + * 

+ * {@code
+ * // Create using all default configuration values
+ * S3TransferManager tm = S3TranferManager.create();
+ *
+ * // Using custom configuration values to set max download speed
+ * S3TransferManager tm = S3TransferManager.builder()
+ *         .configuration(TransferManagerConfiguration.builder()
+ *             .maximumDownloadBytesSecond(5 * 1024 * 1024) // 5MiB
+ *             .build()
+ *         .build();
+ * }
+ *
+ */ +@SdkPublicApi +public interface S3TransferManager extends ToCopyableBuilder, SdkAutoCloseable { + /** + * Download an object identified by the bucket and key from S3 to the given + * file. + *

+ * Usage Example: + * 

+     * {@code
+     * // Initiate transfer
+     * Download myFileDownload = tm.download(BUCKET, KEY, Paths.get("/tmp/myFile.txt");
+     * // Wait for transfer to complete
+     * myFileDownload().completionFuture().join();
+     * }
+     *
+ */ + default Download download(String bucket, String key, Path file) { + return download(DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest( + GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build() + )) + .build(), + file); + } + + /** + * Download an object using an S3 presigned URL to the given file. + *

+ * Usage Example: + * 

+     * {@code
+     * // Initiate transfer
+     * Download myFileDownload = tm.download(myPresignedUrl, Paths.get("/tmp/myFile.txt");
+     * // Wait for transfer to complete
+     * myFileDownload()completionFuture().join();
+     * }
+     *
+ */ + default Download download(URL presignedUrl, Path file) { + throw new UnsupportedOperationException(); + } + + /** + * Download an object in S3 to the given file. + * + *

+ * Usage Example: + * 

+     * {@code
+     * // Initiate the transfer
+     * Download myDownload = tm.download(DownloadObjectRequest.builder()
+     *         .downloadObjectSpecification(DownloadObjectSpecification.fromApiRequest(
+     *             GetObjectRequest.builder()
+     *                 .bucket(BUCKET)
+     *                 .key(KEY)
+     *                 .build()))
+     *          // Set the known length of the object to avoid a HeadObject call
+     *         .size(1024 * 1024 * 5)
+     *         .build(),
+     *         Paths.get("/tmp/myFile.txt"));
+     * // Wait for the transfer to complete
+     * myDownload.completionFuture().join();
+     * }
+     *
+ */ + Download download(DownloadRequest request, Path file); + + /** + * Resume a previously paused object download. + */ + Download resumeDownload(DownloadState downloadState); + + /** + * Download the set of objects from the bucket with the given prefix to a directory. + *

+ * The transfer manager will use '/' as the path delimiter. + *

+ * Usage Example: + * 

+     * {@code
+     * DownloadDirectory myDownload = downloadDirectory(myBucket, myPrefix, Paths.get("/tmp");
+     * myDowload.completionFuture().join();
+     * }
+     *
+ * + * @param bucket The bucket. + * @param prefix The prefix. + * @param destinationDirectory The directory where the objects will be downloaded to. + */ + DownloadDirectory downloadDirectory(String bucket, String prefix, Path destinationDirectory); + + /** + * Upload a file to S3. + *

+ * Usage Example: + * 

+     * {@code
+     * Upload myUpload = tm.upload(myBucket, myKey, Paths.get("myFile.txt"));
+     * myUpload.completionFuture().join();
+     * }
+     *
+ */ + default Upload upload(String bucket, String key, Path file) { + return upload(UploadRequest.builder() + .uploadSpecification(UploadObjectSpecification.fromApiRequest(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .build())) + .build(), + file); + } + + /** + * Upload a file to S3 using the given presigned URL. + *

+ * Usage Example: + * 

+     * {@code
+     * Upload myUpload = tm.upload(myPresignedUrl, Paths.get("myFile.txt"));
+     * myUpload.completionFuture().join();
+     * }
+     *
+ */ + default Upload upload(URL presignedUrl, Path file) { + throw new UnsupportedOperationException(); + } + + /** + * Upload a file to S3. + */ + Upload upload(UploadRequest request, Path file); + + /** + * Resume a previously paused object upload. + */ + Upload resumeUpload(UploadState uploadState); + + /** + * Upload the given directory to the S3 bucket under the given prefix. + * + * @param bucket The bucket. + * @param prefix The prefix. + * @param sourceDirectory The directory containing the objects to be uploaded. + */ + UploadDirectory uploadDirectory(String bucket, String prefix, Path sourceDirectory); + + /** + * Create an {@code S3TransferManager} using the default values. + */ + static S3TransferManager create() { + return builder().build(); + } + + static S3TransferManager.Builder builder() { + return DefaultS3TransferManager.builder(); + } + + interface Builder extends CopyableBuilder { + /** + * The custom S3AsyncClient this transfer manager will use to make calls + * to S3. + */ + Builder s3client(S3AsyncClient s3Client); + + /** + * The max number of requests the Transfer Manager will have at any + * point in time. This must be less than or equal to the max concurrent + * setting on the S3 client. + */ + Builder maxConcurrency(Integer maxConcurrency); + + /** + * The aggregate max upload rate in bytes per second over all active + * upload transfers. The default is unlimited. + */ + Builder maxUploadBytesPerSecond(Long maxUploadBytesPerSecond); + + /** + * The aggregate max download rate in bytes per second over all active + * download transfers. The default value is unlimited. + */ + Builder maxDownloadBytesPerSecond(Long maxDownloadBytesPerSecond); + + /** + * The multipart download configuration. + */ + Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration); + + /** + * The multipart upload configuration. + */ + Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration); + + /** + * Add a progress listener to the currently configured list of + * listeners. + */ + Builder addProgressListener(TransferProgressListener progressListener); + + /** + * Set the list of progress listeners. + */ + Builder progressListeners(Collection progressListeners); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Transfer.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Transfer.java new file mode 100644 index 000000000000..a0a6e2aa2948 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Transfer.java @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Represents the upload or download of one or more objects to or from S3. + */ +@SdkPublicApi +public interface Transfer { + /** + * @return The future that will be completed when this transfer is complete. + */ + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferMetrics.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferMetrics.java new file mode 100644 index 000000000000..f40eb69c0e73 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferMetrics.java @@ -0,0 +1,76 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.time.Duration; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.Validate; + +/** + * Metrics for a completed transfer. + */ +@SdkPublicApi +public final class TransferMetrics { + private final Duration elapsedTime; + private final long bytesTransferred; + + private TransferMetrics(BuilderImpl builder) { + this.elapsedTime = Validate.notNull(builder.elapsedTimed, "elapsedTime must not be null"); + this.bytesTransferred = builder.bytesTransferred; + } + + public Duration elapsedTime() { + return elapsedTime; + } + + public long bytesTransferred() { + return bytesTransferred; + } + + public static Builder builder() { + return new BuilderImpl(); + } + + public interface Builder { + Builder elapsedTime(Duration elapsedTime); + + Builder bytesTransferred(long bytesTransferred); + + TransferMetrics build(); + } + + private static final class BuilderImpl implements Builder { + private Duration elapsedTimed; + private long bytesTransferred; + + @Override + public Builder elapsedTime(Duration elapsedTimed) { + this.elapsedTimed = elapsedTimed; + return this; + } + + @Override + public Builder bytesTransferred(long bytesTransferred) { + this.bytesTransferred = bytesTransferred; + return this; + } + + @Override + public TransferMetrics build() { + return new TransferMetrics(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferObjectRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferObjectRequest.java new file mode 100644 index 000000000000..99f3a6fc2a50 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferObjectRequest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A request representing a transfer of an object to and from S3. + */ +@SdkPublicApi +public abstract class TransferObjectRequest extends TransferRequest { + protected TransferObjectRequest(BuilderImpl builder) { + super(builder); + } + + @Override + public abstract Builder toBuilder(); + + protected interface Builder extends TransferRequest.Builder { + @Override + TransferObjectRequest build(); + } + + protected abstract static class BuilderImpl extends TransferRequest.BuilderImpl { + protected BuilderImpl() { + } + + protected BuilderImpl(TransferObjectRequest other) { + super(other); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferOverrideConfiguration.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferOverrideConfiguration.java new file mode 100644 index 000000000000..007599d1dbf3 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferOverrideConfiguration.java @@ -0,0 +1,139 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Override configuration for a single transfer. Override configurations take + * precedence those set on {@link S3TransferManager}. + */ +@SdkPublicApi +public final class TransferOverrideConfiguration implements + ToCopyableBuilder { + private final Long maxTransferBytesPerSecond; + private final MultipartDownloadConfiguration multipartDownloadConfiguration; + private final MultipartUploadConfiguration multipartUploadConfiguration; + + private TransferOverrideConfiguration(BuilderImpl builder) { + this.maxTransferBytesPerSecond = builder.maxTransferBytesPerSecond; + this.multipartDownloadConfiguration = builder.multipartDownloadConfiguration; + this.multipartUploadConfiguration = builder.multipartUploadConfiguration; + } + + /** + * @return The maximum rate for this transfer in bytes per second. + */ + public Long maxTransferBytesPerSecond() { + return maxTransferBytesPerSecond; + } + + /** + * @return The multipart download configuration to use for this transfer. + */ + public MultipartDownloadConfiguration multipartDownloadConfiguration() { + return multipartDownloadConfiguration; + } + + /** + * @return The multipart upload configuration to use for this transfer. + */ + public MultipartUploadConfiguration multipartUploadConfiguration() { + return multipartUploadConfiguration; + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public static Builder builder() { + return new BuilderImpl(); + } + + public interface Builder extends CopyableBuilder { + /** + * Set the maximum rate for this transfer in bytes per second. + * + * @param maxTransferBytesPerSecond The maximum rate. + * @return This object for method chaining. + */ + Builder maxTransferBytesPerSecond(Long maxTransferBytesPerSecond); + + /** + * Set the multipart download configuration to use for this transfer. + * + * @param multipartDownloadConfiguration The multipart download + * configuration. + * @return This object for method chaining. + */ + Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration); + + /** + * Set the multipart upload configuration to use for this transfer. + * + * @param multipartUploadConfiguration The multipart upload + * configuration. + * @return This object for method chaining. + */ + Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration); + + /** + * @return The build override configuration. + */ + TransferOverrideConfiguration build(); + } + + private static final class BuilderImpl implements Builder { + private Long maxTransferBytesPerSecond; + private MultipartDownloadConfiguration multipartDownloadConfiguration; + private MultipartUploadConfiguration multipartUploadConfiguration; + + private BuilderImpl(TransferOverrideConfiguration other) { + this.maxTransferBytesPerSecond = other.maxTransferBytesPerSecond; + this.multipartDownloadConfiguration = other.multipartDownloadConfiguration; + this.multipartUploadConfiguration = other.multipartUploadConfiguration; + } + + private BuilderImpl() { + } + + @Override + public Builder maxTransferBytesPerSecond(Long maxTransferBytesPerSecond) { + this.maxTransferBytesPerSecond = maxTransferBytesPerSecond; + return this; + } + + @Override + public Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration) { + this.multipartDownloadConfiguration = multipartDownloadConfiguration; + return this; + } + + @Override + public Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration) { + this.multipartUploadConfiguration = multipartUploadConfiguration; + return this; + } + + @Override + public TransferOverrideConfiguration build() { + return new TransferOverrideConfiguration(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferProgressListener.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferProgressListener.java new file mode 100644 index 000000000000..e04e4498a58b --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferProgressListener.java @@ -0,0 +1,86 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.time.Duration; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; + +/** + * Progress listener for a Transfer. + *

+ * Implementations must be thread-safe. + */ +@SdkPublicApi +@ThreadSafe +public interface TransferProgressListener { + /** + * Called when a new progress event is available for a Transfer. + * + * @param ctx The context object for the given transfer event. + */ + void transferProgressEvent(EventContext ctx); + + interface EventContext { + /** + * The transfer this listener of associated with. + */ + Transfer transfer(); + } + + interface Initiated extends EventContext { + /** + * The amount of time that has elapsed since the transfer was + * initiated. + */ + Duration elapsedTime(); + } + + interface BytesTransferred extends Initiated { + /** + * The transfer request for the object whose bytes were transferred. + */ + TransferObjectRequest objectRequest(); + + /** + * The number of bytes transferred for this event. + */ + long bytes(); + + /** + * The total length of the object. + */ + long objectLength(); + + /** + * If the transfer of the given object is complete. + */ + boolean complete(); + } + + interface Completed extends Initiated { + } + + interface Cancelled extends Initiated { + } + + interface Failed extends Initiated { + /** + * The error. + */ + Throwable error(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferRequest.java new file mode 100644 index 000000000000..31cc94465af0 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferRequest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Base class for all transfer requests. + */ +@SdkPublicApi +public abstract class TransferRequest implements ToCopyableBuilder { + private TransferOverrideConfiguration overrideConfiguration; + private List progressListeners; + + protected TransferRequest(BuilderImpl builder) { + this.overrideConfiguration = builder.overrideConfiguration; + this.progressListeners = builder.progressListeners; + } + + /** + * @return The optional override configuration for this request. + */ + public Optional overrideConfiguration() { + return Optional.ofNullable(overrideConfiguration); + } + + /** + * @return The optional progress listeners for this request. + */ + public List progressListeners() { + return progressListeners; + } + + public interface Builder extends CopyableBuilder { + /** + * Set the optional override configuration for this request. Override + * configurations take precedence those set on {@link + * S3TransferManager}. + * + * @param overrideConfiguration The override configuration. + * @return This object for method chaining. + */ + Builder overrideConfiguration(TransferOverrideConfiguration overrideConfiguration); + + /** + * Set the optional list of progress listeners for this request. This + * list overwrites any previously added progress listeners. + * @param progressListeners The progress listeners. + * @return This object for method chaining. + */ + Builder progressListeners(Collection progressListeners); + + /** + * Add an additional progress listener to the current configured list of listeners. + * @param progressListener The progress listener. + * @return This object for method chaining. + */ + Builder addProgressListener(TransferProgressListener progressListener); + } + + protected abstract static class BuilderImpl implements Builder { + private TransferOverrideConfiguration overrideConfiguration; + private List progressListeners; + + protected BuilderImpl(TransferRequest other) { + this.overrideConfiguration = other.overrideConfiguration; + if (other.progressListeners != null) { + this.progressListeners = new ArrayList<>(other.progressListeners); + } + } + + protected BuilderImpl() { + } + + @Override + public Builder overrideConfiguration(TransferOverrideConfiguration overrideConfiguration) { + this.overrideConfiguration = overrideConfiguration; + return this; + } + + @Override + public Builder progressListeners(Collection progressListeners) { + this.progressListeners = new ArrayList<>(progressListeners); + return this; + } + + @Override + public Builder addProgressListener(TransferProgressListener progressListener) { + if (this.progressListeners == null) { + this.progressListeners = new ArrayList<>(); + } + this.progressListeners.add(progressListener); + return this; + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Upload.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Upload.java new file mode 100644 index 000000000000..7d9de9105caf --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/Upload.java @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * An upload transfer of a single object to S3. + */ +@SdkPublicApi +public interface Upload extends Transfer { + @Override + CompletableFuture completionFuture(); + + UploadState pause(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadDirectory.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadDirectory.java new file mode 100644 index 000000000000..13d329b96cb8 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadDirectory.java @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * An upload transfer of an directory to S3. + */ +@SdkPublicApi +public interface UploadDirectory extends Transfer { + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadObjectSpecification.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadObjectSpecification.java new file mode 100644 index 000000000000..c0c546d346d7 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadObjectSpecification.java @@ -0,0 +1,70 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.net.URL; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * Union of the various ways to specify how to upload an object from S3. + */ +@SdkPublicApi +public abstract class UploadObjectSpecification { + + UploadObjectSpecification() { + } + + /** + * @return {@code true} if this is an API request, {@code false} otherwise. + */ + public boolean isApiRequest() { + return false; + } + + /** + * @return {@code true} if this is a presigned URL, {@code false} otherwise. + */ + public boolean isPresignedUrl() { + return false; + } + + /** + * @return This specification as an API request. + * @throws IllegalStateException If this is not an API request. + */ + public PutObjectRequest asApiRequest() { + throw new IllegalStateException("Not an API request"); + } + + /** + * @return This specification as a presigned URL. + * @throws IllegalStateException If this is not a presigned URL. + */ + public URL asPresignedUrl() { + throw new IllegalStateException("Not a presigned URL"); + } + + /** + * Create a specification from a {@link PutObjectRequest}. + * + * @param apiRequest The request. + * @return The new API request specification. + */ + static UploadObjectSpecification fromApiRequest(PutObjectRequest apiRequest) { + return new ApiRequestUploadObjectSpecification(apiRequest); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java new file mode 100644 index 000000000000..285e854ce761 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.util.Collection; +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Upload an object to S3 using {@link S3TransferManager}. + */ +@SdkPublicApi +public final class UploadRequest extends TransferRequest { + private final UploadObjectSpecification uploadSpecification; + private final Long size; + + private UploadRequest(BuilderImpl builder) { + super(builder); + this.uploadSpecification = builder.uploadSpecification; + this.size = builder.size; + } + + /** + * @return The upload specification. + */ + public UploadObjectSpecification uploadSpecification() { + return uploadSpecification; + } + + /** + * @return The size of the object to be uploaded. + */ + public Long size() { + return size; + } + + public static Builder builder() { + return new BuilderImpl(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UploadRequest that = (UploadRequest) o; + return Objects.equals(uploadSpecification, that.uploadSpecification) && + Objects.equals(size, that.size); + } + + @Override + public int hashCode() { + int hashCode = 1; + hashCode = hashCode * 31 + Objects.hashCode(uploadSpecification); + hashCode = hashCode * 31 + Objects.hashCode(size); + return hashCode; + } + + @Override + public TransferRequest.Builder toBuilder() { + return null; + } + + public interface Builder extends TransferRequest.Builder { + + /** + * Set the upload specification. + * + * @param uploadSpecification The upload specification. + * @return This object for method chaining. + */ + Builder uploadSpecification(UploadObjectSpecification uploadSpecification); + + /** + * Set the size of the object to be uploaded. + * + * @param size The object size. + * @return This object for method chaining. + */ + Builder size(Long size); + + @Override + Builder overrideConfiguration(TransferOverrideConfiguration overrideConfiguration); + + @Override + Builder progressListeners(Collection progressListeners); + + @Override + Builder addProgressListener(TransferProgressListener progressListener); + + UploadRequest build(); + } + + private static class BuilderImpl extends TransferRequest.BuilderImpl implements Builder { + private UploadObjectSpecification uploadSpecification; + private Long size; + + @Override + public Builder uploadSpecification(UploadObjectSpecification uploadSpecification) { + this.uploadSpecification = uploadSpecification; + return this; + } + + @Override + public Builder size(Long size) { + this.size = size; + return this; + } + + @Override + public Builder overrideConfiguration(TransferOverrideConfiguration config) { + super.overrideConfiguration(config); + return this; + } + + @Override + public Builder progressListeners(Collection progressListeners) { + super.progressListeners(progressListeners); + return this; + } + + @Override + public Builder addProgressListener(TransferProgressListener progressListener) { + super.addProgressListener(progressListener); + return this; + } + + @Override + public UploadRequest build() { + return new UploadRequest(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadState.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadState.java new file mode 100644 index 000000000000..6f9e5cad2071 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadState.java @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import java.io.InputStream; +import java.io.OutputStream; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * The state of a paused upload. The upload can be resumed using + * {@link S3TransferManager#resumeUpload(UploadState)}. + */ +@SdkPublicApi +public class UploadState { + /** + * Persist this state to the given stream. + * + * @param os The stream to write this state to. + */ + public void persistTo(OutputStream os) { + throw new UnsupportedOperationException(); + } + + /** + * Load a persisted state. + * + * @param is The stream to read the state from. + * @return The loaded state. + */ + public static DownloadState loadFrom(InputStream is) { + throw new UnsupportedOperationException(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/ConfigHelper.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/ConfigHelper.java new file mode 100644 index 000000000000..3567ff05f924 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/ConfigHelper.java @@ -0,0 +1,62 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; +import software.amazon.awssdk.custom.s3.transfer.MultipartDownloadConfiguration; +import software.amazon.awssdk.custom.s3.transfer.TransferOverrideConfiguration; +import software.amazon.awssdk.utils.Validate; + +/** + * Helper class to determine appropriate configuration values based on global + * settings set on the transfer manager and per request overrides. + */ +@SdkInternalApi +final class ConfigHelper { + private final MultipartDownloadConfiguration globalDownloadConfig; + + ConfigHelper(MultipartDownloadConfiguration globalDownloadConfig) { + this.globalDownloadConfig = Validate.notNull(globalDownloadConfig, "globalDownloadConfig must not be null"); + } + + boolean useMultipartDownloads(DownloadRequest downloadRequest) { + return multipartDownloadConfiguration(downloadRequest).enableMultipartDownloads(); + } + + long multipartDownloadThreshold(DownloadRequest downloadRequest) { + return multipartDownloadConfiguration(downloadRequest).multipartDownloadThreshold(); + } + + long minDownloadPartSize(DownloadRequest downloadRequest) { + return multipartDownloadConfiguration(downloadRequest).minDownloadPartSize(); + } + + int maxDownloadPartCount(DownloadRequest downloadRequest) { + return multipartDownloadConfiguration(downloadRequest).maxDownloadPartCount(); + } + + /** + * Resolve the correct {@link MultipartDownloadConfiguration} to use for + * this request. The override set on the request takes precedence, then + * the config set globally on TransferManager. + */ + private MultipartDownloadConfiguration multipartDownloadConfiguration(DownloadRequest downloadRequest) { + return downloadRequest.overrideConfiguration() + .map(TransferOverrideConfiguration::multipartDownloadConfiguration) + .orElse(globalDownloadConfig); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java new file mode 100644 index 000000000000..a4daf2d64f27 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java @@ -0,0 +1,223 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.custom.s3.transfer.Download; +import software.amazon.awssdk.custom.s3.transfer.DownloadDirectory; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; +import software.amazon.awssdk.custom.s3.transfer.DownloadState; +import software.amazon.awssdk.custom.s3.transfer.MultipartDownloadConfiguration; +import software.amazon.awssdk.custom.s3.transfer.MultipartUploadConfiguration; +import software.amazon.awssdk.custom.s3.transfer.S3TransferManager; +import software.amazon.awssdk.custom.s3.transfer.TransferProgressListener; +import software.amazon.awssdk.custom.s3.transfer.Upload; +import software.amazon.awssdk.custom.s3.transfer.UploadDirectory; +import software.amazon.awssdk.custom.s3.transfer.UploadRequest; +import software.amazon.awssdk.custom.s3.transfer.UploadState; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +/** + * Default implementation of the {@link S3TransferManager}. + */ +@SdkInternalApi +public final class DefaultS3TransferManager implements S3TransferManager { + private final S3AsyncClient s3Client; + private final boolean manageS3Client; + private final Integer maxConcurrency; + private final Long maxUploadBytesPerSecond; + private final Long maxDownloadBytesPerSecond; + private final MultipartDownloadConfiguration multipartDownloadConfiguration; + private final MultipartUploadConfiguration multipartUploadConfiguration; + private final List progressListeners; + + private final DownloadManager downloadManager; + + private DefaultS3TransferManager(BuilderImpl builder) { + if (builder.s3Client != null) { + this.s3Client = builder.s3Client; + this.manageS3Client = false; + } else { + this.s3Client = S3AsyncClient.create(); + this.manageS3Client = true; + } + + this.maxConcurrency = builder.maxConcurrency; + this.maxUploadBytesPerSecond = builder.maxUploadBytesPerSecond; + this.maxDownloadBytesPerSecond = builder.maxDownloadBytesPerSecond; + this.multipartDownloadConfiguration = resolveMultipartDownloadConfiguration(builder.multipartDownloadConfiguration); + this.multipartUploadConfiguration = resolveMultipartUploadConfiguration(builder.multipartUploadConfiguration); + this.progressListeners = resolveProgressListeners(builder.progressListeners); + + this.downloadManager = new DownloadManager(this.s3Client, this.multipartDownloadConfiguration); + } + + @Override + public Download download(DownloadRequest request, Path file) { + return download(request, TransferResponseTransformer.forFile(file)); + } + + @Override + public Download resumeDownload(DownloadState downloadState) { + throw new UnsupportedOperationException(); + } + + @Override + public DownloadDirectory downloadDirectory(String bucket, String prefix, Path destinationDirectory) { + throw new UnsupportedOperationException(); + } + + @Override + public Upload upload(UploadRequest request, Path file) { + throw new UnsupportedOperationException(); + } + + @Override + public Upload resumeUpload(UploadState uploadState) { + throw new UnsupportedOperationException(); + } + + @Override + public UploadDirectory uploadDirectory(String bucket, String prefix, Path sourceDirectory) { + throw new UnsupportedOperationException(); + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + @Override + public void close() { + if (manageS3Client) { + s3Client.close(); + } + } + + public static Builder builder() { + return new BuilderImpl(); + } + + private Download download(DownloadRequest request, TransferResponseTransformer responseTransformer) { + return downloadManager.downloadObject(request, responseTransformer); + } + + private static MultipartDownloadConfiguration resolveMultipartDownloadConfiguration( + MultipartDownloadConfiguration configured) { + if (configured != null) { + return configured; + } + return MultipartDownloadConfiguration.defaultConfig(); + } + + private static MultipartUploadConfiguration resolveMultipartUploadConfiguration(MultipartUploadConfiguration configured) { + if (configured != null) { + return configured; + } + return MultipartUploadConfiguration.defaultConfig(); + } + + private static List resolveProgressListeners(List configured) { + if (configured != null) { + return configured; + } + return Collections.emptyList(); + } + + private static class BuilderImpl implements S3TransferManager.Builder { + private S3AsyncClient s3Client; + private Integer maxConcurrency; + private Long maxUploadBytesPerSecond; + private Long maxDownloadBytesPerSecond; + private MultipartDownloadConfiguration multipartDownloadConfiguration; + private MultipartUploadConfiguration multipartUploadConfiguration; + private List progressListeners; + + private BuilderImpl(DefaultS3TransferManager transferManager) { + this.s3Client = transferManager.s3Client; + this.maxConcurrency = transferManager.maxConcurrency; + this.maxDownloadBytesPerSecond = transferManager.maxDownloadBytesPerSecond; + this.maxUploadBytesPerSecond = transferManager.maxUploadBytesPerSecond; + this.multipartDownloadConfiguration = transferManager.multipartDownloadConfiguration; + this.multipartUploadConfiguration = transferManager.multipartUploadConfiguration; + this.progressListeners = new ArrayList<>(transferManager.progressListeners); + } + + private BuilderImpl() { + } + + @Override + public S3TransferManager.Builder s3client(S3AsyncClient s3Client) { + this.s3Client = s3Client; + return this; + } + + @Override + public Builder maxConcurrency(Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + @Override + public Builder maxUploadBytesPerSecond(Long maxUploadBytesPerSecond) { + this.maxUploadBytesPerSecond = maxUploadBytesPerSecond; + return this; + } + + @Override + public Builder maxDownloadBytesPerSecond(Long maxDownloadBytesPerSecond) { + this.maxDownloadBytesPerSecond = maxDownloadBytesPerSecond; + return this; + } + + @Override + public Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration) { + this.multipartDownloadConfiguration = multipartDownloadConfiguration; + return this; + } + + @Override + public Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration) { + this.multipartUploadConfiguration = multipartUploadConfiguration; + return this; + } + + @Override + public S3TransferManager.Builder addProgressListener(TransferProgressListener progressListener) { + if (this.progressListeners == null) { + this.progressListeners = new ArrayList<>(); + } + this.progressListeners.add(progressListener); + return this; + } + + @Override + public S3TransferManager.Builder progressListeners(Collection progressListeners) { + this.progressListeners = new ArrayList<>(progressListeners); + return this; + } + + @Override + public S3TransferManager build() { + return new DefaultS3TransferManager(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManager.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManager.java new file mode 100644 index 000000000000..f7e8b3339113 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManager.java @@ -0,0 +1,277 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.custom.s3.transfer.CompletedDownload; +import software.amazon.awssdk.custom.s3.transfer.Download; +import software.amazon.awssdk.custom.s3.transfer.DownloadObjectSpecification; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; +import software.amazon.awssdk.custom.s3.transfer.DownloadState; +import software.amazon.awssdk.custom.s3.transfer.MultipartDownloadConfiguration; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Logger; + +/** + * Performs download operations for {@link software.amazon.awssdk.custom.s3.transfer.S3TransferManager}. + */ +@ThreadSafe +@SdkInternalApi +final class DownloadManager { + private static final Logger log = Logger.loggerFor(DownloadManager.class); + + private final S3AsyncClient s3Client; + private final MultipartDownloadConfiguration globalDownloadConfig; + private final ConfigHelper configHelper; + + DownloadManager(S3AsyncClient s3Client, MultipartDownloadConfiguration globalDownloadConfig) { + this.s3Client = s3Client; + this.globalDownloadConfig = globalDownloadConfig; + this.configHelper = new ConfigHelper(this.globalDownloadConfig); + } + + Download downloadObject(DownloadRequest downloadRequest, TransferResponseTransformer partTransformerCreator) { + DownloadObjectSpecification spec = downloadRequest.downloadSpecification(); + if (spec.isPresignedUrl()) { + throw new UnsupportedOperationException("Downloading Presigned URL not supported"); + } + + GetObjectRequest getObjectRequest = spec.asApiRequest(); + return doApiDownload(downloadRequest, getObjectRequest, partTransformerCreator); + } + + private Download doApiDownload(DownloadRequest downloadRequest, + GetObjectRequest getObjectRequest, + TransferResponseTransformer partTransformerCreator) { + + CompletableFuture partsCompleteFuture; + if (!configHelper.useMultipartDownloads(downloadRequest)) { + log.debug(() -> String.format("Multipart downloads disabled for %s, downloading as a single part", downloadRequest)); + partsCompleteFuture = doSinglePartDownload(downloadRequest, getObjectRequest, partTransformerCreator); + } else { + CompletableFuture objectSize = determineObjectSize(downloadRequest); + + partsCompleteFuture = objectSize.thenCompose(size -> { + long threshold = configHelper.multipartDownloadThreshold(downloadRequest); + if (size >= threshold) { + log.debug(() -> String.format("Downloading %s in multiple parts", downloadRequest)); + return doMultipartDownload(downloadRequest, getObjectRequest, size, partTransformerCreator); + } + log.debug(() -> String.format("Size %d of %s does not meet configured threshold of %d bytes. " + + "Downloading as a single part", size, downloadRequest, threshold)); + return doSinglePartDownload(downloadRequest, getObjectRequest, partTransformerCreator); + }); + } + return new DownloadImpl(partsCompleteFuture); + } + + private CompletableFuture doSinglePartDownload(DownloadRequest downloadRequest, + GetObjectRequest getObjectRequest, + TransferResponseTransformer transformerCreator) { + SinglePartDownloadContext ctx = SinglePartDownloadContext.builder() + .downloadRequest(downloadRequest) + .getObjectRequest(getObjectRequest.toBuilder() + .overrideConfiguration(o -> o.addApiName(TransferManagerUtilities.apiName())) + .build()) + .build(); + try { + AsyncResponseTransformer transformer = transformerCreator.transformerForObject(ctx); + return s3Client.getObject(getObjectRequest, transformer).thenApply(r -> null); + } catch (Throwable t) { + return CompletableFutureUtils.failedFuture(t); + } + + } + + private CompletableFuture doMultipartDownload(DownloadRequest downloadRequest, + GetObjectRequest getObjectRequest, + long size, + TransferResponseTransformer transformerCreator) { + try { + List contexts = multipartContexts(downloadRequest, size, getObjectRequest); + // Ensure we resolve the transformers first in case the callback throws an error while creating one of them + List transformersWithContext = + createPartTransformersForContexts(contexts, transformerCreator); + + return allOfCancelForwarded(transformersWithContext.stream() + .map(pair -> s3Client.getObject(pair.context.partDownloadSpecification().asApiRequest(), pair.transformer)) + .toArray(CompletableFuture[]::new)); + } catch (Throwable t) { + return CompletableFutureUtils.failedFuture(t); + } + } + + private List createPartTransformersForContexts(List contexts, + TransferResponseTransformer transformerCreator) { + return contexts.stream() + .map(ctx -> new TransformerContextPair(transformerCreator.transformerForObjectPart(ctx), ctx)) + .collect(Collectors.toList()); + } + + private List multipartContexts(DownloadRequest downloadRequest, + long size, + GetObjectRequest getObjectRequest) { + int maxPartCount = configHelper.maxDownloadPartCount(downloadRequest); + + // First calculate maximum part count based on min part size. This + // should always be positive since we enforce that min part size is <= + // threshold + long partCount = size / configHelper.minDownloadPartSize(downloadRequest); + + // Clamp to configured max part count + long finalPartCount = Math.min(partCount, maxPartCount); + + // Calculate final part size + long partSize = (long) Math.ceil((double) size / finalPartCount); + + log.debug(() -> String.format("Downloading %s in %d parts of size %d", downloadRequest, finalPartCount, partSize)); + + List contexts = new ArrayList<>((int) finalPartCount); + + GetObjectRequest.Builder requestBuilder = getObjectRequest.toBuilder() + .overrideConfiguration(o -> o.addApiName(TransferManagerUtilities.apiName())); + + long startByte = 0L; + long remaining = size; + for (int p = 0; p < finalPartCount; ++p) { + int partNum = p + 1; + boolean lastPart = partNum == finalPartCount; + long sz = remaining > partSize ? partSize : remaining; + // range is inclusive + long a = startByte; + long b = a + sz - 1; + + String range = TransferManagerUtilities.rangeHeaderValue(a, b); + GetObjectRequest request = requestBuilder.range(range) + .build(); + + MultipartDownloadContext ctx = MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(request)) + .partOffset(a) + .partNumber(partNum) + .size(sz) + .isLastPart(lastPart) + .build(); + + contexts.add(ctx); + + startByte = a + sz; + remaining -= sz; + } + + return contexts; + } + + /** + * Determine the size of the object being requested. First checks if this + * information is provided by the user, otherwise it gets it from S3. + */ + // TODO: How do we want to deal with a request or presigned URL that has the Range header set? + private CompletableFuture determineObjectSize(DownloadRequest downloadRequest) { + Long providedSize = downloadRequest.size(); + + if (providedSize != null) { + return CompletableFuture.completedFuture(providedSize); + } + + DownloadObjectSpecification spec = downloadRequest.downloadSpecification(); + if (spec.isApiRequest()) { + log.debug(() -> String.format("Object size not given for %s, retrieving size from S3", downloadRequest)); + return getObjectSizeFromS3(spec.asApiRequest()); + } + + return CompletableFutureUtils.failedFuture( + new UnsupportedOperationException("Don't know how to get the size for spec " + spec)); + } + + /** + * Perform a HeadObject call to determine the size of the object. + */ + private CompletableFuture getObjectSizeFromS3(GetObjectRequest getObjectRequest) { + HeadObjectRequest headObjectRequest = RequestConversionUtils.toHeadObjectRequest(getObjectRequest).toBuilder() + .overrideConfiguration(o -> o.addApiName(TransferManagerUtilities.apiName())) + .build(); + return s3Client.headObject(headObjectRequest).thenApply(HeadObjectResponse::contentLength); + } + + private static final class DownloadImpl implements Download { + private final CompletableFuture completionFuture; + private final CompletableFuture partsCompleteFuture; + + DownloadImpl(CompletableFuture partsCompleteFuture) { + this.partsCompleteFuture = partsCompleteFuture; + this.completionFuture = this.partsCompleteFuture.thenApply(ignored -> null); + CompletableFutureUtils.forwardExceptionTo(this.completionFuture, this.partsCompleteFuture); + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public DownloadState pause() { + // TODO: Implement download pause + throw new UnsupportedOperationException(); + } + } + + private static class TransformerContextPair { + private final AsyncResponseTransformer transformer; + private final MultipartDownloadContext context; + + TransformerContextPair(AsyncResponseTransformer transformer, + MultipartDownloadContext context) { + this.transformer = transformer; + this.context = context; + } + } + + /** + * Similar to {@link CompletableFuture#allOf(CompletableFuture[])}, but + * when the returned future is completed exceptionally, forwards the + * cancel to the individual futures. + * + * @param futures The futures. + * @return The new future that is completed when all the futures in {@code + * futures} are. + */ + private static CompletableFuture allOfCancelForwarded(CompletableFuture[] futures) { + CompletableFuture[] futuresCopy = new CompletableFuture[futures.length]; + System.arraycopy(futures, 0, futuresCopy, 0, futures.length); + CompletableFuture allOfFuture = CompletableFuture.allOf(futuresCopy); + allOfFuture.whenComplete((r, e) -> { + if (e != null) { + for (CompletableFuture cf : futuresCopy) { + cf.completeExceptionally(e); + } + } + }); + return allOfFuture; + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/FileTransferResponseTransformer.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/FileTransferResponseTransformer.java new file mode 100644 index 000000000000..e20077e14b78 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/FileTransferResponseTransformer.java @@ -0,0 +1,44 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.nio.file.Path; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * Response transfer for writing objects to disk. + */ +@SdkInternalApi +public final class FileTransferResponseTransformer implements TransferResponseTransformer { + private final Path path; + + public FileTransferResponseTransformer(Path path) { + this.path = path; + } + + @Override + public AsyncResponseTransformer transformerForObjectPart(MultipartDownloadContext context) { + return AsyncResponseTransformer.toFile(path, context.partOffset(), false, false); + } + + @Override + public AsyncResponseTransformer transformerForObject(SinglePartDownloadContext context) { + return AsyncResponseTransformer.toFile(path); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartDownloadContext.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartDownloadContext.java new file mode 100644 index 000000000000..76adc6f861ca --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartDownloadContext.java @@ -0,0 +1,224 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.custom.s3.transfer.DownloadObjectSpecification; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; + +/** + * Context object for a multipart download. + */ +@SdkInternalApi +public final class MultipartDownloadContext { + private DownloadRequest downloadRequest; + private DownloadObjectSpecification partDownloadSpecification; + private int partNumber; + private long partOffset; + private final long size; + private boolean isLastPart; + + private MultipartDownloadContext(BuilderImpl builder) { + this.downloadRequest = builder.downloadRequest; + this.partDownloadSpecification = builder.partDownloadSpecification; + this.partNumber = builder.partNumber; + this.partOffset = builder.partOffset; + this.size = builder.size; + this.isLastPart = builder.isLastPart; + } + + /** + * @return The original download request given to the Transfer Manager. + */ + public DownloadRequest downloadRequest() { + return downloadRequest; + } + + /** + * @return The download specification used to get this part from S3. + */ + public DownloadObjectSpecification partDownloadSpecification() { + return partDownloadSpecification; + } + + /** + * @return The part number. + */ + public int partNumber() { + return partNumber; + } + + /** + * @return The offset from the beginning of the object where this part begins. + */ + public long partOffset() { + return partOffset; + } + + /** + * @return The size of the part requested. + */ + public long size() { + return size; + } + + /** + * @return Whether this is the last part of the object. + */ + public boolean isLastPart() { + return isLastPart; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MultipartDownloadContext that = (MultipartDownloadContext) o; + return partNumber == that.partNumber && + partOffset == that.partOffset && + size == that.size && + isLastPart == that.isLastPart && + Objects.equals(downloadRequest, that.downloadRequest) && + Objects.equals(partDownloadSpecification, that.partDownloadSpecification); + } + + @Override + public int hashCode() { + int hashCode = 1; + hashCode = 31 * hashCode + Objects.hashCode(downloadRequest); + hashCode = 31 * hashCode + Objects.hashCode(partDownloadSpecification); + hashCode = 31 * hashCode + Objects.hashCode(partNumber); + hashCode = 31 * hashCode + Objects.hashCode(partOffset); + hashCode = 31 * hashCode + Objects.hashCode(size); + hashCode = 31 * hashCode + Objects.hashCode(isLastPart); + return hashCode; + } + + public static Builder builder() { + return new BuilderImpl(); + } + + public interface Builder { + /** + * Set the original download request given to the Transfer Manager. + * + * @param downloadRequest The download request. + * @return This object for method chaining. + */ + Builder downloadRequest(DownloadRequest downloadRequest); + + /** + * Set the download specification used to get this part from S3. + * + * @param partDownloadSpecification The download specification. + * @return This object for method chaining. + */ + Builder partDownloadSpecification(DownloadObjectSpecification partDownloadSpecification); + + /** + * Set the part number. + * + * @param partNumber The part number. + * @return This object for method chaining. + */ + Builder partNumber(int partNumber); + + /** + * Set the offset from the beginning of the object where this part + * begins. + * + * @param partOffset The part offset. + * @return This object for method chaining. + */ + Builder partOffset(long partOffset); + + /** + * Set the size of the part requested. + * + * @param size The size of the part. + * @return This object for method chaining. + */ + Builder size(long size); + + /** + * Set whether this is the last part of the object. + * + * @param isLastPart Whether this is the last part. + * @return This object for method chaining. + */ + Builder isLastPart(boolean isLastPart); + + /** + * @return The build context. + */ + MultipartDownloadContext build(); + } + + private static class BuilderImpl implements Builder { + private DownloadRequest downloadRequest; + private DownloadObjectSpecification partDownloadSpecification; + private int partNumber; + private long partOffset; + private long size; + private boolean isLastPart; + + @Override + public Builder downloadRequest(DownloadRequest downloadRequest) { + this.downloadRequest = downloadRequest; + return this; + } + + @Override + public Builder partDownloadSpecification(DownloadObjectSpecification partDownloadSpecification) { + this.partDownloadSpecification = partDownloadSpecification; + return this; + } + + @Override + public Builder partNumber(int partNumber) { + this.partNumber = partNumber; + return this; + } + + @Override + public Builder partOffset(long partOffset) { + this.partOffset = partOffset; + return this; + } + + @Override + public Builder size(long size) { + this.size = size; + return this; + } + + @Override + public Builder isLastPart(boolean isLastPart) { + this.isLastPart = isLastPart; + return this; + } + + @Override + public MultipartDownloadContext build() { + return new MultipartDownloadContext(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartUploadContext.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartUploadContext.java new file mode 100644 index 000000000000..dc9582af8ff9 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/MultipartUploadContext.java @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.custom.s3.transfer.UploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; + +/** + * The context object for the upload of an object part to S3. + */ +@SdkInternalApi +public final class MultipartUploadContext { + private final UploadRequest uploadRequest; + private final CreateMultipartUploadRequest createMultipartRequest; + private final UploadPartRequest uploadPartRequest; + private final long partOffset; + + private MultipartUploadContext(BuilderImpl builder) { + this.uploadRequest = builder.uploadRequest; + this.createMultipartRequest = builder.createMultipartRequest; + this.uploadPartRequest = builder.uploadPartRequest; + this.partOffset = builder.partOffset; + } + + /** + * The original upload request given to the transfer manager. + */ + public UploadRequest uploadRequest() { + return uploadRequest; + } + + /** + * The request sent to S3 to initiate the multipart upload. + */ + public CreateMultipartUploadRequest createMultipartRequest() { + return createMultipartRequest; + } + + /** + * The upload request to be sent to S3 for this part. + */ + public UploadPartRequest uploadPartRequest() { + return uploadPartRequest; + } + + /** + * The offset from the beginning of the object where this part begins. + */ + public long partOffset() { + return partOffset; + } + + public interface Builder { + /** + * Set the original upload request given to the transfer manager. + * + * @param uploadRequest The upload request. + * @return This object for method chaining. + */ + Builder uploadRequest(UploadRequest uploadRequest); + + /** + * Set the request sent to S3 to initiate the multipart upload. + * + * @param createMultipartRequest The request. + * @return This object for method chaining. + */ + Builder createMultipartRequest(CreateMultipartUploadRequest createMultipartRequest); + + /** + * Set the upload request to be sent to S3 for this part. + * + * @param uploadPartRequest The request. + * @return This object for method chaining. + */ + Builder uploadPartRequest(UploadPartRequest uploadPartRequest); + + /** + * Set the offset from the beginning of the object where this part + * begins. + * + * @param partOffset The offset. + * @return This object for method chaining. + */ + Builder partOffset(long partOffset); + + /** + * @return the built context. + */ + MultipartUploadContext build(); + } + + private static final class BuilderImpl implements Builder { + private UploadRequest uploadRequest; + private CreateMultipartUploadRequest createMultipartRequest; + private UploadPartRequest uploadPartRequest; + private long partOffset; + + @Override + public Builder uploadRequest(UploadRequest uploadRequest) { + this.uploadRequest = uploadRequest; + return this; + } + + @Override + public Builder createMultipartRequest(CreateMultipartUploadRequest createMultipartRequest) { + this.createMultipartRequest = createMultipartRequest; + return this; + } + + @Override + public Builder uploadPartRequest(UploadPartRequest uploadPartRequest) { + this.uploadPartRequest = uploadPartRequest; + return this; + } + + @Override + public Builder partOffset(long partOffset) { + this.partOffset = partOffset; + return this; + } + + @Override + public MultipartUploadContext build() { + return new MultipartUploadContext(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtils.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtils.java new file mode 100644 index 000000000000..d8bb55ace9c2 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtils.java @@ -0,0 +1,51 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; + +/** + * Utility class for converting between similar S3 requests. E.g. GetObjectRequest -> HeadObjectRequest. + */ +//TODO: This can be replaced by code generation, which will also be more reliable +@SdkInternalApi +final class RequestConversionUtils { + private RequestConversionUtils() { + } + + /** + * Convert {@link GetObjectRequest} to a {@link HeadObjectRequest}. + */ + static HeadObjectRequest toHeadObjectRequest(GetObjectRequest getObjectRequest) { + return HeadObjectRequest.builder() + .bucket(getObjectRequest.bucket()) + .ifMatch(getObjectRequest.ifMatch()) + .ifModifiedSince(getObjectRequest.ifModifiedSince()) + .ifNoneMatch(getObjectRequest.ifNoneMatch()) + .ifUnmodifiedSince(getObjectRequest.ifUnmodifiedSince()) + .key(getObjectRequest.key()) + .partNumber(getObjectRequest.partNumber()) + .range(getObjectRequest.range()) + .requestPayer(getObjectRequest.requestPayerAsString()) + .sseCustomerAlgorithm(getObjectRequest.sseCustomerAlgorithm()) + .sseCustomerKey(getObjectRequest.sseCustomerKey()) + .sseCustomerKeyMD5(getObjectRequest.sseCustomerKeyMD5()) + .versionId(getObjectRequest.versionId()) + .build(); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartDownloadContext.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartDownloadContext.java new file mode 100644 index 000000000000..79ea42cfd774 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartDownloadContext.java @@ -0,0 +1,88 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +/** + * The context object for a single part download. + */ +@SdkInternalApi +public final class SinglePartDownloadContext { + private final DownloadRequest downloadRequest; + private final GetObjectRequest getObjectRequest; + + private SinglePartDownloadContext(BuilderImpl builder) { + this.downloadRequest = builder.downloadRequest; + this.getObjectRequest = builder.getObjectRequest; + } + + /** + * @return The original download request given to the transfer manager. + */ + public DownloadRequest downloadRequest() { + return downloadRequest; + } + + public GetObjectRequest getObjectRequest() { + return getObjectRequest; + } + + public static Builder builder() { + return new BuilderImpl(); + } + + public interface Builder { + /** + * Set the original download request given to the transfer manager. + * + * @param downloadRequest The original download request. + * @return This object for method chaining. + */ + Builder downloadRequest(DownloadRequest downloadRequest); + + Builder getObjectRequest(GetObjectRequest getObjectRequest); + + /** + * @return The build context object. + */ + SinglePartDownloadContext build(); + } + + private static final class BuilderImpl implements Builder { + private DownloadRequest downloadRequest; + private GetObjectRequest getObjectRequest; + + @Override + public Builder downloadRequest(DownloadRequest downloadRequest) { + this.downloadRequest = downloadRequest; + return this; + } + + @Override + public Builder getObjectRequest(GetObjectRequest getObjectRequest) { + this.getObjectRequest = getObjectRequest; + return this; + } + + @Override + public SinglePartDownloadContext build() { + return new SinglePartDownloadContext(this); + } + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartUploadContext.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartUploadContext.java new file mode 100644 index 000000000000..3eac53cabb19 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/SinglePartUploadContext.java @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; + +@SdkInternalApi +public interface SinglePartUploadContext { +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferManagerUtilities.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferManagerUtilities.java new file mode 100644 index 000000000000..efeac7c2aef6 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferManagerUtilities.java @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.ApiName; + +/** + * Utility methods used for the transfer manager. + */ +@SdkInternalApi +public final class TransferManagerUtilities { + private static final ApiName API_NAME = ApiName.builder().name("S3TransferManager").version("1.0").build(); + + private TransferManagerUtilities() { + } + + /** + * Create a formatted byte range header value using the given start and end + * positions. + * + * @param start The start position in the range. + * @param end The end position in the range. + * + * @return The formatted {@code Range} header value. + */ + // TODO: This is useful for outside of Transfer Manager as well + public static String rangeHeaderValue(long start, long end) { + return String.format("bytes=%d-%d", start, end); + } + + public static ApiName apiName() { + return API_NAME; + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferRequestBody.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferRequestBody.java new file mode 100644 index 000000000000..ebafbffd1aea --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferRequestBody.java @@ -0,0 +1,44 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.nio.ByteBuffer; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * A factory capable of creating the streams for individual parts of a given + * object to be uploaded to S3. + *

+ * There is no ordering guaranatee for when {@link + * #requestBodyForPart(MultipartUploadContext)} is called. + */ +@SdkInternalApi +public interface TransferRequestBody { + /** + * Return the stream for the object part described by given {@link + * MultipartUploadContext}. + * + * @param context The context describing the part to be uploaded. + * @return The part stream. + */ + Publisher requestBodyForPart(MultipartUploadContext context); + + /** + * Return the stream for a entire object to be uploaded as a single part. + */ + Publisher requestBodyForObject(SinglePartUploadContext context); +} \ No newline at end of file diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferResponseTransformer.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferResponseTransformer.java new file mode 100644 index 000000000000..b968a06ed44b --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/TransferResponseTransformer.java @@ -0,0 +1,51 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import java.nio.file.Path; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * A factory capable of creating the {@link AsyncResponseTransformer} to handle + * each downloaded object part. + *

+ * There is no ordering or synchronization guarantee between {@link + * #transformerForObjectPart(MultipartDownloadContext)} invocations. + */ +@SdkInternalApi +public interface TransferResponseTransformer { + /** + * Create a {@code AsyncResponseTransformer} to handle a single object part + * for multipart downloads. + */ + AsyncResponseTransformer transformerForObjectPart(MultipartDownloadContext context); + + /** + * Create a {@code AsyncResponseTransformer} to handle an entire object for + * a single part download. + */ + AsyncResponseTransformer transformerForObject(SinglePartDownloadContext context); + + /** + * @return A factory capable of creating transformers that will write the + * object to disk. This supports both single and multipart downloads. + */ + static TransferResponseTransformer forFile(Path file) { + return new FileTransferResponseTransformer(file); + } +} diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/util/SizeConstant.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/util/SizeConstant.java new file mode 100644 index 000000000000..59c294265673 --- /dev/null +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/util/SizeConstant.java @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.util; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Helpful constants for common size units. + */ +@SdkPublicApi +public final class SizeConstant { + /** + * 1 Kibibyte. + */ + public static final long KiB = 1024; + + /** + * 1 Mebibyte. + */ + public static final long MiB = 1024 * KiB; + + /** + * 1 Gibibyte. + */ + public static final long GiB = 1024 * MiB; + + private SizeConstant() { + } +} diff --git a/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/TransferManagerUtilitiesTest.java b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/TransferManagerUtilitiesTest.java new file mode 100644 index 000000000000..344a1d2adc7a --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/TransferManagerUtilitiesTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; +import software.amazon.awssdk.custom.s3.transfer.internal.TransferManagerUtilities; + +/** + * Tests for {@link TransferManagerUtilities}. + */ +public class TransferManagerUtilitiesTest { + @Test + public void createsCorrectRangeHeader() { + String rangeHeader = TransferManagerUtilities.rangeHeaderValue(1234, 5678); + assertThat(rangeHeader).isEqualTo("bytes=1234-5678"); + } +} diff --git a/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManagerTest.java b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManagerTest.java new file mode 100644 index 000000000000..d20e044ebcda --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/DownloadManagerTest.java @@ -0,0 +1,451 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.custom.s3.transfer.DownloadRequest; +import software.amazon.awssdk.custom.s3.transfer.TransferOverrideConfiguration; +import software.amazon.awssdk.custom.s3.transfer.util.SizeConstant; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.custom.s3.transfer.DownloadObjectSpecification; +import software.amazon.awssdk.custom.s3.transfer.MultipartDownloadConfiguration; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +/** + * Tests for {@link DownloadManager}. + */ +@RunWith(MockitoJUnitRunner.class) +public class DownloadManagerTest { + private static final String BUCKET = "foo"; + private static final String KEY = "key"; + private static final GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .build(); + + @Mock + private S3AsyncClient s3Client; + + private DownloadManager dm; + + @Before + public void methodSetup() { + dm = new DownloadManager(s3Client, MultipartDownloadConfiguration.defaultConfig()); + + when(s3Client.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .thenAnswer(i -> CompletableFuture.completedFuture(GetObjectResponse.builder().build())); + } + + @Test + public void sizeProvided_DoesNotCallHeadObject_SinglePart() { + DownloadRequest req = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(1L) + .build(); + + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObject(any())).thenReturn(mockTransformer); + dm.downloadObject(req, transformerCreator).completionFuture().join(); + + verify(s3Client, times(0)).headObject(any(HeadObjectRequest.class)); + verify(s3Client, times(1)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + } + + @Test + public void sizeProvided_DoesNotCallHeadObject_Multipart() { + DownloadRequest req = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB) + .build(); + + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObjectPart(any())).thenReturn(mockTransformer); + dm.downloadObject(req, transformerCreator).completionFuture().join(); + + verify(s3Client, times(0)).headObject(any(HeadObjectRequest.class)); + verify(s3Client, times(3)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + } + + @Test + public void sizeNotProvided_CallsHeadObject() { + DownloadRequest req = DownloadRequest.forBucketAndKey(BUCKET, KEY); + + when(s3Client.headObject(any(HeadObjectRequest.class))) + .thenReturn(CompletableFuture.completedFuture(HeadObjectResponse.builder() + .contentLength(1L) + .build())); + + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObject(any())).thenReturn(mockTransformer); + dm.downloadObject(req, transformerCreator).completionFuture().join(); + + verify(s3Client, times(1)).headObject(any(HeadObjectRequest.class)); + // 1byte object, should be single part + verify(s3Client, times(1)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + } + + @Test + public void sizeNotProvided_CallsHeadObject_Multipart() { + DownloadRequest req = DownloadRequest.forBucketAndKey(BUCKET, KEY); + + when(s3Client.headObject(any(HeadObjectRequest.class))) + .thenReturn(CompletableFuture.completedFuture(HeadObjectResponse.builder() + .contentLength(16 * SizeConstant.MiB) + .build())); + + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObject(any())).thenReturn(mockTransformer); + dm.downloadObject(req, transformerCreator).completionFuture().join(); + + verify(s3Client, times(1)).headObject(any(HeadObjectRequest.class)); + verify(s3Client, times(3)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + } + + @Test + public void sizeNotProvided_HeadObjectFails_PropagatedToCompletionFuture() { + DownloadRequest req = DownloadRequest.forBucketAndKey(BUCKET, KEY); + + AwsServiceException expected = S3Exception.builder().build(); + when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(CompletableFutureUtils.failedFuture(expected)); + TransferResponseTransformer mockTransformerCreator = mock(TransferResponseTransformer.class); + + assertThatThrownBy(dm.downloadObject(req, mockTransformerCreator).completionFuture()::join) + .hasCause(expected); + } + + @Test + public void singlePartDownload_FactoryThrows_PropagatedToCompletionFuture() { + DownloadRequest req = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(1L) + .build(); + + UnsupportedOperationException expected = new UnsupportedOperationException("I don't support single part downloads!"); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObject(any())).thenThrow(expected); + assertThatThrownBy(dm.downloadObject(req, transformerCreator).completionFuture()::join) + .hasCause(expected); + } + + @Test + public void multiPartDownload_FactoryThrows_PropagatedToCompletionFuture() { + DownloadRequest req = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.GiB) + .build(); + + UnsupportedOperationException expected = new UnsupportedOperationException("I don't support multipart downloads!"); + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + when(transformerCreator.transformerForObjectPart(any())).thenThrow(expected); + assertThatThrownBy(dm.downloadObject(req, transformerCreator).completionFuture()::join) + .hasCause(expected); + } + + @Test + public void respectsOverrideConfiguration_MultipartDisabled() { + MultipartDownloadConfiguration overrideDownloadConfig = MultipartDownloadConfiguration.defaultConfig() + .toBuilder() + .enableMultipartDownloads(false) + .build(); + + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB) + .overrideConfiguration(TransferOverrideConfiguration.builder() + .multipartDownloadConfiguration(overrideDownloadConfig) + .build()) + .build(); + + + TransferResponseTransformer mockTransformerCreator = mock(TransferResponseTransformer.class); + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + when(mockTransformerCreator.transformerForObject(any(SinglePartDownloadContext.class))).thenReturn(mockTransformer); + + dm.downloadObject(downloadRequest, mockTransformerCreator).completionFuture().join(); + + verify(mockTransformerCreator, times(0)).transformerForObjectPart(any(MultipartDownloadContext.class)); + } + + @Test + public void respectsOverrideConfiguration_MultipartEnabled() { + MultipartDownloadConfiguration globalDownloadConfig = MultipartDownloadConfiguration.defaultConfig() + .toBuilder() + // Disable multipart downloads globally + .enableMultipartDownloads(false) + .build(); + + dm = new DownloadManager(s3Client, globalDownloadConfig); + + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB) + .overrideConfiguration(TransferOverrideConfiguration.builder() + // Use the default config which enables multipart downloads + .multipartDownloadConfiguration(MultipartDownloadConfiguration.defaultConfig()) + .build()) + .build(); + + TransferResponseTransformer mockTransformerCreator = mock(TransferResponseTransformer.class); + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + when(mockTransformerCreator.transformerForObjectPart(any(MultipartDownloadContext.class))).thenReturn(mockTransformer); + + dm.downloadObject(downloadRequest, mockTransformerCreator).completionFuture().join(); + + verify(mockTransformerCreator, times(0)).transformerForObject(any(SinglePartDownloadContext.class)); + } + + @Test + public void respectsOverrideConfiguration_DifferentPartitioning() { + MultipartDownloadConfiguration overrideConfig = MultipartDownloadConfiguration.builder() + .minDownloadPartSize(8 * SizeConstant.MiB) + .build(); + + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(24 * SizeConstant.MiB) + .overrideConfiguration(TransferOverrideConfiguration.builder() + .multipartDownloadConfiguration(overrideConfig) + .build()) + .build(); + + MultipartDownloadContext[] expectedContexts = new MultipartDownloadContext[] { + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(1) + .partOffset(0) + .isLastPart(false) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(0, 8 * SizeConstant.MiB - 1)) + .build())) + .build(), + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(2) + .partOffset(8 * SizeConstant.MiB) + .isLastPart(false) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(8 * SizeConstant.MiB, 16 * SizeConstant.MiB - 1)) + .build())) + .build(), + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(3) + .partOffset(16 * SizeConstant.MiB) + .isLastPart(true) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(16 * SizeConstant.MiB, 24 * SizeConstant.MiB - 1)) + .build())) + .build() + }; + + MultipartDownloadConfiguration globalConfig = MultipartDownloadConfiguration.builder() + .maxDownloadPartCount(MultipartDownloadConfiguration.DEFAULT_MAX_DOWNLOAD_PART_COUNT- 1) + .build(); + + multipartDownloadPartitioningTest(downloadRequest, globalConfig, expectedContexts); + } + + @Test + public void multipartDownload_EvenSplit() { + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB) + .build(); + + MultipartDownloadConfiguration multipartDownloadConfiguration = MultipartDownloadConfiguration.builder() + .enableMultipartDownloads(true) + .multipartDownloadThreshold(16 * SizeConstant.MiB) + .maxDownloadPartCount(2) + .minDownloadPartSize(8 * SizeConstant.MiB) + .build(); + + MultipartDownloadContext[] expectedContexts = new MultipartDownloadContext[] { + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(1) + .partOffset(0) + .isLastPart(false) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(0, 8 * SizeConstant.MiB - 1)) + .build())) + .build(), + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(2) + .partOffset(8 * SizeConstant.MiB) + .isLastPart(true) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(8 * SizeConstant.MiB, 16 * SizeConstant.MiB - 1)) + .build())) + .build() + }; + + multipartDownloadPartitioningTest(downloadRequest, multipartDownloadConfiguration, expectedContexts); + } + + @Test + public void multipartDownload_UnevenSplit() { + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB + 1) + .build(); + + MultipartDownloadConfiguration multipartDownloadConfiguration = MultipartDownloadConfiguration.builder() + .enableMultipartDownloads(true) + .multipartDownloadThreshold(16 * SizeConstant.MiB) + .maxDownloadPartCount(2) + .minDownloadPartSize(8 * SizeConstant.MiB) + .build(); + + MultipartDownloadContext[] expectedContexts = new MultipartDownloadContext[] { + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(1) + .partOffset(0) + .isLastPart(false) + // DownloadManager takes the ceiling of the floating + // division when determining part size so we know the + // first part has the larger size + .size(8 * SizeConstant.MiB + 1) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(0, 8 * SizeConstant.MiB)) + .build())) + .build(), + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(2) + .partOffset(8 * SizeConstant.MiB + 1) + .isLastPart(true) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(8 * SizeConstant.MiB + 1, 16 * SizeConstant.MiB)) + .build())) + .build() + }; + + multipartDownloadPartitioningTest(downloadRequest, multipartDownloadConfiguration, expectedContexts); + } + + @Test + public void multipartDownload_HighMaxPartCount_RetainsMinPartSize() { + DownloadRequest downloadRequest = DownloadRequest.builder() + .downloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest)) + .size(16 * SizeConstant.MiB) + .build(); + + MultipartDownloadConfiguration multipartDownloadConfiguration = MultipartDownloadConfiguration.builder() + .enableMultipartDownloads(true) + .multipartDownloadThreshold(16 * SizeConstant.MiB) + .maxDownloadPartCount(32) + .minDownloadPartSize(8 * SizeConstant.MiB) + .build(); + + MultipartDownloadContext[] expectedContexts = new MultipartDownloadContext[] { + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(1) + .partOffset(0) + .isLastPart(false) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(0, 8 * SizeConstant.MiB - 1)) + .build())) + .build(), + MultipartDownloadContext.builder() + .downloadRequest(downloadRequest) + .partNumber(2) + .partOffset(8 * SizeConstant.MiB) + .isLastPart(true) + .size(8 * SizeConstant.MiB) + .partDownloadSpecification(DownloadObjectSpecification.fromApiRequest(getObjectRequest.toBuilder() + .range(TransferManagerUtilities.rangeHeaderValue(8 * SizeConstant.MiB, 16 * SizeConstant.MiB - 1)) + .build())) + .build() + }; + + multipartDownloadPartitioningTest(downloadRequest, multipartDownloadConfiguration, expectedContexts); + } + + private void multipartDownloadPartitioningTest(DownloadRequest downloadRequest, + MultipartDownloadConfiguration multipartDownloadConfiguration, + MultipartDownloadContext[] expectedContexts) { + + TransferResponseTransformer transformerCreator = mock(TransferResponseTransformer.class); + + List contextAndTransformerPairs = new ArrayList<>(); + List createdContexts = new ArrayList<>(); + when(transformerCreator.transformerForObjectPart(any(MultipartDownloadContext.class))) + .thenAnswer((Answer) invocationOnMock -> { + MultipartDownloadContext ctx = invocationOnMock.getArgumentAt(0, MultipartDownloadContext.class); + createdContexts.add(ctx); + AsyncResponseTransformer transformer = mock(AsyncResponseTransformer.class); + contextAndTransformerPairs.add(new ContextAndTransformerPair(ctx, transformer)); + return transformer; + }); + + dm = new DownloadManager(s3Client, multipartDownloadConfiguration); + + dm.downloadObject(downloadRequest, transformerCreator).completionFuture().join(); + + verify(transformerCreator, times(expectedContexts.length)).transformerForObjectPart(any(MultipartDownloadContext.class)); + assertThat(createdContexts).containsExactlyInAnyOrder(expectedContexts); + for (ContextAndTransformerPair pair : contextAndTransformerPairs) { + verify(s3Client).getObject(any(GetObjectRequest.class), eq(pair.transformer)); + } + } + + private static final class ContextAndTransformerPair { + private final MultipartDownloadContext ctx; + private final AsyncResponseTransformer transformer; + + ContextAndTransformerPair(MultipartDownloadContext ctx, AsyncResponseTransformer transformer) { + this.ctx = ctx; + this.transformer = transformer; + } + } +} diff --git a/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtilsTest.java b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtilsTest.java new file mode 100644 index 000000000000..c598f35c15dc --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/internal/RequestConversionUtilsTest.java @@ -0,0 +1,106 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import java.time.Instant; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.junit.Test; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; + +/** + * Tests for {@link RequestConversionUtils}. + */ +public class RequestConversionUtilsTest { + private static final String ALPHA = "abcdefghijklmnopqrstuvwxyz0123456789"; + private static final Random RNG = new Random(); + + @Test + public void toHeadObjectCopiesAllProperties() { + GetObjectRequest randomGetObject = randomGetObjectRequest(); + HeadObjectRequest convertedToHeadObject = RequestConversionUtils.toHeadObjectRequest(randomGetObject); + + Map> headObjectFields = sdkFieldMap(HeadObjectRequest.builder().sdkFields()); + Map> getObjectFields = sdkFieldMap(GetObjectRequest.builder().sdkFields()); + + for (Map.Entry> headObjectEntry : headObjectFields.entrySet()) { + SdkField headObjField = headObjectEntry.getValue(); + SdkField getObjectField = getObjectFields.get(headObjectEntry.getKey()); + + Object headObjectVal = headObjField.getValueOrDefault(convertedToHeadObject); + Object getObjectVal = getObjectField.getValueOrDefault(randomGetObject); + + assertThat(headObjectVal).isEqualTo(getObjectVal); + } + } + + private GetObjectRequest randomGetObjectRequest() { + GetObjectRequest.Builder builder = GetObjectRequest.builder(); + setFieldsToRandomValues(builder.sdkFields(), builder); + return builder.build(); + } + + private void setFieldsToRandomValues(Collection> fields, Object builder) { + for (SdkField f : fields) { + setFieldToRandomValue(f, builder); + } + } + + private static void setFieldToRandomValue(SdkField sdkField, Object obj) { + Class targetClass = sdkField.marshallingType().getTargetClass(); + if (targetClass.equals(String.class)) { + sdkField.set(obj, randomString(8)); + } else if (targetClass.equals(Integer.class)) { + sdkField.set(obj, randomInteger()); + } else if (targetClass.equals(Instant.class)) { + sdkField.set(obj, randomInstant()); + } else { + throw new IllegalArgumentException("Unknown SdkField type: " + targetClass); + } + } + + private static Map> sdkFieldMap(Collection> sdkFields) { + Map> map = new HashMap<>(sdkFields.size()); + for (SdkField f : sdkFields) { + String locName = f.locationName(); + if (map.put(locName, f) != null) { + throw new IllegalArgumentException("Multiple SdkFields map to same location name"); + } + } + return map; + } + + private static String randomString(int len) { + StringBuilder sb = new StringBuilder(len); + while (len-- > 0) { + sb.append(ALPHA.charAt(RNG.nextInt(ALPHA.length()))); + } + return sb.toString(); + } + + private static Instant randomInstant() { + return Instant.ofEpochMilli(RNG.nextLong()); + } + + private static Integer randomInteger() { + return RNG.nextInt(); + } +} diff --git a/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/utils/S3TestUtils.java b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/utils/S3TestUtils.java new file mode 100644 index 000000000000..f4741a453ec1 --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/utils/S3TestUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.utils; + +import java.util.Iterator; +import java.util.List; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.testutils.Waiter; + +public class S3TestUtils { + + public static void deleteBucketAndAllContents(S3Client s3, String bucketName) { + try { + System.out.println("Deleting S3 bucket: " + bucketName); + ListObjectsResponse response = Waiter.run(() -> s3.listObjects(r -> r.bucket(bucketName))) + .ignoringException(NoSuchBucketException.class) + .orFail(); + List objectListing = response.contents(); + + if (objectListing != null) { + while (true) { + for (Iterator iterator = objectListing.iterator(); iterator.hasNext(); ) { + S3Object objectSummary = (S3Object) iterator.next(); + s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build()); + } + + if (response.isTruncated()) { + objectListing = s3.listObjects(ListObjectsRequest.builder() + .bucket(bucketName) + .marker(response.marker()) + .build()) + .contents(); + } else { + break; + } + } + } + + + ListObjectVersionsResponse versions = s3 + .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build()); + + if (versions.deleteMarkers() != null) { + versions.deleteMarkers().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder() + .versionId(v.versionId()) + .bucket(bucketName) + .key(v.key()) + .build())); + } + + if (versions.versions() != null) { + versions.versions().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder() + .versionId(v.versionId()) + .bucket(bucketName) + .key(v.key()) + .build())); + } + + s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()); + } catch (Exception e) { + System.err.println("Failed to delete bucket: " + bucketName); + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfermanager/src/test/resources/log4j.properties b/services-custom/s3-transfermanager/src/test/resources/log4j.properties new file mode 100644 index 000000000000..b821297c6731 --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/resources/log4j.properties @@ -0,0 +1,33 @@ +# +# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.appender.A1=org.apache.log4j.ConsoleAppender +log4j.appender.A1.layout=org.apache.log4j.PatternLayout + +# Print the date in ISO 8601 format +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n + +# Adjust to see more / less logging +#log4j.logger.com.amazonaws.ec2=DEBUG + +# HttpClient 3 Wire Logging +#log4j.logger.httpclient.wire=DEBUG + +# HttpClient 4 Wire Logging +#log4j.logger.org.apache.http.wire=DEBUG +#log4j.logger.org.apache.http=DEBUG +#log4j.logger.org.apache.http.wire=WARN +#log4j.logger.software.amazon.awssdk=DEBUG diff --git a/utils/src/main/java/software/amazon/awssdk/utils/Validate.java b/utils/src/main/java/software/amazon/awssdk/utils/Validate.java index 960aa9d455ab..866b9270c734 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/Validate.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/Validate.java @@ -601,6 +601,13 @@ public static long isPositive(long num, String fieldName) { return num; } + /** + * Asserts that the given number is non-negative. + * + * @param num Number to validate + * @param fieldName Field name to display in exception message if negative. + * @return Number if not negative. + */ public static int isNotNegative(int num, String fieldName) { if (num < 0) { @@ -610,6 +617,22 @@ public static int isNotNegative(int num, String fieldName) { return num; } + /** + * Asserts that the given number is non-negative. + * + * @param num Number to validate + * @param fieldName Field name to display in exception message if negative. + * @return Number if not negative. + */ + public static long isNotNegative(long num, String fieldName) { + + if (num < 0) { + throw new IllegalArgumentException(String.format("%s must not be negative", fieldName)); + } + + return num; + } + /** * Asserts that the given duration is positive (non-negative and non-zero). *