S3AsyncClient Support #19
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package software.amazon.payloadoffloading; | ||
|
|
||
| import java.util.concurrent.CompletableFuture; | ||
| import software.amazon.awssdk.core.exception.SdkClientException; | ||
| import software.amazon.awssdk.services.s3.model.S3Exception; | ||
|
|
||
| /** | ||
| * An AWS storage service that supports saving high payload sizes. | ||
| */ | ||
| public interface PayloadStoreAsync { | ||
|
|
||
| /** | ||
| * Stores payload in a store that has higher payload size limit than that is supported by original payload store. | ||
| * <p> | ||
| * This call is asynchronous, and so documented return values and exceptions are propagated through | ||
| * the returned {@link CompletableFuture}. | ||
| * | ||
| * @param payload | ||
| * @return future value of a pointer that must be used to retrieve the original payload later. | ||
| * @throws SdkClientException If any internal errors are encountered on the client side while | ||
| * attempting to make the request or handle the response. For example | ||
| * if a network connection is not available. | ||
| * @throws S3Exception If an error response is returned by actual PayloadStore indicating | ||
| * either a problem with the data in the request, or a server side issue. | ||
| */ | ||
| CompletableFuture<String> storeOriginalPayload(String payload); | ||
|
|
||
| /** | ||
| * Stores payload in a store that has higher payload size limit than that is supported by original payload store. | ||
| * <p> | ||
| * This call is asynchronous, and so documented return values and exceptions are propagated through | ||
| * the returned {@link CompletableFuture}. | ||
| * | ||
| * @param payload | ||
| * @param s3Key | ||
| * @return future value of a pointer that must be used to retrieve the original payload later. | ||
| * @throws SdkClientException If any internal errors are encountered on the client side while | ||
| * attempting to make the request or handle the response. For example | ||
| * if a network connection is not available. | ||
| * @throws S3Exception If an error response is returned by actual PayloadStore indicating | ||
| * either a problem with the data in the request, or a server side issue. | ||
| */ | ||
| CompletableFuture<String> storeOriginalPayload(String payload, String s3Key); | ||
|
|
||
| /** | ||
| * Retrieves the original payload using the given payloadPointer. The pointer must | ||
| * have been obtained using {@link #storeOriginalPayload(String)} | ||
| * <p> | ||
| * This call is asynchronous, and so documented return values and exceptions are propagated through | ||
| * the returned {@link CompletableFuture}. | ||
| * | ||
| * @param payloadPointer | ||
| * @return future value of the original payload | ||
| * @throws SdkClientException If any internal errors are encountered on the client side while | ||
| * attempting to make the request or handle the response. For example | ||
| * if payloadPointer is invalid or a network connection is not available. | ||
| * @throws S3Exception If an error response is returned by actual PayloadStore indicating | ||
| * a server side issue. | ||
| */ | ||
| CompletableFuture<String> getOriginalPayload(String payloadPointer); | ||
|
|
||
| /** | ||
| * Deletes the original payload using the given payloadPointer. The pointer must | ||
| * have been obtained using {@link #storeOriginalPayload(String)} | ||
| * <p> | ||
| * This call is asynchronous, and so documented return values and exceptions are propagated through | ||
| * the returned {@link CompletableFuture}. | ||
| * | ||
| * @param payloadPointer | ||
| * @return future value that completes when the delete operation finishes | ||
| * @throws SdkClientException If any internal errors are encountered on the client side while | ||
| * attempting to make the request or handle the response to/from PayloadStore. | ||
| * For example, if payloadPointer is invalid or a network connection is not available. | ||
| * @throws S3Exception If an error response is returned by actual PayloadStore indicating | ||
| * a server side issue. | ||
| */ | ||
| CompletableFuture<Void> deleteOriginalPayload(String payloadPointer); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| package software.amazon.payloadoffloading; | ||
|
|
||
| import java.io.UncheckedIOException; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import java.util.concurrent.CompletionException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import software.amazon.awssdk.core.ResponseBytes; | ||
| import software.amazon.awssdk.core.async.AsyncRequestBody; | ||
| import software.amazon.awssdk.core.async.AsyncResponseTransformer; | ||
| import software.amazon.awssdk.core.exception.SdkClientException; | ||
| import software.amazon.awssdk.core.exception.SdkException; | ||
| import software.amazon.awssdk.services.s3.S3AsyncClient; | ||
| import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; | ||
| import software.amazon.awssdk.services.s3.model.GetObjectRequest; | ||
| import software.amazon.awssdk.services.s3.model.ObjectCannedACL; | ||
| import software.amazon.awssdk.services.s3.model.PutObjectRequest; | ||
|
|
||
| /** | ||
| * Dao layer to access S3. | ||
| */ | ||
| public class S3AsyncDao { | ||
| private static final Logger LOG = LoggerFactory.getLogger(S3AsyncDao.class); | ||
| private final S3AsyncClient s3Client; | ||
| private final ServerSideEncryptionStrategy serverSideEncryptionStrategy; | ||
| private final ObjectCannedACL objectCannedACL; | ||
|
|
||
| public S3AsyncDao(S3AsyncClient s3Client) { | ||
| this(s3Client, null, null); | ||
| } | ||
|
|
||
| public S3AsyncDao( | ||
| S3AsyncClient s3Client, | ||
| ServerSideEncryptionStrategy serverSideEncryptionStrategy, | ||
| ObjectCannedACL objectCannedACL) { | ||
| this.s3Client = s3Client; | ||
| this.serverSideEncryptionStrategy = serverSideEncryptionStrategy; | ||
| this.objectCannedACL = objectCannedACL; | ||
| } | ||
|
|
||
| public CompletableFuture<String> getTextFromS3(String s3BucketName, String s3Key) { | ||
| GetObjectRequest getObjectRequest = GetObjectRequest.builder() | ||
| .bucket(s3BucketName) | ||
| .key(s3Key) | ||
| .build(); | ||
|
|
||
| return s3Client.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()) | ||
| .thenApply(ResponseBytes::asUtf8String) | ||
| .handle((v, tIn) -> { | ||
| if (tIn != null) { | ||
| Throwable t = Util.unwrapFutureException(tIn); | ||
| if (t instanceof SdkException) { | ||
| String errorMessage = "Failed to get the S3 object which contains the payload."; | ||
| LOG.error(errorMessage, t); | ||
| throw SdkException.create(errorMessage, t); | ||
| } | ||
| if (t instanceof UncheckedIOException) { | ||
| String errorMessage = "Failure when handling the message which was read from S3 object."; | ||
| LOG.error(errorMessage, t); | ||
| throw SdkClientException.create(errorMessage, t); | ||
| } | ||
| throw new CompletionException(t); | ||
| } | ||
| return v; | ||
| }); | ||
| } | ||
|
|
||
| public CompletableFuture<Void> storeTextInS3(String s3BucketName, String s3Key, String payloadContentStr) { | ||
| PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() | ||
| .bucket(s3BucketName) | ||
| .key(s3Key); | ||
|
|
||
| if (objectCannedACL != null) { | ||
| putObjectRequestBuilder.acl(objectCannedACL); | ||
| } | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/dev/kms-using-sdks.html | ||
| if (serverSideEncryptionStrategy != null) { | ||
| serverSideEncryptionStrategy.decorate(putObjectRequestBuilder); | ||
| } | ||
|
|
||
| return s3Client.putObject(putObjectRequestBuilder.build(), AsyncRequestBody.fromString(payloadContentStr)) | ||
| .handle((v, tIn) -> { | ||
| if (tIn != null) { | ||
| Throwable t = Util.unwrapFutureException(tIn); | ||
| if (t instanceof SdkException) { | ||
| String errorMessage = "Failed to store the message content in an S3 object."; | ||
| LOG.error(errorMessage, t); | ||
| throw SdkException.create(errorMessage, t); | ||
| } | ||
| throw new CompletionException(t); | ||
| } | ||
| return null; | ||
| }); | ||
| } | ||
|
|
||
| public CompletableFuture<Void> deletePayloadFromS3(String s3BucketName, String s3Key) { | ||
| DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() | ||
| .bucket(s3BucketName) | ||
| .key(s3Key) | ||
| .build(); | ||
| return s3Client.deleteObject(deleteObjectRequest) | ||
| .handle((v, tIn) -> { | ||
| if (tIn != null) { | ||
| Throwable t = Util.unwrapFutureException(tIn); | ||
| if (t instanceof SdkException) { | ||
| String errorMessage = "Failed to delete the S3 object which contains the payload"; | ||
| LOG.error(errorMessage, t); | ||
| throw SdkException.create(errorMessage, t); | ||
| } | ||
| throw new CompletionException(t); | ||
| } | ||
|
|
||
| LOG.info("S3 object deleted, Bucket name: " + s3BucketName + ", Object key: " + s3Key + "."); | ||
| return null; | ||
| }); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package software.amazon.payloadoffloading; | ||
|
|
||
| import java.util.UUID; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import java.util.concurrent.CompletionException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import software.amazon.payloadoffloading.PayloadS3Pointer; | ||
|
|
||
| /** | ||
| * S3 based implementation for PayloadStoreAsync. | ||
| */ | ||
| public class S3BackedPayloadStoreAsync implements PayloadStoreAsync { | ||
| private static final Logger LOG = LoggerFactory.getLogger(S3BackedPayloadStoreAsync.class); | ||
|
|
||
| private final String s3BucketName; | ||
| private final S3AsyncDao s3Dao; | ||
|
|
||
| public S3BackedPayloadStoreAsync(S3AsyncDao s3Dao, String s3BucketName) { | ||
| this.s3BucketName = s3BucketName; | ||
| this.s3Dao = s3Dao; | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<String> storeOriginalPayload(String payload) { | ||
| String s3Key = UUID.randomUUID().toString(); | ||
| return storeOriginalPayload(payload, s3Key); | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<String> storeOriginalPayload(String payload, String s3Key) { | ||
| return s3Dao.storeTextInS3(s3BucketName, s3Key, payload) | ||
| .thenApply(v -> { | ||
| LOG.info("S3 object created, Bucket name: " + s3BucketName + ", Object key: " + s3Key + "."); | ||
|
|
||
| // Convert S3 pointer (bucket name, key, etc) to JSON string | ||
| PayloadS3Pointer s3Pointer = new PayloadS3Pointer(s3BucketName, s3Key); | ||
|
|
||
| return s3Pointer.toJson(); | ||
| }); | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<String> getOriginalPayload(String payloadPointer) { | ||
| try { | ||
| PayloadS3Pointer s3Pointer = PayloadS3Pointer.fromJson(payloadPointer); | ||
|
|
||
| String s3BucketName = s3Pointer.getS3BucketName(); | ||
| String s3Key = s3Pointer.getS3Key(); | ||
|
|
||
| return s3Dao.getTextFromS3(s3BucketName, s3Key) | ||
| .thenApply(originalPayload -> { | ||
| LOG.info("S3 object read, Bucket name: " + s3BucketName + ", Object key: " + s3Key + "."); | ||
| return originalPayload; | ||
| }); | ||
| } catch (Exception e) { | ||
| CompletableFuture<String> futureEx = new CompletableFuture<>(); | ||
| futureEx.completeExceptionally((e instanceof RuntimeException) ? e : new CompletionException(e)); | ||
| return futureEx; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<Void> deleteOriginalPayload(String payloadPointer) { | ||
| try { | ||
| PayloadS3Pointer s3Pointer = PayloadS3Pointer.fromJson(payloadPointer); | ||
|
|
||
| String s3BucketName = s3Pointer.getS3BucketName(); | ||
| String s3Key = s3Pointer.getS3Key(); | ||
| return s3Dao.deletePayloadFromS3(s3BucketName, s3Key); | ||
| } catch (Exception e) { | ||
| CompletableFuture<Void> futureEx = new CompletableFuture<>(); | ||
| futureEx.completeExceptionally((e instanceof RuntimeException) ? e : new CompletionException(e)); | ||
| return futureEx; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does it make sense to have a different method name here?