Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): add object existence validation option to get presigned url #2848

Merged
merged 18 commits into from
Jul 9, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions aws-storage-s3/api/aws-storage-s3.api
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ public final class com/amplifyframework/storage/s3/options/AWSS3StorageGetPresig
public static fun defaultInstance ()Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;
public fun equals (Ljava/lang/Object;)Z
public static fun from (Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
public fun getValidateObjectExistence ()Z
public fun hashCode ()I
public fun toString ()Ljava/lang/String;
public fun useAccelerateEndpoint ()Z
Expand All @@ -187,6 +188,7 @@ public final class com/amplifyframework/storage/s3/options/AWSS3StorageGetPresig
public synthetic fun build ()Lcom/amplifyframework/storage/options/StorageOptions;
public fun build ()Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;
public fun setUseAccelerateEndpoint (Z)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
public fun setValidateObjectExistence (Z)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
}

public final class com/amplifyframework/storage/s3/options/AWSS3StorageListOptions : com/amplifyframework/storage/options/StorageListOptions {
Expand Down Expand Up @@ -283,11 +285,13 @@ public final class com/amplifyframework/storage/s3/request/AWSS3StorageDownloadF

public final class com/amplifyframework/storage/s3/request/AWSS3StorageGetPresignedUrlRequest {
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;IZ)V
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;IZZ)V
public fun getAccessLevel ()Lcom/amplifyframework/storage/StorageAccessLevel;
public fun getExpires ()I
public fun getKey ()Ljava/lang/String;
public fun getTargetIdentityId ()Ljava/lang/String;
public fun useAccelerateEndpoint ()Z
public fun validateObjectExistence ()Z
}

public final class com/amplifyframework/storage/s3/request/AWSS3StorageListRequest {
Expand Down Expand Up @@ -331,6 +335,7 @@ public abstract interface class com/amplifyframework/storage/s3/service/StorageS
public abstract fun resumeTransfer (Lcom/amplifyframework/storage/s3/transfer/TransferObserver;)V
public abstract fun uploadFile (Ljava/lang/String;Ljava/lang/String;Ljava/io/File;Lcom/amplifyframework/storage/ObjectMetadata;Z)Lcom/amplifyframework/storage/s3/transfer/TransferObserver;
public abstract fun uploadInputStream (Ljava/lang/String;Ljava/lang/String;Ljava/io/InputStream;Lcom/amplifyframework/storage/ObjectMetadata;Z)Lcom/amplifyframework/storage/s3/transfer/TransferObserver;
public abstract fun validateObjectExists (Ljava/lang/String;)V
}

public abstract interface class com/amplifyframework/storage/s3/service/StorageService$Factory {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ import android.content.Context
import androidx.test.core.app.ApplicationProvider
import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin
import com.amplifyframework.storage.StorageCategory
import com.amplifyframework.storage.StorageException
import com.amplifyframework.storage.StoragePath
import com.amplifyframework.storage.options.StorageGetUrlOptions
import com.amplifyframework.storage.options.StorageUploadFileOptions
import com.amplifyframework.storage.s3.options.AWSS3StorageGetPresignedUrlOptions
import com.amplifyframework.storage.s3.test.R
import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil
import com.amplifyframework.testutils.random.RandomTempFile
import com.amplifyframework.testutils.sync.SynchronousAuth
import com.amplifyframework.testutils.sync.SynchronousStorage
import java.io.File
import org.junit.Assert.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Assert.assertTrue
import org.junit.BeforeClass
import org.junit.Test
Expand Down Expand Up @@ -79,4 +82,25 @@ class AWSS3StoragePathGetUrlTest {
assertEquals("/public/$SMALL_FILE_NAME", result.url.path)
assertTrue(result.url.query.contains("X-Amz-Expires=30"))
}

@Test
fun testGetUrlWithObjectExistenceValidationEnabled() {
val result = synchronousStorage.getUrl(
SMALL_FILE_PATH,
AWSS3StorageGetPresignedUrlOptions.builder().setValidateObjectExistence(true).expires(30).build()
)

assertEquals("/public/$SMALL_FILE_NAME", result.url.path)
assertTrue(result.url.query.contains("X-Amz-Expires=30"))
}

@Test
fun testGetUrlWithStorageExceptionObjectNotFoundThrown() {
assertThrows(StorageException::class.java) {
synchronousStorage.getUrl(
StoragePath.fromString("public/SOME_UNKNOWN_FILE"),
AWSS3StorageGetPresignedUrlOptions.builder().setValidateObjectExistence(true).expires(30).build()
)
}
lawmicha marked this conversation as resolved.
Show resolved Hide resolved
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,8 @@ public StorageGetUrlOperation<?> getUrl(
@NonNull Consumer<StorageException> onError) {
boolean useAccelerateEndpoint = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).useAccelerateEndpoint();
boolean validateObjectExistence = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).getValidateObjectExistence();
AWSS3StorageGetPresignedUrlRequest request = new AWSS3StorageGetPresignedUrlRequest(
key,
options.getAccessLevel() != null
Expand All @@ -328,7 +330,8 @@ public StorageGetUrlOperation<?> getUrl(
options.getExpires() != 0
? options.getExpires()
: defaultUrlExpiration,
useAccelerateEndpoint
useAccelerateEndpoint,
validateObjectExistence
);

AWSS3StorageGetPresignedUrlOperation operation =
Expand All @@ -355,10 +358,15 @@ public StorageGetUrlOperation<?> getUrl(
) {
boolean useAccelerateEndpoint = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).useAccelerateEndpoint();

boolean validateObjectExistence = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).getValidateObjectExistence();

AWSS3StoragePathGetPresignedUrlRequest request = new AWSS3StoragePathGetPresignedUrlRequest(
path,
options.getExpires() != 0 ? options.getExpires() : defaultUrlExpiration,
useAccelerateEndpoint
useAccelerateEndpoint,
validateObjectExistence
);

AWSS3StoragePathGetPresignedUrlOperation operation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,17 @@
import java.net.URL;
import java.util.concurrent.ExecutorService;

import aws.sdk.kotlin.services.s3.model.NotFound;

/**
* An operation to retrieve pre-signed object URL from AWS S3.
* @deprecated Class should not be public and explicitly cast to. Cast to StorageGetUrlOperation.
* Internal usages are moving to AWSS3StoragePathGetPresignedUrlOperation
* An operation to retrieve pre-signed object URL from AWS S3.
*
* @deprecated Class should not be public and explicitly cast to. Cast to StorageGetUrlOperation.
* Internal usages are moving to AWSS3StoragePathGetPresignedUrlOperation
*/
@Deprecated
public final class AWSS3StorageGetPresignedUrlOperation
extends StorageGetUrlOperation<AWSS3StorageGetPresignedUrlRequest> {
extends StorageGetUrlOperation<AWSS3StorageGetPresignedUrlRequest> {
private final StorageService storageService;
private final ExecutorService executorService;
private final AuthCredentialsProvider authCredentialsProvider;
Expand All @@ -58,13 +61,13 @@ public final class AWSS3StorageGetPresignedUrlOperation
* @param onError Notified upon URL generation error
*/
public AWSS3StorageGetPresignedUrlOperation(
@NonNull StorageService storageService,
@NonNull ExecutorService executorService,
@NonNull AuthCredentialsProvider authCredentialsProvider,
@NonNull AWSS3StorageGetPresignedUrlRequest request,
@NonNull AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration,
@NonNull Consumer<StorageGetUrlResult> onSuccess,
@NonNull Consumer<StorageException> onError
@NonNull StorageService storageService,
@NonNull ExecutorService executorService,
@NonNull AuthCredentialsProvider authCredentialsProvider,
@NonNull AWSS3StorageGetPresignedUrlRequest request,
@NonNull AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration,
@NonNull Consumer<StorageGetUrlResult> onSuccess,
@NonNull Consumer<StorageException> onError
) {
super(request);
this.storageService = storageService;
Expand All @@ -79,27 +82,47 @@ public AWSS3StorageGetPresignedUrlOperation(
@Override
public void start() {
executorService.submit(() -> {
awsS3StoragePluginConfiguration.getAWSS3PluginPrefixResolver(authCredentialsProvider).
awsS3StoragePluginConfiguration.getAWSS3PluginPrefixResolver(authCredentialsProvider).
resolvePrefix(getRequest().getAccessLevel(),
getRequest().getTargetIdentityId(),
prefix -> {
try {
String serviceKey = prefix.concat(getRequest().getKey());

if (getRequest().validateObjectExistence()) {
try {
storageService.validateObjectExists(serviceKey);
phantumcode marked this conversation as resolved.
Show resolved Hide resolved
} catch (NotFound nfe) {
onError.accept(new StorageException(
"Unable to generate URL for non-existent path: $serviceKey",
nfe,
"Please ensure the path is valid or the object has been uploaded."
));
return;
} catch (Exception exception) {
onError.accept(new StorageException(
"Encountered an issue while validating the existence of object",
exception,
"See included exception for more details and suggestions to fix."
));
return;
}
}

URL url = storageService.getPresignedUrl(
serviceKey,
getRequest().getExpires(),
getRequest().useAccelerateEndpoint());
serviceKey,
getRequest().getExpires(),
getRequest().useAccelerateEndpoint());
onSuccess.accept(StorageGetUrlResult.fromUrl(url));
} catch (Exception exception) {
onError.accept(new StorageException(
"Encountered an issue while generating pre-signed URL",
exception,
"See included exception for more details and suggestions to fix."
"Encountered an issue while generating pre-signed URL",
exception,
"See included exception for more details and suggestions to fix."
));
}

},
onError);
}, onError);
}
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/
package com.amplifyframework.storage.s3.operation

import aws.sdk.kotlin.services.s3.model.NotFound
import com.amplifyframework.auth.AuthCredentialsProvider
import com.amplifyframework.core.Consumer
import com.amplifyframework.storage.StorageException
Expand Down Expand Up @@ -48,6 +49,30 @@ internal class AWSS3StoragePathGetPresignedUrlOperation(
return@submit
}

if (request.validateObjectExistence) {
try {
storageService.validateObjectExists(serviceKey)
} catch (nfe: NotFound) {
lawmicha marked this conversation as resolved.
Show resolved Hide resolved
onError.accept(
StorageException(
"Unable to generate URL for non-existent path: $serviceKey",
nfe,
"Please ensure the path is valid or the object has been uploaded."
)
)
return@submit
} catch (exception: Exception) {
onError.accept(
StorageException(
"Encountered an issue while validating the existence of object",
exception,
"See included exception for more details and suggestions to fix."
)
)
return@submit
}
}

try {
val url = storageService.getPresignedUrl(
serviceKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@
*/
public final class AWSS3StorageGetPresignedUrlOptions extends StorageGetUrlOptions {
private final boolean useAccelerationMode;
private final boolean validateObjectExistence;

private AWSS3StorageGetPresignedUrlOptions(final Builder builder) {
super(builder);
this.useAccelerationMode = builder.useAccelerateEndpoint;
this.validateObjectExistence = builder.validateObjectExistence;
}

/**
Expand Down Expand Up @@ -59,6 +61,8 @@ public static Builder from(@NonNull AWSS3StorageGetPresignedUrlOptions options)
return builder()
.accessLevel(options.getAccessLevel())
.targetIdentityId(options.getTargetIdentityId())
.expires(options.getExpires())
.setValidateObjectExistence(options.getValidateObjectExistence())
.expires(options.getExpires());
}

Expand All @@ -80,6 +84,16 @@ public boolean useAccelerateEndpoint() {
return useAccelerationMode;
}

/**
* Gets the flag to determine whether to validate whether an S3 object exists.
* Note: Setting this to `true` will result in a latency cost since confirming the existence
* of the underlying S3 object will likely require a round-trip network call.
* @return boolean flag
*/
public boolean getValidateObjectExistence() {
return validateObjectExistence;
}

@Override
@SuppressWarnings("deprecation")
public boolean equals(Object obj) {
Expand All @@ -91,7 +105,8 @@ public boolean equals(Object obj) {
AWSS3StorageGetPresignedUrlOptions that = (AWSS3StorageGetPresignedUrlOptions) obj;
return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) &&
ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) &&
ObjectsCompat.equals(getExpires(), that.getExpires());
ObjectsCompat.equals(getExpires(), that.getExpires()) &&
ObjectsCompat.equals(getValidateObjectExistence(), that.getValidateObjectExistence());
}
}

Expand All @@ -101,7 +116,8 @@ public int hashCode() {
return ObjectsCompat.hash(
getAccessLevel(),
getTargetIdentityId(),
getExpires()
getExpires(),
getValidateObjectExistence()
);
}

Expand All @@ -113,6 +129,7 @@ public String toString() {
"accessLevel=" + getAccessLevel() +
", targetIdentityId=" + getTargetIdentityId() +
", expires=" + getExpires() +
", validateObjectExistence=" + getValidateObjectExistence() +
'}';
}

Expand All @@ -123,6 +140,7 @@ public String toString() {
*/
public static final class Builder extends StorageGetUrlOptions.Builder<Builder> {
private boolean useAccelerateEndpoint;
private boolean validateObjectExistence;

/**
* Configure to use acceleration mode on new StorageGetPresignedUrlOptions instances.
Expand All @@ -134,6 +152,16 @@ public Builder setUseAccelerateEndpoint(boolean useAccelerateEndpoint) {
return this;
}

/**
* Configure to validate object existence flag on new StorageGetPresignedUrlOptions instances.
* @param validateObjectExistence boolean flag to represent flag to validate object existence.
* @return Current Builder instance for fluent chaining
*/
public Builder setValidateObjectExistence(boolean validateObjectExistence) {
this.validateObjectExistence = validateObjectExistence;
return this;
}

@Override
@NonNull
public AWSS3StorageGetPresignedUrlOptions build() {
Expand Down
Loading