Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): add object existence validation option to get presigned url #2848

Merged
merged 18 commits into from
Jul 9, 2024
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions aws-storage-s3/api/aws-storage-s3.api
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ public final class com/amplifyframework/storage/s3/options/AWSS3StorageGetPresig
public static fun defaultInstance ()Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;
public fun equals (Ljava/lang/Object;)Z
public static fun from (Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
public fun getValidateObjectExistence ()Z
public fun hashCode ()I
public fun toString ()Ljava/lang/String;
public fun useAccelerateEndpoint ()Z
Expand All @@ -187,6 +188,7 @@ public final class com/amplifyframework/storage/s3/options/AWSS3StorageGetPresig
public synthetic fun build ()Lcom/amplifyframework/storage/options/StorageOptions;
public fun build ()Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions;
public fun setUseAccelerateEndpoint (Z)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
public fun setValidateObjectExistence (Z)Lcom/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions$Builder;
}

public final class com/amplifyframework/storage/s3/options/AWSS3StorageListOptions : com/amplifyframework/storage/options/StorageListOptions {
Expand Down Expand Up @@ -283,11 +285,13 @@ public final class com/amplifyframework/storage/s3/request/AWSS3StorageDownloadF

public final class com/amplifyframework/storage/s3/request/AWSS3StorageGetPresignedUrlRequest {
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;IZ)V
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;IZZ)V
public fun getAccessLevel ()Lcom/amplifyframework/storage/StorageAccessLevel;
public fun getExpires ()I
public fun getKey ()Ljava/lang/String;
public fun getTargetIdentityId ()Ljava/lang/String;
public fun useAccelerateEndpoint ()Z
public fun validateObjectExistence ()Z
}

public final class com/amplifyframework/storage/s3/request/AWSS3StorageListRequest {
Expand Down Expand Up @@ -331,6 +335,7 @@ public abstract interface class com/amplifyframework/storage/s3/service/StorageS
public abstract fun resumeTransfer (Lcom/amplifyframework/storage/s3/transfer/TransferObserver;)V
public abstract fun uploadFile (Ljava/lang/String;Ljava/lang/String;Ljava/io/File;Lcom/amplifyframework/storage/ObjectMetadata;Z)Lcom/amplifyframework/storage/s3/transfer/TransferObserver;
public abstract fun uploadInputStream (Ljava/lang/String;Ljava/lang/String;Ljava/io/InputStream;Lcom/amplifyframework/storage/ObjectMetadata;Z)Lcom/amplifyframework/storage/s3/transfer/TransferObserver;
public abstract fun validateObjectExists (Ljava/lang/String;)V
}

public abstract interface class com/amplifyframework/storage/s3/service/StorageService$Factory {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,28 @@
package com.amplifyframework.storage.s3

import android.content.Context
import android.util.Log
import androidx.test.core.app.ApplicationProvider
import aws.sdk.kotlin.services.s3.model.NotFound
import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin
import com.amplifyframework.storage.StorageCategory
import com.amplifyframework.storage.StorageException
import com.amplifyframework.storage.StoragePath
import com.amplifyframework.storage.options.StorageGetUrlOptions
import com.amplifyframework.storage.options.StorageUploadFileOptions
import com.amplifyframework.storage.s3.options.AWSS3StorageGetPresignedUrlOptions
import com.amplifyframework.storage.s3.test.R
import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil
import com.amplifyframework.testutils.random.RandomTempFile
import com.amplifyframework.testutils.sync.SynchronousAuth
import com.amplifyframework.testutils.sync.SynchronousStorage
import java.io.File
import org.junit.Assert.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Assert.assertTrue
import org.junit.BeforeClass
import org.junit.Test
import java.net.URL

/**
* Instrumentation test for operational work on download.
Expand Down Expand Up @@ -79,4 +85,42 @@ class AWSS3StoragePathGetUrlTest {
assertEquals("/public/$SMALL_FILE_NAME", result.url.path)
assertTrue(result.url.query.contains("X-Amz-Expires=30"))
}

@Test
fun testGetUrlWithObjectExistenceValidationEnabled() {
val result = synchronousStorage.getUrl(
SMALL_FILE_PATH,
AWSS3StorageGetPresignedUrlOptions.builder().setValidateObjectExistence(true).expires(30).build()
)

assertEquals("/public/$SMALL_FILE_NAME", result.url.path)
assertTrue(result.url.query.contains("X-Amz-Expires=30"))
}

@Test
fun testGetUrlWithStorageExceptionObjectNotFoundThrown() {
val exception = assertThrows(StorageException::class.java) {
synchronousStorage.getUrl(
StoragePath.fromString("public/SOME_UNKNOWN_FILE"),
AWSS3StorageGetPresignedUrlOptions.builder().setValidateObjectExistence(true).expires(30).build()
)
}

assertTrue(exception.cause is NotFound)
}
lawmicha marked this conversation as resolved.
Show resolved Hide resolved

@Test
fun testGetUrlWithObjectExistenceValidationDisabledForNonExistentObject() {
val result = synchronousStorage.getUrl(
StoragePath.fromString("public/SOME_UNKNOWN_FILE"),
AWSS3StorageGetPresignedUrlOptions.builder().setValidateObjectExistence(false).expires(30).build()
)

assertEquals("/public/SOME_UNKNOWN_FILE", result.url.path)
assertTrue(result.url.query.contains("X-Amz-Expires=30"))

assertThrows(java.io.FileNotFoundException::class.java) {
result.url.readBytes()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,8 @@ public StorageGetUrlOperation<?> getUrl(
@NonNull Consumer<StorageException> onError) {
boolean useAccelerateEndpoint = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).useAccelerateEndpoint();
boolean validateObjectExistence = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).getValidateObjectExistence();
AWSS3StorageGetPresignedUrlRequest request = new AWSS3StorageGetPresignedUrlRequest(
key,
options.getAccessLevel() != null
Expand All @@ -328,7 +330,8 @@ public StorageGetUrlOperation<?> getUrl(
options.getExpires() != 0
? options.getExpires()
: defaultUrlExpiration,
useAccelerateEndpoint
useAccelerateEndpoint,
validateObjectExistence
);

AWSS3StorageGetPresignedUrlOperation operation =
Expand All @@ -355,10 +358,15 @@ public StorageGetUrlOperation<?> getUrl(
) {
boolean useAccelerateEndpoint = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).useAccelerateEndpoint();

boolean validateObjectExistence = options instanceof AWSS3StorageGetPresignedUrlOptions &&
((AWSS3StorageGetPresignedUrlOptions) options).getValidateObjectExistence();

AWSS3StoragePathGetPresignedUrlRequest request = new AWSS3StoragePathGetPresignedUrlRequest(
path,
options.getExpires() != 0 ? options.getExpires() : defaultUrlExpiration,
useAccelerateEndpoint
useAccelerateEndpoint,
validateObjectExistence
);

AWSS3StoragePathGetPresignedUrlOperation operation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,14 @@
import java.util.concurrent.ExecutorService;

/**
* An operation to retrieve pre-signed object URL from AWS S3.
* @deprecated Class should not be public and explicitly cast to. Cast to StorageGetUrlOperation.
* Internal usages are moving to AWSS3StoragePathGetPresignedUrlOperation
* An operation to retrieve pre-signed object URL from AWS S3.
*
* @deprecated Class should not be public and explicitly cast to. Cast to StorageGetUrlOperation.
* Internal usages are moving to AWSS3StoragePathGetPresignedUrlOperation
*/
@Deprecated
public final class AWSS3StorageGetPresignedUrlOperation
extends StorageGetUrlOperation<AWSS3StorageGetPresignedUrlRequest> {
extends StorageGetUrlOperation<AWSS3StorageGetPresignedUrlRequest> {
private final StorageService storageService;
private final ExecutorService executorService;
private final AuthCredentialsProvider authCredentialsProvider;
Expand All @@ -58,13 +59,13 @@ public final class AWSS3StorageGetPresignedUrlOperation
* @param onError Notified upon URL generation error
*/
public AWSS3StorageGetPresignedUrlOperation(
@NonNull StorageService storageService,
@NonNull ExecutorService executorService,
@NonNull AuthCredentialsProvider authCredentialsProvider,
@NonNull AWSS3StorageGetPresignedUrlRequest request,
@NonNull AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration,
@NonNull Consumer<StorageGetUrlResult> onSuccess,
@NonNull Consumer<StorageException> onError
@NonNull StorageService storageService,
@NonNull ExecutorService executorService,
@NonNull AuthCredentialsProvider authCredentialsProvider,
@NonNull AWSS3StorageGetPresignedUrlRequest request,
@NonNull AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration,
@NonNull Consumer<StorageGetUrlResult> onSuccess,
@NonNull Consumer<StorageException> onError
) {
super(request);
this.storageService = storageService;
Expand All @@ -79,16 +80,26 @@ public AWSS3StorageGetPresignedUrlOperation(
@Override
public void start() {
executorService.submit(() -> {
awsS3StoragePluginConfiguration.getAWSS3PluginPrefixResolver(authCredentialsProvider).
awsS3StoragePluginConfiguration.getAWSS3PluginPrefixResolver(authCredentialsProvider).
resolvePrefix(getRequest().getAccessLevel(),
getRequest().getTargetIdentityId(),
prefix -> {
try {
String serviceKey = prefix.concat(getRequest().getKey());

if (getRequest().validateObjectExistence()) {
try {
storageService.validateObjectExists(serviceKey);
phantumcode marked this conversation as resolved.
Show resolved Hide resolved
} catch (StorageException exception) {
onError.accept(exception);
return;
}
}

URL url = storageService.getPresignedUrl(
serviceKey,
getRequest().getExpires(),
getRequest().useAccelerateEndpoint());
serviceKey,
getRequest().getExpires(),
getRequest().useAccelerateEndpoint());
onSuccess.accept(StorageGetUrlResult.fromUrl(url));
} catch (Exception exception) {
onError.accept(new StorageException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,24 @@ internal class AWSS3StoragePathGetPresignedUrlOperation(
return@submit
}

if (request.validateObjectExistence) {
try {
storageService.validateObjectExists(serviceKey)
} catch (se: StorageException) {
phantumcode marked this conversation as resolved.
Show resolved Hide resolved
onError.accept(se)
return@submit
} catch (exception: Exception) {
onError.accept(
StorageException(
"Encountered an issue while validating the existence of object",
exception,
"See included exception for more details and suggestions to fix."
)
)
return@submit
}
}

try {
val url = storageService.getPresignedUrl(
serviceKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@
*/
public final class AWSS3StorageGetPresignedUrlOptions extends StorageGetUrlOptions {
private final boolean useAccelerationMode;
private final boolean validateObjectExistence;

private AWSS3StorageGetPresignedUrlOptions(final Builder builder) {
super(builder);
this.useAccelerationMode = builder.useAccelerateEndpoint;
this.validateObjectExistence = builder.validateObjectExistence;
}

/**
Expand Down Expand Up @@ -59,6 +61,8 @@ public static Builder from(@NonNull AWSS3StorageGetPresignedUrlOptions options)
return builder()
.accessLevel(options.getAccessLevel())
.targetIdentityId(options.getTargetIdentityId())
.expires(options.getExpires())
.setValidateObjectExistence(options.getValidateObjectExistence())
.expires(options.getExpires());
}

Expand All @@ -80,6 +84,16 @@ public boolean useAccelerateEndpoint() {
return useAccelerationMode;
}

/**
* Gets the flag to determine whether to validate whether an S3 object exists.
* Note: Setting this to `true` will result in a latency cost since confirming the existence
* of the underlying S3 object will likely require a round-trip network call.
* @return boolean flag
*/
public boolean getValidateObjectExistence() {
return validateObjectExistence;
}

@Override
@SuppressWarnings("deprecation")
public boolean equals(Object obj) {
Expand All @@ -91,7 +105,8 @@ public boolean equals(Object obj) {
AWSS3StorageGetPresignedUrlOptions that = (AWSS3StorageGetPresignedUrlOptions) obj;
return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) &&
ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) &&
ObjectsCompat.equals(getExpires(), that.getExpires());
ObjectsCompat.equals(getExpires(), that.getExpires()) &&
ObjectsCompat.equals(getValidateObjectExistence(), that.getValidateObjectExistence());
}
}

Expand All @@ -101,7 +116,8 @@ public int hashCode() {
return ObjectsCompat.hash(
getAccessLevel(),
getTargetIdentityId(),
getExpires()
getExpires(),
getValidateObjectExistence()
);
}

Expand All @@ -113,6 +129,7 @@ public String toString() {
"accessLevel=" + getAccessLevel() +
", targetIdentityId=" + getTargetIdentityId() +
", expires=" + getExpires() +
", validateObjectExistence=" + getValidateObjectExistence() +
'}';
}

Expand All @@ -123,6 +140,7 @@ public String toString() {
*/
public static final class Builder extends StorageGetUrlOptions.Builder<Builder> {
private boolean useAccelerateEndpoint;
private boolean validateObjectExistence;

/**
* Configure to use acceleration mode on new StorageGetPresignedUrlOptions instances.
Expand All @@ -134,6 +152,16 @@ public Builder setUseAccelerateEndpoint(boolean useAccelerateEndpoint) {
return this;
}

/**
* Configure to validate object existence flag on new StorageGetPresignedUrlOptions instances.
* @param validateObjectExistence boolean flag to represent flag to validate object existence.
* @return Current Builder instance for fluent chaining
*/
public Builder setValidateObjectExistence(boolean validateObjectExistence) {
this.validateObjectExistence = validateObjectExistence;
return this;
}

@Override
@NonNull
public AWSS3StorageGetPresignedUrlOptions build() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public final class AWSS3StorageGetPresignedUrlRequest {
private final String targetIdentityId;
private final int expires;
private final boolean useAccelerateEndpoint;
private final boolean validateObjectExistence;

/**
* Constructs a new AWSS3StorageGetUrlRequest.
Expand All @@ -58,6 +59,37 @@ public AWSS3StorageGetPresignedUrlRequest(
this.targetIdentityId = targetIdentityId;
this.expires = expires;
this.useAccelerateEndpoint = useAccelerateEndpoint;
this.validateObjectExistence = false;
}

/**
* Constructs a new AWSS3StorageGetUrlRequest.
* Although this has public access, it is intended for internal use and should not be used directly by host
* applications. The behavior of this may change without warning.
*
* @param key key for item to obtain URL for
* @param accessLevel Storage access level
* @param targetIdentityId If set, this should override the current user's identity ID.
* If null, the operation will fetch the current identity ID.
* @param expires The number of seconds before the URL expires
* @param useAccelerateEndpoint Flag to enable acceleration mode
* @param validateObjectExistence Flag to validate if object exists in storage
*/
@SuppressWarnings("deprecation")
public AWSS3StorageGetPresignedUrlRequest(
@NonNull String key,
@NonNull StorageAccessLevel accessLevel,
@Nullable String targetIdentityId,
int expires,
boolean useAccelerateEndpoint,
boolean validateObjectExistence
) {
this.key = key;
this.accessLevel = accessLevel;
this.targetIdentityId = targetIdentityId;
this.expires = expires;
this.useAccelerateEndpoint = useAccelerateEndpoint;
this.validateObjectExistence = validateObjectExistence;
}

/**
Expand Down Expand Up @@ -104,5 +136,14 @@ public int getExpires() {
public boolean useAccelerateEndpoint() {
return useAccelerateEndpoint;
}

/**
* Gets the flag to determine whether to validate for object existence.
*
* @return boolean flag
*/
public boolean validateObjectExistence() {
return validateObjectExistence;
}
}

Loading
Loading