diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 889bb277876..c9208704407 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -63,30 +63,7 @@ jobs: with: command: ./gradlew test jacocoTestReport - Aws-Integration-Tests: - runs-on: ubuntu-latest - - env: - S3_ACCESS_KEY_ID: root - S3_SECRET_ACCESS_KEY: password - - services: - minio: - image: bitnami/minio:latest - ports: - - 9000:9000 - env: - MINIO_ROOT_USER: root - MINIO_ROOT_PASSWORD: password - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-build - - - name: AWS Tests - uses: ./.github/actions/run-tests - with: - command: ./gradlew -p extensions test -DincludeTags="AwsS3IntegrationTest" Daps-Integration-Tests: runs-on: ubuntu-latest diff --git a/extensions/common/aws/aws-s3-core/README.md b/extensions/common/aws/aws-s3-core/README.md deleted file mode 100644 index ad7d1c0ec85..00000000000 --- a/extensions/common/aws/aws-s3-core/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# S3 Core - -This extension registers an AWS credentials provider that can be used by all the S3 related extensions. - -The credentials lookup works in this order: -- vault (through the keys specified in the [Configuration](#configuration)) -- if there are no vault keys, the AWS `DefaultCredentialProvider` will look at: -> 1. Java System Properties - aws.accessKeyId and aws.secretAccessKey -> 2. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY -> 3. Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI -> 4. Credentials delivered through the Amazon EC2 container service if AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable, -> 5. Instance profile credentials delivered through the Amazon EC2 metadata service - -## Configuration - -| Parameter name | Description | Mandatory | Default value | -|-----------------------------------------|------------------------------------------------------------------|-----------|---------------| -| `edc.aws.access.key` | The key of the secret where the AWS Access Key Id is stored. | false | null | -| `edc.aws.secret.access.key` | The key of the secret where the AWS Secret Access Key is stored. | false | 5 | -| `edc.aws.endpoint.override` | If valued, the AWS clients will point to the specified endpoint. | false | null | -| `edc.aws.client.async.thread-pool-size` | The size of the thread pool used for the async clients. | false | 50 | diff --git a/extensions/common/aws/aws-s3-core/build.gradle.kts b/extensions/common/aws/aws-s3-core/build.gradle.kts deleted file mode 100644 index 289a4a38d5c..00000000000 --- a/extensions/common/aws/aws-s3-core/build.gradle.kts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -plugins { - `java-library` -} - -dependencies { - api(project(":spi:control-plane:transfer-spi")) - - api(libs.failsafe.core) - - api(libs.aws.iam) - api(libs.aws.s3) - api(libs.aws.sts) -} - - diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProvider.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProvider.java deleted file mode 100644 index abec4f65d31..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProvider.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - Initial implementation - * - */ - -package org.eclipse.edc.aws.s3; - -import org.eclipse.edc.connector.transfer.spi.types.SecretToken; -import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; -import software.amazon.awssdk.services.iam.IamAsyncClient; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.sts.StsAsyncClient; - -/** - * Provide various AWS client shapes - *
- * Caching by region: - * - S3Client - * - S3AsyncClient - * - StsAsyncClient - *
- * Single instance for the aws-global region: - * - IamAsyncClient - *
- * Instantiated on-fly given a SecretToken: - * - S3Client - */ -@ExtensionPoint -public interface AwsClientProvider { - - /** - * Returns the client for the specified region with the secret token credentials - */ - S3Client s3Client(String region, SecretToken secretToken); - - /** - * Returns the s3 client for the specified region - */ - S3Client s3Client(String region); - - /** - * Returns the s3 async client for the specified region - */ - S3AsyncClient s3AsyncClient(String region); - - /** - * Returns the iam async client for the global region - */ - IamAsyncClient iamAsyncClient(); - - /** - * Returns the sts async client for the specified region - */ - StsAsyncClient stsAsyncClient(String region); - - /** - * Releases resources used by the provider. - */ - void shutdown(); -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderConfiguration.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderConfiguration.java deleted file mode 100644 index 7b1ff853285..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderConfiguration.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.aws.s3; - -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; - -import java.net.URI; -import java.util.Objects; - -public class AwsClientProviderConfiguration { - - static final int DEFAULT_AWS_ASYNC_CLIENT_THREAD_POOL_SIZE = 50; - - private AwsCredentialsProvider credentialsProvider; - private URI endpointOverride; - private int threadPoolSize = DEFAULT_AWS_ASYNC_CLIENT_THREAD_POOL_SIZE; - - private AwsClientProviderConfiguration() { - - } - - public AwsCredentialsProvider getCredentialsProvider() { - return credentialsProvider; - } - - public URI getEndpointOverride() { - return endpointOverride; - } - - public int getThreadPoolSize() { - return threadPoolSize; - } - - public static class Builder { - - private final AwsClientProviderConfiguration configuration = new AwsClientProviderConfiguration(); - - private Builder() { - - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) { - configuration.credentialsProvider = credentialsProvider; - return this; - } - - public Builder endpointOverride(URI endpointOverride) { - configuration.endpointOverride = endpointOverride; - return this; - } - - public Builder threadPoolSize(int threadPoolSize) { - configuration.threadPoolSize = threadPoolSize; - return this; - } - - public AwsClientProviderConfiguration build() { - Objects.requireNonNull(configuration.credentialsProvider, "AWS Credentials Provider is mandatory"); - - return configuration; - } - } -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderImpl.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderImpl.java deleted file mode 100644 index 4a4fbaea24f..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsClientProviderImpl.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - Initial implementation - * - */ - -package org.eclipse.edc.aws.s3; - -import org.eclipse.edc.connector.transfer.spi.types.SecretToken; -import org.eclipse.edc.spi.EdcException; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.client.builder.SdkClientBuilder; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.iam.IamAsyncClient; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3BaseClientBuilder; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.sts.StsAsyncClient; -import software.amazon.awssdk.utils.SdkAutoCloseable; -import software.amazon.awssdk.utils.ThreadFactoryBuilder; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR; - -public class AwsClientProviderImpl implements AwsClientProvider { - - private final AwsCredentialsProvider credentialsProvider; - private final AwsClientProviderConfiguration configuration; - private final Executor executor; - private final Map s3Clients = new ConcurrentHashMap<>(); - private final Map s3AsyncClients = new ConcurrentHashMap<>(); - private final Map stsAsyncClients = new ConcurrentHashMap<>(); - private final IamAsyncClient iamAsyncClient; - - public AwsClientProviderImpl(AwsClientProviderConfiguration configuration) { - this.credentialsProvider = configuration.getCredentialsProvider(); - this.configuration = configuration; - this.executor = Executors.newFixedThreadPool(configuration.getThreadPoolSize(), new ThreadFactoryBuilder().threadNamePrefix("aws-client").build()); - this.iamAsyncClient = createIamAsyncClient(); - } - - @Override - public S3Client s3Client(String region, SecretToken token) { - if (token instanceof AwsTemporarySecretToken) { - var temporary = (AwsTemporarySecretToken) token; - var credentials = AwsSessionCredentials.create(temporary.getAccessKeyId(), temporary.getSecretAccessKey(), temporary.getSessionToken()); - return createS3Client(credentials, region); - } else if (token instanceof AwsSecretToken) { - var secretToken = (AwsSecretToken) token; - var credentials = AwsBasicCredentials.create(secretToken.getAccessKeyId(), secretToken.getSecretAccessKey()); - return createS3Client(credentials, region); - } else { - throw new EdcException(String.format("SecretToken %s is not supported", token.getClass())); - } - } - - @Override - public S3Client s3Client(String region) { - return s3Clients.computeIfAbsent(region, this::createS3Client); - } - - @Override - public S3AsyncClient s3AsyncClient(String region) { - return s3AsyncClients.computeIfAbsent(region, this::createS3AsyncClient); - } - - @Override - public IamAsyncClient iamAsyncClient() { - return iamAsyncClient; - } - - @Override - public StsAsyncClient stsAsyncClient(String region) { - return stsAsyncClients.computeIfAbsent(region, this::createStsClient); - } - - @Override - public void shutdown() { - iamAsyncClient.close(); - s3Clients.values().forEach(SdkAutoCloseable::close); - s3AsyncClients.values().forEach(SdkAutoCloseable::close); - stsAsyncClients.values().forEach(SdkAutoCloseable::close); - } - - private S3Client createS3Client(AwsCredentials credentials, String region) { - var credentialsProvider = StaticCredentialsProvider.create(credentials); - var builder = S3Client.builder() - .credentialsProvider(credentialsProvider) - .region(Region.of(region)); - - handleBaseEndpointOverride(builder); - - return builder.build(); - } - - private S3Client createS3Client(String region) { - S3ClientBuilder builder = S3Client.builder() - .credentialsProvider(credentialsProvider) - .region(Region.of(region)); - - handleBaseEndpointOverride(builder); - - return builder.build(); - } - - private S3AsyncClient createS3AsyncClient(String region) { - var builder = S3AsyncClient.builder() - .asyncConfiguration(b -> b.advancedOption(FUTURE_COMPLETION_EXECUTOR, executor)) - .credentialsProvider(credentialsProvider) - .region(Region.of(region)); - - handleBaseEndpointOverride(builder); - - return builder.build(); - } - - private StsAsyncClient createStsClient(String region) { - var builder = StsAsyncClient.builder() - .asyncConfiguration(b -> b.advancedOption(FUTURE_COMPLETION_EXECUTOR, executor)) - .credentialsProvider(credentialsProvider) - .region(Region.of(region)); - - handleEndpointOverride(builder); - - return builder.build(); - } - - private IamAsyncClient createIamAsyncClient() { - var builder = IamAsyncClient.builder() - .asyncConfiguration(b -> b.advancedOption(FUTURE_COMPLETION_EXECUTOR, executor)) - .credentialsProvider(credentialsProvider) - .region(Region.AWS_GLOBAL); - - handleEndpointOverride(builder); - - return builder.build(); - } - - private void handleBaseEndpointOverride(S3BaseClientBuilder builder) { - var endpointOverride = configuration.getEndpointOverride(); - if (endpointOverride != null) { - builder.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) - .endpointOverride(endpointOverride); - } - } - - private void handleEndpointOverride(SdkClientBuilder builder) { - var endpointOverride = configuration.getEndpointOverride(); - if (endpointOverride != null) { - builder.endpointOverride(endpointOverride); - } - } -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsSecretToken.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsSecretToken.java deleted file mode 100644 index 760a07f1df7..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsSecretToken.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.aws.s3; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.eclipse.edc.connector.transfer.spi.types.SecretToken; - -import java.util.Objects; - -public class AwsSecretToken implements SecretToken { - private final String accessKeyId; - private final String secretAccessKey; - - public AwsSecretToken(@JsonProperty("accessKeyId") String accessKeyId, @JsonProperty("secretAccessKey") String secretAccessKey) { - this.accessKeyId = accessKeyId; - this.secretAccessKey = secretAccessKey; - } - - @Override - public long getExpiration() { - return 0; - } - - public String getAccessKeyId() { - return accessKeyId; - } - - public String getSecretAccessKey() { - return secretAccessKey; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AwsSecretToken that = (AwsSecretToken) o; - return Objects.equals(accessKeyId, that.accessKeyId) && Objects.equals(secretAccessKey, that.secretAccessKey); - } - - @Override - public int hashCode() { - return Objects.hash(accessKeyId, secretAccessKey); - } -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsTemporarySecretToken.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsTemporarySecretToken.java deleted file mode 100644 index 0be4c457d6b..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/AwsTemporarySecretToken.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.aws.s3; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.eclipse.edc.connector.transfer.spi.types.SecretToken; - -public class AwsTemporarySecretToken implements SecretToken { - private final String sessionToken; - private final long expiration; - private final String accessKeyId; - private final String secretAccessKey; - - public AwsTemporarySecretToken(@JsonProperty("accessKeyId") String accessKeyId, @JsonProperty("secretAccessKey") String secretAccessKey, @JsonProperty("sessionToken") String sessionToken, @JsonProperty("expiration") long expiration) { - this.sessionToken = sessionToken; - this.expiration = expiration; - this.accessKeyId = accessKeyId; - this.secretAccessKey = secretAccessKey; - } - - public String getSessionToken() { - return sessionToken; - } - - @Override - public long getExpiration() { - return expiration; - } - - public String getAccessKeyId() { - return accessKeyId; - } - - public String getSecretAccessKey() { - return secretAccessKey; - } -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java deleted file mode 100644 index 2c47b60fca8..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.aws.s3; - -public interface S3BucketSchema { - String TYPE = "AmazonS3"; - String REGION = "region"; - String BUCKET_NAME = "bucketName"; - String ACCESS_KEY_ID = "accessKeyId"; - String SECRET_ACCESS_KEY = "secretAccessKey"; -} diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3CoreExtension.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3CoreExtension.java deleted file mode 100644 index 9fed0f5b26d..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3CoreExtension.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.aws.s3; - -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provider; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.jetbrains.annotations.NotNull; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; - -import java.net.URI; -import java.util.Optional; - -import static java.lang.String.format; -import static org.eclipse.edc.aws.s3.AwsClientProviderConfiguration.DEFAULT_AWS_ASYNC_CLIENT_THREAD_POOL_SIZE; - -@Extension(value = S3CoreExtension.NAME) -public class S3CoreExtension implements ServiceExtension { - - public static final String NAME = "S3"; - @Setting(value = "The key of the secret where the AWS Access Key Id is stored") - private static final String AWS_ACCESS_KEY = "edc.aws.access.key"; - @Setting(value = "The key of the secret where the AWS Secret Access Key is stored") - private static final String AWS_SECRET_KEY = "edc.aws.secret.access.key"; - @Setting(value = "If valued, the AWS clients will point to the specified endpoint") - private static final String AWS_ENDPOINT_OVERRIDE = "edc.aws.endpoint.override"; - @Setting(value = "The size of the thread pool used for the async clients") - private static final String AWS_ASYNC_CLIENT_THREAD_POOL_SIZE = "edc.aws.client.async.thread-pool-size"; - @Inject - private Vault vault; - - @Inject - private Monitor monitor; - - @Override - public String name() { - return NAME; - } - - @Provider - public AwsClientProvider awsClientProvider(ServiceExtensionContext context) { - var endpointOverride = Optional.of(AWS_ENDPOINT_OVERRIDE) - .map(key -> context.getSetting(key, null)) - .map(URI::create) - .orElse(null); - - var threadPoolSize = context.getSetting(AWS_ASYNC_CLIENT_THREAD_POOL_SIZE, DEFAULT_AWS_ASYNC_CLIENT_THREAD_POOL_SIZE); - - var configuration = AwsClientProviderConfiguration.Builder.newInstance() - .credentialsProvider(createCredentialsProvider(context)) - .endpointOverride(endpointOverride) - .threadPoolSize(threadPoolSize) - .build(); - - return new AwsClientProviderImpl(configuration); - } - - @NotNull - private AwsCredentialsProvider createCredentialsProvider(ServiceExtensionContext context) { - var accessKey = vault.resolveSecret(context.getSetting(AWS_ACCESS_KEY, AWS_ACCESS_KEY)); - var secretKey = vault.resolveSecret(context.getSetting(AWS_SECRET_KEY, AWS_SECRET_KEY)); - - if (accessKey == null || secretKey == null) { - monitor.info(format("S3: %s and %s were not found in the vault, DefaultCredentialsProvider will be used", AWS_ACCESS_KEY, AWS_SECRET_KEY)); - return DefaultCredentialsProvider.create(); - } - - return () -> AwsBasicCredentials.create(accessKey, secretKey); - } - -} diff --git a/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index e63d7277986..00000000000 --- a/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Bayerische Motoren Werke Aktiengesellschaft (BMW AG) -# -# - -org.eclipse.edc.aws.s3.S3CoreExtension - diff --git a/extensions/common/aws/aws-s3-test/README.md b/extensions/common/aws/aws-s3-test/README.md deleted file mode 100644 index b23c5adce51..00000000000 --- a/extensions/common/aws/aws-s3-test/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# AWS Test - -## Local testing using MinIO - -To run AWS integration tests you will need a MinIO instance running: -``` -docker run -d -p 9000:9000 -e MINIO_ROOT_USER=root -e MINIO_ROOT_PASSWORD=password bitnami/minio:latest -``` - -Then set the two environment variables: -``` -S3_ACCESS_KEY_ID=root -S3_SECRET_ACCESS_KEY=password -``` - -## Test using your AWS credential - -`IT_AWS_ENDPOINT` can be used to override [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) URI -for running integration tests against AWS S3 by environment variable: - -``` -$ IT_AWS_ENDPOINT=https://s3.us-east-1.amazonaws.com/ \ - IT_AWS_REGION=us-east-1 \ - IT_AWS_PROFILE=myprofie \ - ./gradlew clean test -DincludeTags="AwsS3IntegrationTest" --tests '*S3StatusCheckerIntegrationTest' -``` - -`IT_AWS_REGION` must be set to your region code in order to avoid -["A conflicting conditional operation is currently in progress against this resource." error](http://stackoverflow.com/questions/13898057/aws-error-message-a-conflicting-conditional-operation-is-currently-in-progress). - -`IT_AWS_PROFILE` can be used to specify -[named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) -referring your own credential. -You can also use access key and secret access key by `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. diff --git a/extensions/common/aws/aws-s3-test/build.gradle.kts b/extensions/common/aws/aws-s3-test/build.gradle.kts deleted file mode 100644 index 73a89ab01c6..00000000000 --- a/extensions/common/aws/aws-s3-test/build.gradle.kts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -plugins { - `java-library` - `java-test-fixtures` - `maven-publish` -} - -dependencies { - testFixturesApi(project(":core:common:junit")) - testFixturesImplementation(project(":extensions:common:aws:aws-s3-core")) - - testFixturesImplementation(libs.awaitility) - testFixturesImplementation(libs.assertj) - testFixturesImplementation(libs.junit.jupiter.api) - testFixturesRuntimeOnly(libs.junit.jupiter.engine) - testFixturesApi(libs.aws.s3) -} - - diff --git a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java deleted file mode 100644 index 01981e85cd3..00000000000 --- a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2020 - 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * NTT DATA - added endpoint override - * - */ - -package org.eclipse.edc.aws.s3.testfixtures; - -import okhttp3.Request; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsClientProviderConfiguration; -import org.eclipse.edc.aws.s3.AwsClientProviderImpl; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadBucketRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; - -import java.io.File; -import java.io.IOException; -import java.net.ConnectException; -import java.net.URI; -import java.time.Duration; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.junit.testfixtures.TestUtils.testHttpClient; -import static org.eclipse.edc.util.configuration.ConfigurationFunctions.propOrEnv; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Base class for tests that need an S3 bucket created and deleted on every test run. - */ -public abstract class AbstractS3Test { - - protected static final String REGION = propOrEnv("it.aws.region", Region.US_EAST_1.id()); - // Adding REGION to bucket prevents errors of - // "A conflicting conditional operation is currently in progress against this resource." - // when bucket is rapidly added/deleted and consistency propagation causes this error. - // (Should not be necessary if REGION remains static, but added to prevent future frustration.) - // [see http://stackoverflow.com/questions/13898057/aws-error-message-a-conflicting-conditional-operation-is-currently-in-progress] - protected static final String MINIO_ENDPOINT = "http://localhost:9000"; - protected static final URI S3_ENDPOINT = URI.create(propOrEnv("it.aws.endpoint", MINIO_ENDPOINT)); - protected final UUID processId = UUID.randomUUID(); - protected String bucketName = createBucketName(); - protected S3AsyncClient s3AsyncClient; - private final AwsClientProviderConfiguration configuration = AwsClientProviderConfiguration.Builder.newInstance() - .credentialsProvider(this::getCredentials) - .endpointOverride(S3_ENDPOINT) - .build(); - protected AwsClientProvider clientProvider = new AwsClientProviderImpl(configuration); - - @BeforeAll - static void prepareAll() { - await().atLeast(Duration.ofSeconds(2)) - .atMost(Duration.ofSeconds(15)) - .with() - .pollInterval(Duration.ofSeconds(2)) - .ignoreException(IOException.class) // thrown by pingMinio - .ignoreException(ConnectException.class) - .until(AbstractS3Test::isBackendAvailable); - } - - private static boolean isBackendAvailable() throws IOException { - if (isMinio()) { - return isMinioAvailable(); - } else { - return true; - } - } - - private static boolean isMinio() { - return MINIO_ENDPOINT.equals(S3_ENDPOINT.toString()); - } - - /** - * pings MinIO's health endpoint - * - * @return true if HTTP status [200..300[ - */ - private static boolean isMinioAvailable() throws IOException { - var httpClient = testHttpClient(); - var healthRq = new Request.Builder().url(S3_ENDPOINT + "/minio/health/live").get().build(); - try (var response = httpClient.execute(healthRq)) { - return response.isSuccessful(); - } - } - - @BeforeEach - public void setupClient() { - s3AsyncClient = clientProvider.s3AsyncClient(REGION); - - createBucket(bucketName); - } - - @AfterEach - void cleanup() { - deleteBucket(bucketName); - } - - @NotNull - protected String createBucketName() { - return "test-bucket-" + processId + "-" + REGION; - } - - protected void createBucket(String bucketName) { - if (bucketExists(bucketName)) { - fail("Bucket " + bucketName + " exists. Choose a different bucket name to continue test"); - } - - s3AsyncClient.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()).join(); - - if (!bucketExists(bucketName)) { - fail("Setup incomplete, tests will fail"); - } - } - - protected void deleteBucket(String bucketName) { - try { - if (s3AsyncClient == null) { - return; - } - - // Empty the bucket before deleting it, otherwise the AWS S3 API fails - deleteBucketObjects(bucketName); - - s3AsyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()).join(); - } catch (Exception e) { - System.err.println("Unable to delete bucket " + bucketName + e); - } - - if (bucketExists(bucketName)) { - fail("Incomplete teardown, subsequent tests might fail"); - } - } - - protected CompletableFuture putTestFile(String key, File file, String bucketName) { - return s3AsyncClient.putObject(PutObjectRequest.builder().bucket(bucketName).key(key).build(), file.toPath()); - } - - protected void putStringOnBucket(String bucketName, String key, String content) { - var request = PutObjectRequest.builder().bucket(bucketName).key(key).build(); - var response = s3AsyncClient.putObject(request, AsyncRequestBody.fromString(content)); - assertThat(response).succeedsWithin(10, TimeUnit.SECONDS); - } - - protected @NotNull AwsCredentials getCredentials() { - var profile = propOrEnv("it.aws.profile", null); - if (profile != null) { - return ProfileCredentialsProvider.create(profile).resolveCredentials(); - } - - var accessKeyId = propOrEnv("S3_ACCESS_KEY_ID", null); - Objects.requireNonNull(accessKeyId, "S3_ACCESS_KEY_ID cannot be null!"); - var secretKey = propOrEnv("S3_SECRET_ACCESS_KEY", null); - Objects.requireNonNull(secretKey, "S3_SECRET_ACCESS_KEY cannot be null"); - - return AwsBasicCredentials.create(accessKeyId, secretKey); - } - - private void deleteBucketObjects(String bucketName) { - var objectListing = s3AsyncClient.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()).join(); - - CompletableFuture.allOf(objectListing.contents().stream() - .map(object -> s3AsyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(object.key()).build())) - .toArray(CompletableFuture[]::new)).join(); - - for (var objectSummary : objectListing.contents()) { - s3AsyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build()).join(); - } - - if (objectListing.isTruncated()) { - deleteBucketObjects(bucketName); - } - } - - private boolean bucketExists(String bucketName) { - try { - HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucketName).build(); - return s3AsyncClient.headBucket(request).join() - .sdkHttpResponse() - .isSuccessful(); - } catch (CompletionException e) { - if (e.getCause() instanceof NoSuchBucketException) { - return false; - } else { - throw e; - } - } - } - -} diff --git a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java deleted file mode 100644 index 9a56e2532c0..00000000000 --- a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.aws.s3.testfixtures.annotations; - -import org.eclipse.edc.junit.annotations.IntegrationTest; -import org.junit.jupiter.api.Tag; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation for AWS S3 integration testing. It applies specific Junit Tag. - */ -@Target({ ElementType.TYPE }) -@Retention(RetentionPolicy.RUNTIME) -@IntegrationTest -@Tag("AwsS3IntegrationTest") -public @interface AwsS3IntegrationTest { -} diff --git a/extensions/common/vault/vault-aws/README.md b/extensions/common/vault/vault-aws/README.md deleted file mode 100644 index 1d33084caec..00000000000 --- a/extensions/common/vault/vault-aws/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# AWS Secrets Manager Vault - -The vault-aws extension is an implementation of the Vault interface, which stores secrets in AWS Secrets Manager. -Arbitrary key names are possible through the key sanitation feature. - -## Limitations -- 50 TpS (Transactions per Second) for storing secrets and deleting secrets. -- 10,000 TpS (Transactions per Second) for retrieving secrets. - -## Configuration - -### Credentials resolution -The vault-aws extension uses the AWS SDK Secrets Manager client. Credentials for accessing Secrets Manager are resolved using the default credential provider chain. -The default AWS credentials provider chain that looks for credentials in this order: - -1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY -2. Java System Properties - aws.accessKeyId and aws.secretKey -3. Web Identity Token credentials from the environment or container -4. Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI -5. Credentials delivered through the Amazon EC2 container service if AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable is set and security manager has permission to access the variable, -6. Instance profile credentials delivered through the Amazon EC2 metadata service - -### Client retry behaviour -The AWS SDK has retry behaviour built in. It can be controlled globally through the environment variables AWS_MAX_ATTEMPTS, AWS_RETRY_MODE. -Please see [the SDK documentation](https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html) for details. - -### Other configuration options - -| Parameter name | Description | Mandatory | Default value | -|:----------------------------------------------------|:-----------------------------------|:----------|:---------------------------------------| -| `edc.vault.aws.region` | AWS region for AWS Secrets Manager | true | | - -## Decisions -- Use default credentials provider to be as flexible as possible in credentials resolution. -- Secrets will not be overwritten if they exist to prevent potential leakage of credentials to third parties. -- Keys strings are sanitized to comply with key requirements of AWS Secrets Manager. Sanitizing replaces all illegal characters with '-' and appends the hash code of the original key to minimize the risk of key collision after the transformation, because the replacement operation is a many-to-one function. A warning will be logged if the key contains illegal characters. - -## Change log diff --git a/extensions/common/vault/vault-aws/build.gradle.kts b/extensions/common/vault/vault-aws/build.gradle.kts deleted file mode 100644 index 475a4d6d312..00000000000 --- a/extensions/common/vault/vault-aws/build.gradle.kts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2020, 2021, 2022 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - initial API and implementation - * - */ - -plugins { - `java-library` -} - -dependencies { - api(project(":spi:common:core-spi")) - implementation(libs.aws.secretsmanager) - implementation(project(":core:common:util")) - testImplementation(libs.mockito.inline) -} diff --git a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVault.java b/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVault.java deleted file mode 100644 index 3f60556b2ba..00000000000 --- a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVault.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2022 - 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - initial implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.security.Vault; -import org.jetbrains.annotations.Nullable; -import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; -import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; -import software.amazon.awssdk.services.secretsmanager.model.DeleteSecretRequest; -import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; -import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; - -/** - * Vault adapter for AWS Secrets Manager. - */ -public class AwsSecretsManagerVault implements Vault { - - private final SecretsManagerClient smClient; - private final Monitor monitor; - private final AwsSecretsManagerVaultSanitationStrategy sanitizer; - - public AwsSecretsManagerVault(SecretsManagerClient smClient, Monitor monitor, AwsSecretsManagerVaultSanitationStrategy sanitizer) { - this.smClient = smClient; - this.monitor = monitor; - this.sanitizer = sanitizer; - } - - /** - * Retrieves a secret. Any string can be used as a key. Keys that do not comply with AWS Secrets Managers requirements - * will be transformed. - * - * @param key the key of the secret - * @return the secret value or null if secret could not be found - */ - @Override - public @Nullable String resolveSecret(String key) { - var sanitizedKey = sanitizer.sanitizeKey(key); - var request = GetSecretValueRequest.builder().secretId(sanitizedKey).build(); - try { - monitor.debug(String.format("Resolving secret '%s' from AWS Secrets manager", sanitizedKey)); - return smClient.getSecretValue(request).secretString(); - } catch (ResourceNotFoundException e) { - monitor.debug(String.format("Couldn't resolve secret with key %s", sanitizedKey), e); - } catch (RuntimeException serviceException) { - monitor.severe(serviceException.getMessage(), serviceException); - } - return null; - } - - /** - * Creates a new secret. Does not overwrite secrets. - * - * @param key the secret key - * @param value the serialized secret value - * @return success or failure - */ - @Override - public Result storeSecret(String key, String value) { - var sanitizedKey = sanitizer.sanitizeKey(key); - var request = CreateSecretRequest.builder().name(sanitizedKey) - .secretString(value).build(); - try { - monitor.debug(String.format("Storing secret '%s' to AWS Secrets manager", sanitizedKey)); - smClient.createSecret(request); - return Result.success(); - } catch (RuntimeException serviceException) { - monitor.severe(serviceException.getMessage(), serviceException); - return Result.failure(serviceException.getMessage()); - } - } - - /** - * Deletes a secret without the possibility of recovery. - * - * @param key the secret's key - * @return success or failure - */ - @Override - public Result deleteSecret(String key) { - var sanitizedKey = sanitizer.sanitizeKey(key); - var request = DeleteSecretRequest.builder().secretId(sanitizedKey) - .forceDeleteWithoutRecovery(true).build(); - try { - monitor.debug(String.format("Deleting secret '%s' from AWS Secrets manager", sanitizedKey)); - smClient.deleteSecret(request); - return Result.success(); - } catch (RuntimeException serviceException) { - monitor.severe(serviceException.getMessage(), serviceException); - return Result.failure(serviceException.getMessage()); - } - } - - -} diff --git a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultDefaultSanitationStrategy.java b/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultDefaultSanitationStrategy.java deleted file mode 100644 index 04e0c571aa8..00000000000 --- a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultDefaultSanitationStrategy.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2023 - 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - initial implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.spi.monitor.Monitor; - -public class AwsSecretsManagerVaultDefaultSanitationStrategy implements AwsSecretsManagerVaultSanitationStrategy { - private final Monitor monitor; - - public AwsSecretsManagerVaultDefaultSanitationStrategy(Monitor monitor) { - this.monitor = monitor; - } - - /** - * Many-to-one mapping from all strings into set of strings that only contains valid AWS Secrets Manager key names. - * The implementation replaces all illegal characters with '_' and attaches the hash code of the original string to - * minimize the likelihood of key collisions. - * - * @param originalKey any key - * @return Valid AWS Secrets Manager key - */ - @Override - public String sanitizeKey(String originalKey) { - var key = originalKey; - if (originalKey.length() > 500) { - key = originalKey.substring(0, 500); - } - var sb = new StringBuilder(); - boolean replacedIllegalCharacters = false; - for (int i = 0; i < key.length(); i++) { - var c = key.charAt(i); - if (!Character.isLetterOrDigit(c) && c != '/' && c != '_' && c != '+' && c != '.' && c != '@' && c != '-') { - replacedIllegalCharacters = true; - sb.append('-'); - } else { - sb.append(c); - } - } - var newKey = sb.append('_').append(originalKey.hashCode()).toString(); - if (replacedIllegalCharacters) { - monitor.warning(String.format("AWS Secret Manager vault reduced length or replaced illegal characters " + - "in original key name: %s. New name is %s", originalKey, newKey)); - } - return newKey; - } -} \ No newline at end of file diff --git a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtension.java b/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtension.java deleted file mode 100644 index 5939547f12c..00000000000 --- a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtension.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2023 - 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - initial implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Provides; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.security.CertificateResolver; -import org.eclipse.edc.spi.security.PrivateKeyResolver; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.security.VaultCertificateResolver; -import org.eclipse.edc.spi.security.VaultPrivateKeyResolver; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; - -import static org.eclipse.edc.util.configuration.ConfigurationFunctions.propOrEnv; -import static org.eclipse.edc.util.string.StringUtils.isNullOrEmpty; - -/** - * This extension registers an implementation of the Vault interface for AWS Secrets Manager. - * It also registers a VaultPrivateKeyResolver and VaultCertificateResolver, which store and retrieve certificates - * using the AWS Secretes Manager Vault implementation. - * The extension requires the "edc.vault.aws.region" parameter to be set to the AWS region in which secrets should be stored. - */ -@Provides({ Vault.class, PrivateKeyResolver.class, CertificateResolver.class }) -@Extension(value = org.eclipse.edc.vault.aws.AwsSecretsManagerVaultExtension.NAME) -public class AwsSecretsManagerVaultExtension implements ServiceExtension { - public static final String NAME = "AWS Secrets Manager Vault"; - - @Setting - private static final String VAULT_AWS_REGION = "edc.vault.aws.region"; - - @Override - public String name() { - return NAME; - } - - @Override - public void initialize(ServiceExtensionContext context) { - var vaultRegion = getMandatorySetting(context, VAULT_AWS_REGION); - - var smClient = buildSmClient(vaultRegion); - var vault = new AwsSecretsManagerVault(smClient, context.getMonitor(), - new AwsSecretsManagerVaultDefaultSanitationStrategy(context.getMonitor())); - - context.registerService(Vault.class, vault); - context.registerService(PrivateKeyResolver.class, new VaultPrivateKeyResolver(vault)); - context.registerService(CertificateResolver.class, new VaultCertificateResolver(vault)); - } - - private SecretsManagerClient buildSmClient(String vaultRegion) { - var builder = SecretsManagerClient.builder() - .region(Region.of(vaultRegion)); - return builder.build(); - } - - private String getMandatorySetting(ServiceExtensionContext context, String setting) { - var value = context.getSetting(setting, null); - if (isNullOrEmpty(value)) { - value = propOrEnv(setting, null); - if (isNullOrEmpty(value)) { - throw new EdcException(String.format("'%s' must be supplied but was null", setting)); - } - } - return value; - } - -} diff --git a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultSanitationStrategy.java b/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultSanitationStrategy.java deleted file mode 100644 index b85a58caf0b..00000000000 --- a/extensions/common/vault/vault-aws/src/main/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultSanitationStrategy.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023 - 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - initial implementation - * - */ - -package org.eclipse.edc.vault.aws; - -/** - * Interface for key sanitation strategies. - */ -public interface AwsSecretsManagerVaultSanitationStrategy { - - /** - * Maps any string to a valid AWS Secrets Manager key. - * - * @param originalKey any key - * @return Valid AWS Secrets Manager key - */ - String sanitizeKey(String originalKey); -} diff --git a/extensions/common/vault/vault-aws/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/common/vault/vault-aws/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 66b5dfa4dcf..00000000000 --- a/extensions/common/vault/vault-aws/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Amazon Web Services -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Amazon Web Services - initial API and implementation -# -# - -org.eclipse.edc.vault.aws.AwsSecretsManagerVaultExtension diff --git a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerDefaultKeySanitationStrategyTest.java b/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerDefaultKeySanitationStrategyTest.java deleted file mode 100644 index 0130506b5b7..00000000000 --- a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerDefaultKeySanitationStrategyTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - Initial Implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.spi.monitor.Monitor; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; - - -class AwsSecretsManagerDefaultKeySanitationStrategyTest { - - private final Monitor monitor = mock(Monitor.class); - - private final AwsSecretsManagerVaultSanitationStrategy sanitizer = - new AwsSecretsManagerVaultDefaultSanitationStrategy(monitor); - - @Test - void resolveSecret_sanitizeKeyNameReplacesInvalidCharacters() { - var key2 = "invalid#key"; - - var sanitized = sanitizer.sanitizeKey(key2); - - assertThat(sanitized).isEqualTo("invalid-key" + "_" + key2.hashCode()); - } - - @Test - void resolveSecret_sanitizeKeyNameDoesNotReplaceValidCharacters() { - var sanitizer = new AwsSecretsManagerVaultDefaultSanitationStrategy(monitor); - for (var validCharacter : List.of('_', '+', '-', '@', '/', '.')) { - var validKey = "valid" + validCharacter + "key"; - - assertThat(sanitizer.sanitizeKey(validKey)).isEqualTo(validKey + '_' + validKey.hashCode()); - } - } - - @Test - void resolveSecret_sanitizeKeyNameLimitsKeySize() { - var key = "-".repeat(10000); - - var sanitized = sanitizer.sanitizeKey(key); - - assertThat(sanitized) - .isEqualTo("-".repeat(500) + "_" + key.hashCode()); - assertThat(sanitized.length()).isEqualTo(512); - } - - @Test - void resolveSecret_sanitizeKeyNameLimitsKeySize2() { - var key = "-".repeat(500); - - var sanitized = sanitizer.sanitizeKey(key); - - assertThat(sanitized) - .isEqualTo("-".repeat(500) + "_" + key.hashCode()); - assertThat(sanitized.length()).isLessThanOrEqualTo(512); - } - -} \ No newline at end of file diff --git a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtensionTest.java b/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtensionTest.java deleted file mode 100644 index 437d2c0e4f8..00000000000 --- a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultExtensionTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - Initial implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class AwsSecretsManagerVaultExtensionTest { - - private final Monitor monitor = mock(Monitor.class); - private final AwsSecretsManagerVaultExtension extension = new AwsSecretsManagerVaultExtension(); - - @Test - void configOptionRegionNotProvided_shouldThrowException() { - ServiceExtensionContext invalidContext = mock(ServiceExtensionContext.class); - when(invalidContext.getMonitor()).thenReturn(monitor); - - Assertions.assertThrows(EdcException.class, () -> extension.initialize(invalidContext)); - } - - @Test - void configOptionRegionProvided_shouldNotThrowException() { - ServiceExtensionContext validContext = mock(ServiceExtensionContext.class); - when(validContext.getSetting("edc.vault.aws.region", null)).thenReturn("eu-west-1"); - when(validContext.getMonitor()).thenReturn(monitor); - - extension.initialize(validContext); - } - -} diff --git a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultTest.java b/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultTest.java deleted file mode 100644 index 6905c212100..00000000000 --- a/extensions/common/vault/vault-aws/src/test/java/org/eclipse/edc/vault/aws/AwsSecretsManagerVaultTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2023 Amazon Web Services - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amazon Web Services - Initial Implementation - * - */ - -package org.eclipse.edc.vault.aws; - -import org.eclipse.edc.spi.monitor.Monitor; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.mockito.ArgumentMatchers; -import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; -import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; -import software.amazon.awssdk.services.secretsmanager.model.DeleteSecretRequest; -import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; -import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@TestInstance(Lifecycle.PER_CLASS) -class AwsSecretsManagerVaultTest { - - private static final String KEY = "valid-key"; - private static final String SANITIZED_KEY = "valid-key-sanitized"; - private final Monitor monitor = mock(Monitor.class); - private final SecretsManagerClient secretClient = mock(SecretsManagerClient.class); - private final AwsSecretsManagerVaultSanitationStrategy sanitizer = mock(AwsSecretsManagerVaultSanitationStrategy.class); - private final AwsSecretsManagerVault vault = new AwsSecretsManagerVault(secretClient, monitor, - sanitizer); - - @BeforeAll - void setup() { - when(sanitizer.sanitizeKey(KEY)).thenReturn(SANITIZED_KEY); - } - - @BeforeEach - void resetMocks() { - reset(monitor, secretClient); - } - - @Test - void storeSecret_shouldSanitizeKey() { - var value = "value"; - - vault.storeSecret(KEY, value); - - verify(secretClient).createSecret(CreateSecretRequest.builder().name(SANITIZED_KEY) - .secretString(value).build()); - } - - @Test - void storeSecret_shouldNotOverwriteSecrets() { - var value = "value"; - - vault.storeSecret(KEY, value); - - verify(secretClient).createSecret(CreateSecretRequest.builder().name(SANITIZED_KEY) - .secretString(value).build()); - } - - @Test - void resolveSecret_shouldSanitizeKey() { - vault.resolveSecret(KEY); - - verify(secretClient).getSecretValue(GetSecretValueRequest.builder().secretId(SANITIZED_KEY) - .build()); - } - - @Test - void deleteSecret_shouldSanitizeKey() { - vault.deleteSecret(KEY); - - verify(secretClient).deleteSecret(DeleteSecretRequest.builder().secretId(SANITIZED_KEY) - .forceDeleteWithoutRecovery(true) - .build()); - } - - @Test - void resolveSecret_shouldNotLogSevereIfSecretNotFound() { - when(secretClient.getSecretValue(GetSecretValueRequest.builder().secretId(SANITIZED_KEY) - .build())) - .thenThrow(ResourceNotFoundException.builder().build()); - - var result = vault.resolveSecret(KEY); - - assertThat(result).isNull(); - verify(monitor, times(1)) - .debug(anyString()); - - verify(monitor, times(1)) - .debug(anyString(), any()); - } - - @Test - void resolveSecret_shouldReturnNullAndLogErrorOnGenericException() { - when(secretClient.getSecretValue(GetSecretValueRequest.builder().secretId(SANITIZED_KEY) - .build())) - .thenThrow(new RuntimeException("test")); - - var result = vault.resolveSecret(KEY); - - assertThat(result).isNull(); - verify(monitor).debug(anyString()); - verify(monitor).severe(anyString(), ArgumentMatchers.isA(RuntimeException.class)); - } -} diff --git a/extensions/control-plane/provision/provision-aws-s3/build.gradle.kts b/extensions/control-plane/provision/provision-aws-s3/build.gradle.kts deleted file mode 100644 index 73f67f90a28..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/build.gradle.kts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -plugins { - `java-library` -} - -dependencies { - api(project(":spi:control-plane:control-plane-spi")) - api(project(":extensions:common:aws:aws-s3-core")) - - testImplementation(testFixtures(project(":extensions:common:aws:aws-s3-test"))) -} - - diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/AwsProvisionExtension.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/AwsProvisionExtension.java deleted file mode 100644 index 72ffc656621..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/AwsProvisionExtension.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.transfer.spi.provision.ProvisionManager; -import org.eclipse.edc.connector.transfer.spi.provision.Provisioner; -import org.eclipse.edc.connector.transfer.spi.provision.ResourceManifestGenerator; -import org.eclipse.edc.connector.transfer.spi.status.StatusCheckerRegistry; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.TypeManager; - -/** - * Provides data transfer {@link Provisioner}s backed by AWS services. - */ -@Extension(value = AwsProvisionExtension.NAME) -public class AwsProvisionExtension implements ServiceExtension { - - public static final String NAME = "AWS Provision"; - @Setting - private static final String PROVISION_MAX_RETRY = "edc.aws.provision.retry.retries.max"; - @Setting - private static final String PROVISION_MAX_ROLE_SESSION_DURATION = "edc.aws.provision.role.duration.session.max"; - @Inject - private Vault vault; - @Inject - private Monitor monitor; - @Inject - private AwsClientProvider clientProvider; - - @Inject - private TypeManager typeManager; - - @Override - public String name() { - return NAME; - } - - @Override - public void initialize(ServiceExtensionContext context) { - monitor = context.getMonitor(); - - var provisionManager = context.getService(ProvisionManager.class); - - var retryPolicy = (RetryPolicy) context.getService(RetryPolicy.class); - - int maxRetries = context.getSetting(PROVISION_MAX_RETRY, 10); - int roleMaxSessionDuration = context.getSetting(PROVISION_MAX_ROLE_SESSION_DURATION, 3600); - var provisionerConfiguration = new S3BucketProvisionerConfiguration(maxRetries, roleMaxSessionDuration); - var s3BucketProvisioner = new S3BucketProvisioner(clientProvider, monitor, retryPolicy, provisionerConfiguration); - provisionManager.register(s3BucketProvisioner); - - // register the generator - var manifestGenerator = context.getService(ResourceManifestGenerator.class); - manifestGenerator.registerGenerator(new S3ConsumerResourceDefinitionGenerator()); - - var statusCheckerReg = context.getService(StatusCheckerRegistry.class); - statusCheckerReg.register(S3BucketSchema.TYPE, new S3StatusChecker(clientProvider, retryPolicy)); - - registerTypes(typeManager); - } - - @Override - public void shutdown() { - try { - clientProvider.shutdown(); - } catch (Exception e) { - monitor.severe("Error closing S3 client provider", e); - } - } - - private void registerTypes(TypeManager typeManager) { - typeManager.registerTypes(S3BucketProvisionedResource.class, S3BucketResourceDefinition.class, AwsTemporarySecretToken.class); - } - - -} - - diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java deleted file mode 100644 index c288fec8012..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedDataDestinationResource; - -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; - - -/** - * A provisioned S3 bucket and credentials associated with a transfer process. - */ -@JsonDeserialize(builder = S3BucketProvisionedResource.Builder.class) -@JsonTypeName("dataspaceconnector:s3bucketprovisionedresource") -public class S3BucketProvisionedResource extends ProvisionedDataDestinationResource { - private String role; - - public String getRegion() { - return getDataAddress().getProperty(REGION); - } - - public String getBucketName() { - return getDataAddress().getProperty(BUCKET_NAME); - } - - @Override - public String getResourceName() { - return dataAddress.getProperty(BUCKET_NAME); - } - - public String getRole() { - return role; - } - - private S3BucketProvisionedResource() { - } - - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder extends ProvisionedDataDestinationResource.Builder { - - private Builder() { - super(new S3BucketProvisionedResource()); - dataAddressBuilder.type(S3BucketSchema.TYPE); - } - - @JsonCreator - public static Builder newInstance() { - return new Builder(); - } - - public Builder region(String region) { - dataAddressBuilder.property(REGION, region); - return this; - } - - public Builder bucketName(String bucketName) { - dataAddressBuilder.property(BUCKET_NAME, bucketName); - dataAddressBuilder.keyName("s3-temp-" + bucketName); - return this; - } - - public Builder role(String arn) { - provisionedResource.role = arn; - return this; - } - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisioner.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisioner.java deleted file mode 100644 index c48044af6fe..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisioner.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.connector.transfer.spi.provision.Provisioner; -import org.eclipse.edc.connector.transfer.spi.types.DeprovisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionResponse; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.response.StatusResult; -import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.services.iam.model.Role; -import software.amazon.awssdk.services.sts.model.Credentials; - -import java.util.concurrent.CompletableFuture; - -/** - * Asynchronously provisions S3 buckets. - */ -public class S3BucketProvisioner implements Provisioner { - - private final AwsClientProvider clientProvider; - private final Monitor monitor; - private final RetryPolicy retryPolicy; - private final S3BucketProvisionerConfiguration configuration; - - public S3BucketProvisioner(AwsClientProvider clientProvider, Monitor monitor, RetryPolicy retryPolicy, S3BucketProvisionerConfiguration configuration) { - this.clientProvider = clientProvider; - this.monitor = monitor; - this.configuration = configuration; - this.retryPolicy = RetryPolicy.builder(retryPolicy.getConfig()) - .withMaxRetries(configuration.getMaxRetries()) - .handle(AwsServiceException.class) - .build(); - } - - @Override - public boolean canProvision(ResourceDefinition resourceDefinition) { - return resourceDefinition instanceof S3BucketResourceDefinition; - } - - @Override - public boolean canDeprovision(ProvisionedResource resourceDefinition) { - return resourceDefinition instanceof S3BucketProvisionedResource; - } - - @Override - public CompletableFuture> provision(S3BucketResourceDefinition resourceDefinition, Policy policy) { - return S3ProvisionPipeline.Builder.newInstance(retryPolicy) - .clientProvider(clientProvider) - .roleMaxSessionDuration(configuration.getRoleMaxSessionDuration()) - .monitor(monitor) - .build() - .provision(resourceDefinition) - .thenApply(result -> provisionSuccedeed(resourceDefinition, result.getRole(), result.getCredentials())); - } - - @Override - public CompletableFuture> deprovision(S3BucketProvisionedResource resource, Policy policy) { - return S3DeprovisionPipeline.Builder.newInstance(retryPolicy) - .clientProvider(clientProvider) - .monitor(monitor) - .build() - .deprovision(resource) - .thenApply(ignore -> StatusResult.success(DeprovisionedResource.Builder.newInstance().provisionedResourceId(resource.getId()).build())); - } - - private StatusResult provisionSuccedeed(S3BucketResourceDefinition resourceDefinition, Role role, Credentials credentials) { - var resource = S3BucketProvisionedResource.Builder.newInstance() - .id(resourceDefinition.getBucketName()) - .resourceDefinitionId(resourceDefinition.getId()) - .hasToken(true) - .region(resourceDefinition.getRegionId()) - .bucketName(resourceDefinition.getBucketName()) - .role(role.roleName()) - .transferProcessId(resourceDefinition.getTransferProcessId()) - .resourceName(resourceDefinition.getBucketName()) - .build(); - - var secretToken = new AwsTemporarySecretToken(credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken(), credentials.expiration().toEpochMilli()); - - monitor.debug("S3BucketProvisioner: Bucket request submitted: " + resourceDefinition.getBucketName()); - var response = ProvisionResponse.Builder.newInstance().resource(resource).secretToken(secretToken).build(); - return StatusResult.success(response); - } - -} - - diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerConfiguration.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerConfiguration.java deleted file mode 100644 index 05b0f1d385a..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerConfiguration.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2022 Amadeus - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amadeus - Initial implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -public class S3BucketProvisionerConfiguration { - - private final int maxRetries; - private final int roleMaxSessionDuration; - - public S3BucketProvisionerConfiguration(int maxRetries, int roleMaxSessionDuration) { - this.maxRetries = maxRetries; - this.roleMaxSessionDuration = roleMaxSessionDuration; - } - - public int getMaxRetries() { - return maxRetries; - } - - public int getRoleMaxSessionDuration() { - return roleMaxSessionDuration; - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinition.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinition.java deleted file mode 100644 index 70987733da1..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinition.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - add toBuilder method - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; - -import java.util.Objects; -import java.util.function.Supplier; - -/** - * An S3 bucket and access credentials to be provisioned. - */ -public class S3BucketResourceDefinition extends ResourceDefinition { - private String regionId; - private String bucketName; - private Supplier checker; - - private S3BucketResourceDefinition() { - } - - public String getRegionId() { - return regionId; - } - - public String getBucketName() { - return bucketName; - } - - @Override - public Builder toBuilder() { - return initializeBuilder(new Builder()) - .regionId(regionId) - .bucketName(bucketName); - } - - public static class Builder extends ResourceDefinition.Builder { - - private Builder() { - super(new S3BucketResourceDefinition()); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder regionId(String regionId) { - resourceDefinition.regionId = regionId; - return this; - } - - public Builder bucketName(String bucketName) { - resourceDefinition.bucketName = bucketName; - return this; - } - - @Override - protected void verify() { - super.verify(); - Objects.requireNonNull(resourceDefinition.regionId, "regionId"); - Objects.requireNonNull(resourceDefinition.bucketName, "bucketName"); - } - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java deleted file mode 100644 index 805fe72451f..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * ZF Friedrichshafen AG - improvements (refactoring of generate method) - * SAP SE - refactoring - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.transfer.spi.provision.ConsumerResourceDefinitionGenerator; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; -import org.eclipse.edc.policy.model.Policy; -import software.amazon.awssdk.regions.Region; - -import static java.util.UUID.randomUUID; - -/** - * Generates S3 buckets on the consumer (requesting connector) that serve as data destinations. - */ -public class S3ConsumerResourceDefinitionGenerator implements ConsumerResourceDefinitionGenerator { - - @Override - public ResourceDefinition generate(DataRequest dataRequest, Policy policy) { - if (dataRequest.getDataDestination().getProperty(S3BucketSchema.REGION) == null) { - // FIXME generate region from policy engine - return S3BucketResourceDefinition.Builder.newInstance().id(randomUUID().toString()).bucketName(dataRequest.getDataDestination().getProperty(S3BucketSchema.BUCKET_NAME)).regionId(Region.US_EAST_1.id()).build(); - } - var destination = dataRequest.getDataDestination(); - var id = randomUUID().toString(); - - return S3BucketResourceDefinition.Builder.newInstance().id(id).bucketName(destination.getProperty(S3BucketSchema.BUCKET_NAME)).regionId(destination.getProperty(S3BucketSchema.REGION)).build(); - } - - @Override - public boolean canGenerate(DataRequest dataRequest, Policy policy) { - return S3BucketSchema.TYPE.equals(dataRequest.getDestinationType()); - } -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3DeprovisionPipeline.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3DeprovisionPipeline.java deleted file mode 100644 index 57efddc478f..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3DeprovisionPipeline.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.connector.transfer.spi.types.DeprovisionedResource; -import org.eclipse.edc.spi.monitor.Monitor; -import software.amazon.awssdk.services.iam.IamAsyncClient; -import software.amazon.awssdk.services.iam.model.DeleteRolePolicyRequest; -import software.amazon.awssdk.services.iam.model.DeleteRolePolicyResponse; -import software.amazon.awssdk.services.iam.model.DeleteRoleRequest; -import software.amazon.awssdk.services.iam.model.DeleteRoleResponse; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; -import software.amazon.awssdk.services.s3.model.DeleteBucketResponse; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; - -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.joining; - -public class S3DeprovisionPipeline { - - private final RetryPolicy retryPolicy; - private final AwsClientProvider clientProvider; - private final Monitor monitor; - - public S3DeprovisionPipeline(RetryPolicy retryPolicy, AwsClientProvider clientProvider, Monitor monitor) { - this.retryPolicy = retryPolicy; - this.clientProvider = clientProvider; - this.monitor = monitor; - } - - /** - * Performs a non-blocking deprovisioning operation. - */ - public CompletableFuture deprovision(S3BucketProvisionedResource resource) { - var s3Client = clientProvider.s3AsyncClient(resource.getRegion()); - var iamClient = clientProvider.iamAsyncClient(); - - String bucketName = resource.getBucketName(); - String role = resource.getRole(); - - var listObjectsRequest = ListObjectsV2Request.builder().bucket(bucketName).build(); - monitor.debug("S3DeprovisionPipeline: list objects"); - return s3Client.listObjectsV2(listObjectsRequest) - .thenCompose(listObjectsResponse -> deleteObjects(s3Client, bucketName, listObjectsResponse)) - .thenCompose(deleteObjectsResponse -> deleteBucket(s3Client, bucketName)) - .thenCompose(listAttachedRolePoliciesResponse -> deleteRolePolicy(iamClient, role)) - .thenCompose(deleteRolePolicyResponse -> deleteRole(iamClient, role)) - .thenApply(response -> DeprovisionedResource.Builder.newInstance().provisionedResourceId(resource.getId()).build()); - } - - private CompletableFuture deleteRole(IamAsyncClient iamClient, String role) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - monitor.debug("S3DeprovisionPipeline: delete role"); - return iamClient.deleteRole(DeleteRoleRequest.builder().roleName(role).build()); - }); - } - - private CompletableFuture deleteRolePolicy(IamAsyncClient iamClient, String role) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - monitor.debug("S3DeprovisionPipeline: deleting inline policies for Role " + role); - return iamClient.deleteRolePolicy(DeleteRolePolicyRequest.builder().roleName(role).policyName(role).build()); - }); - } - - private CompletableFuture deleteBucket(S3AsyncClient s3Client, String bucketName) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - monitor.debug("S3DeprovisionPipeline: delete bucket"); - return s3Client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()); - }); - } - - private CompletableFuture deleteObjects(S3AsyncClient s3Client, String bucketName, ListObjectsV2Response listObjectsResponse) { - var identifiers = listObjectsResponse.contents().stream() - .map(s3object -> ObjectIdentifier.builder().key(s3object.key()).build()) - .collect(Collectors.toList()); - - var deleteRequest = DeleteObjectsRequest.builder() - .bucket(bucketName).delete(Delete.builder().objects(identifiers).build()) - .build(); - monitor.debug("S3DeprovisionPipeline: delete bucket contents: " + identifiers.stream().map(ObjectIdentifier::key).collect(joining(", "))); - return s3Client.deleteObjects(deleteRequest); - } - - static class Builder { - private final RetryPolicy retryPolicy; - private Monitor monitor; - private AwsClientProvider clientProvider; - - private Builder(RetryPolicy retryPolicy) { - this.retryPolicy = retryPolicy; - } - - public static Builder newInstance(RetryPolicy policy) { - return new Builder(policy); - } - - public Builder monitor(Monitor monitor) { - this.monitor = monitor; - return this; - } - - public Builder clientProvider(AwsClientProvider clientProvider) { - this.clientProvider = clientProvider; - return this; - } - - public S3DeprovisionPipeline build() { - Objects.requireNonNull(retryPolicy); - Objects.requireNonNull(clientProvider); - Objects.requireNonNull(monitor); - return new S3DeprovisionPipeline(retryPolicy, clientProvider, monitor); - } - } -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionPipeline.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionPipeline.java deleted file mode 100644 index 0fb053e48ec..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionPipeline.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.spi.monitor.Monitor; -import software.amazon.awssdk.services.iam.IamAsyncClient; -import software.amazon.awssdk.services.iam.model.CreateRoleRequest; -import software.amazon.awssdk.services.iam.model.CreateRoleResponse; -import software.amazon.awssdk.services.iam.model.GetUserResponse; -import software.amazon.awssdk.services.iam.model.PutRolePolicyRequest; -import software.amazon.awssdk.services.iam.model.Role; -import software.amazon.awssdk.services.iam.model.Tag; -import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.sts.StsAsyncClient; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; - -import java.util.Objects; -import java.util.concurrent.CompletableFuture; - -import static java.lang.String.format; - -public class S3ProvisionPipeline { - - // Do not modify this trust policy - private static final String ASSUME_ROLE_TRUST = "{" + - " \"Version\": \"2012-10-17\"," + - " \"Statement\": [" + - " {" + - " \"Effect\": \"Allow\"," + - " \"Principal\": {" + - " \"AWS\": \"%s\"" + - " }," + - " \"Action\": \"sts:AssumeRole\"" + - " }" + - " ]" + - "}"; - // Do not modify this bucket policy - private static final String BUCKET_POLICY = "{" + - " \"Version\": \"2012-10-17\"," + - " \"Statement\": [" + - " {" + - " \"Sid\": \"TemporaryAccess\", " + - " \"Effect\": \"Allow\"," + - " \"Action\": \"s3:PutObject\"," + - " \"Resource\": \"arn:aws:s3:::%s/*\"" + - " }" + - " ]" + - "}"; - - private final RetryPolicy retryPolicy; - private final AwsClientProvider clientProvider; - private final Monitor monitor; - private final int roleMaxSessionDuration; - - private S3ProvisionPipeline(RetryPolicy retryPolicy, AwsClientProvider clientProvider, - Monitor monitor, int roleMaxSessionDuration) { - this.retryPolicy = retryPolicy; - this.clientProvider = clientProvider; - this.monitor = monitor; - this.roleMaxSessionDuration = roleMaxSessionDuration; - } - - /** - * Performs a non-blocking provisioning operation. - */ - public CompletableFuture provision(S3BucketResourceDefinition resourceDefinition) { - var s3AsyncClient = clientProvider.s3AsyncClient(resourceDefinition.getRegionId()); - var iamClient = clientProvider.iamAsyncClient(); - var stsClient = clientProvider.stsAsyncClient(resourceDefinition.getRegionId()); - - var request = CreateBucketRequest.builder() - .bucket(resourceDefinition.getBucketName()) - .createBucketConfiguration(CreateBucketConfiguration.builder().build()) - .build(); - - monitor.debug("S3ProvisionPipeline: create bucket " + resourceDefinition.getBucketName()); - return s3AsyncClient.createBucket(request) - .thenCompose(r -> getUser(iamClient)) - .thenCompose(response -> createRole(iamClient, resourceDefinition, response)) - .thenCompose(response -> createRolePolicy(iamClient, resourceDefinition, response)) - .thenCompose(role -> assumeRole(stsClient, role)); - } - - private CompletableFuture createRolePolicy(IamAsyncClient iamAsyncClient, S3BucketResourceDefinition resourceDefinition, CreateRoleResponse response) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - Role role = response.role(); - PutRolePolicyRequest policyRequest = PutRolePolicyRequest.builder() - .policyName(resourceDefinition.getTransferProcessId()) - .roleName(role.roleName()) - .policyDocument(format(BUCKET_POLICY, resourceDefinition.getBucketName())) - .build(); - - monitor.debug("S3ProvisionPipeline: attach bucket policy to role " + role.arn()); - return iamAsyncClient.putRolePolicy(policyRequest) - .thenApply(policyResponse -> role); - }); - } - - private CompletableFuture createRole(IamAsyncClient iamClient, S3BucketResourceDefinition resourceDefinition, GetUserResponse response) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - String userArn = response.user().arn(); - Tag tag = Tag.builder().key("dataspaceconnector:process").value(resourceDefinition.getTransferProcessId()).build(); - - monitor.debug("S3ProvisionPipeline: create role for user" + userArn); - CreateRoleRequest createRoleRequest = CreateRoleRequest.builder() - .roleName(resourceDefinition.getTransferProcessId()).description("EDC transfer process role") - .assumeRolePolicyDocument(format(ASSUME_ROLE_TRUST, userArn)) - .maxSessionDuration(roleMaxSessionDuration) - .tags(tag) - .build(); - - return iamClient.createRole(createRoleRequest); - }); - } - - private CompletableFuture getUser(IamAsyncClient iamAsyncClient) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - monitor.debug("S3ProvisionPipeline: get user"); - return iamAsyncClient.getUser(); - }); - } - - private CompletableFuture assumeRole(StsAsyncClient stsClient, Role role) { - return Failsafe.with(retryPolicy).getStageAsync(() -> { - monitor.debug("S3ProvisionPipeline: attempting to assume the role"); - AssumeRoleRequest roleRequest = AssumeRoleRequest.builder() - .roleArn(role.arn()) - .roleSessionName("transfer") - .externalId("123") - .build(); - - return stsClient.assumeRole(roleRequest) - .thenApply(response -> new S3ProvisionResponse(role, response.credentials())); - }); - } - - static class Builder { - private final RetryPolicy retryPolicy; - private int roleMaxSessionDuration; - private Monitor monitor; - private AwsClientProvider clientProvider; - - private Builder(RetryPolicy retryPolicy) { - this.retryPolicy = retryPolicy; - } - - public static Builder newInstance(RetryPolicy policy) { - return new Builder(policy); - } - - public Builder roleMaxSessionDuration(int roleMaxSessionDuration) { - this.roleMaxSessionDuration = roleMaxSessionDuration; - return this; - } - - public Builder monitor(Monitor monitor) { - this.monitor = monitor; - return this; - } - - public Builder clientProvider(AwsClientProvider clientProvider) { - this.clientProvider = clientProvider; - return this; - } - - public S3ProvisionPipeline build() { - Objects.requireNonNull(retryPolicy); - Objects.requireNonNull(clientProvider); - Objects.requireNonNull(monitor); - return new S3ProvisionPipeline(retryPolicy, clientProvider, monitor, roleMaxSessionDuration); - } - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionResponse.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionResponse.java deleted file mode 100644 index db69edbb214..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ProvisionResponse.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2022 Amadeus - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Amadeus - Initial implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import software.amazon.awssdk.services.iam.model.Role; -import software.amazon.awssdk.services.sts.model.Credentials; - -public class S3ProvisionResponse { - private final Role role; - private final Credentials credentials; - - public S3ProvisionResponse(Role role, Credentials credentials) { - this.role = role; - this.credentials = credentials; - } - - public Role getRole() { - return role; - } - - public Credentials getCredentials() { - return credentials; - } -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusChecker.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusChecker.java deleted file mode 100644 index f2ed44b3873..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusChecker.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.StatusChecker; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.spi.EdcException; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; -import software.amazon.awssdk.services.s3.model.S3Exception; - -import java.util.List; -import java.util.concurrent.CompletionException; - -import static java.lang.String.format; - -public class S3StatusChecker implements StatusChecker { - private final AwsClientProvider clientProvider; - private final RetryPolicy retryPolicy; - - public S3StatusChecker(AwsClientProvider clientProvider, RetryPolicy retryPolicy) { - this.clientProvider = clientProvider; - this.retryPolicy = retryPolicy; - } - - @Override - public boolean isComplete(TransferProcess transferProcess, List resources) { - if (resources.isEmpty()) { - var destination = transferProcess.getDataRequest().getDataDestination(); - var bucketName = destination.getProperty(S3BucketSchema.BUCKET_NAME); - var region = destination.getProperty(S3BucketSchema.REGION); - return checkBucket(bucketName, region); - } else { - for (var resource : resources) { - if (resource instanceof S3BucketProvisionedResource) { - var provisionedResource = (S3BucketProvisionedResource) resource; - try { - var bucketName = provisionedResource.getBucketName(); - var region = provisionedResource.getRegion(); - return checkBucket(bucketName, region); - } catch (CompletionException cpe) { - if (cpe.getCause() instanceof NoSuchBucketException) { - return false; - } - throw cpe; - } - - } - } - - } - - // otherwise, we have an implementation error - throw new EdcException(format("No bucket resource was associated with the transfer process: %s - cannot determine completion.", transferProcess.getId())); - } - - private boolean checkBucket(String bucketName, String region) { - try { - var s3client = clientProvider.s3AsyncClient(region); - - var rq = ListObjectsRequest.builder().bucket(bucketName).build(); - var response = Failsafe.with(retryPolicy) - .getStageAsync(() -> s3client.listObjects(rq)) - .join(); - return response.contents().stream().anyMatch(s3object -> s3object.key().endsWith(".complete")); - } catch (CompletionException ex) { - if (ex.getCause() instanceof S3Exception) { - return false; - } else { - throw ex; - } - } - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/control-plane/provision/provision-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 98509675e34..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2020, 2021 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.connector.provision.aws.s3.AwsProvisionExtension - diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResourceTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResourceTest.java deleted file mode 100644 index d9621902730..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResourceTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import org.eclipse.edc.spi.types.TypeManager; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.StringWriter; - -import static java.util.UUID.randomUUID; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -class S3BucketProvisionedResourceTest { - - private S3BucketProvisionedResource provisionedResource; - - @Test - void verifyDeserialize() throws IOException { - var mapper = new TypeManager().getMapper(); - - StringWriter writer = new StringWriter(); - mapper.writeValue(writer, provisionedResource); - - S3BucketProvisionedResource deserialized = mapper.readValue(writer.toString(), S3BucketProvisionedResource.class); - - assertNotNull(deserialized); - assertEquals("region", deserialized.getRegion()); - assertEquals("bucket", deserialized.getBucketName()); - } - - @BeforeEach - void setUp() { - provisionedResource = S3BucketProvisionedResource.Builder.newInstance() - .id(randomUUID().toString()) - .transferProcessId("123") - .resourceDefinitionId(randomUUID().toString()) - .resourceName("resource") - .region("region") - .bucketName("bucket") - .build(); - } -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerTest.java deleted file mode 100644 index 5403bd0c1d2..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionerTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.spi.monitor.Monitor; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.iam.IamAsyncClient; -import software.amazon.awssdk.services.iam.model.CreateRoleRequest; -import software.amazon.awssdk.services.iam.model.CreateRoleResponse; -import software.amazon.awssdk.services.iam.model.GetUserResponse; -import software.amazon.awssdk.services.iam.model.PutRolePolicyRequest; -import software.amazon.awssdk.services.iam.model.PutRolePolicyResponse; -import software.amazon.awssdk.services.iam.model.Role; -import software.amazon.awssdk.services.iam.model.User; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.CreateBucketResponse; -import software.amazon.awssdk.services.sts.StsAsyncClient; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; -import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; -import software.amazon.awssdk.services.sts.model.Credentials; - -import java.time.Instant; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import static java.util.concurrent.CompletableFuture.failedFuture; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class S3BucketProvisionerTest { - - private S3BucketProvisioner provisioner; - private final IamAsyncClient iamClient = mock(IamAsyncClient.class); - private final StsAsyncClient stsClient = mock(StsAsyncClient.class); - private final S3AsyncClient s3Client = mock(S3AsyncClient.class); - private final AwsClientProvider clientProvider = mock(AwsClientProvider.class); - - @BeforeEach - void setUp() { - when(clientProvider.iamAsyncClient()).thenReturn(iamClient); - when(clientProvider.s3AsyncClient(anyString())).thenReturn(s3Client); - when(clientProvider.stsAsyncClient(anyString())).thenReturn(stsClient); - - var configuration = new S3BucketProvisionerConfiguration(2, 3600); - - provisioner = new S3BucketProvisioner(clientProvider, mock(Monitor.class), RetryPolicy.ofDefaults(), configuration); - } - - @Test - void verify_basic_provision() { - var userResponse = GetUserResponse.builder().user(User.builder().arn("testarn").build()).build(); - var createRoleResponse = CreateRoleResponse.builder().role(Role.builder().roleName("roleName").arn("testarn").build()).build(); - var putRolePolicyResponse = PutRolePolicyResponse.builder().build(); - when(iamClient.getUser()).thenReturn(completedFuture(userResponse)); - when(iamClient.createRole(isA(CreateRoleRequest.class))).thenReturn(completedFuture(createRoleResponse)); - when(iamClient.putRolePolicy(isA(PutRolePolicyRequest.class))).thenReturn(completedFuture(putRolePolicyResponse)); - - var credentials = Credentials.builder() - .accessKeyId("accessKeyId").secretAccessKey("secretAccessKey").sessionToken("sessionToken") - .expiration(Instant.now()).build(); - var assumeRoleResponse = AssumeRoleResponse.builder().credentials(credentials).build(); - when(stsClient.assumeRole(isA(AssumeRoleRequest.class))).thenReturn(completedFuture(assumeRoleResponse)); - - var createBucketResponse = CreateBucketResponse.builder().build(); - when(s3Client.createBucket(isA(CreateBucketRequest.class))).thenReturn(completedFuture(createBucketResponse)); - - S3BucketResourceDefinition definition = S3BucketResourceDefinition.Builder.newInstance().id("test").regionId(Region.US_EAST_1.id()).bucketName("test").transferProcessId("test").build(); - var policy = Policy.Builder.newInstance().build(); - - var response = provisioner.provision(definition, policy).join().getContent(); - - assertThat(response.getResource()).isInstanceOfSatisfying(S3BucketProvisionedResource.class, resource -> assertThat(resource.getRole()).isEqualTo("roleName")); - assertThat(response.getSecretToken()).isInstanceOfSatisfying(AwsTemporarySecretToken.class, secretToken -> { - assertThat(secretToken.getAccessKeyId()).isEqualTo("accessKeyId"); - assertThat(secretToken.getSecretAccessKey()).isEqualTo("secretAccessKey"); - assertThat(secretToken.getSessionToken()).isEqualTo("sessionToken"); - }); - verify(iamClient).putRolePolicy(isA(PutRolePolicyRequest.class)); - } - - @Test - void should_return_failed_future_on_error() { - when(s3Client.createBucket(isA(CreateBucketRequest.class))).thenReturn(failedFuture(new RuntimeException("any"))); - S3BucketResourceDefinition definition = S3BucketResourceDefinition.Builder.newInstance().id("test").regionId(Region.US_EAST_1.id()).bucketName("test").transferProcessId("test").build(); - - var policy = Policy.Builder.newInstance().build(); - - var response = provisioner.provision(definition, policy); - - assertThat(response).failsWithin(1, SECONDS); - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinitionTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinitionTest.java deleted file mode 100644 index e5ddb3c1d5c..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketResourceDefinitionTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -class S3BucketResourceDefinitionTest { - - @Test - void toBuilder_verifyEqualResourceDefinition() { - var definition = S3BucketResourceDefinition.Builder.newInstance() - .id("id") - .transferProcessId("tp-id") - .regionId("region") - .bucketName("bucket") - .build(); - var builder = definition.toBuilder(); - var rebuiltDefinition = builder.build(); - - assertThat(rebuiltDefinition).usingRecursiveComparison().isEqualTo(definition); - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java deleted file mode 100644 index 40cd8874b14..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2022 ZF Friedrichshafen AG - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * ZF Friedrichshafen AG - Initial API and Implementation - * SAP SE - refactoring - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.asset.Asset; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.regions.Region; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; - - -public class S3ConsumerResourceDefinitionGeneratorTest { - - private S3ConsumerResourceDefinitionGenerator generator; - - @BeforeEach - void setUp() { - generator = new S3ConsumerResourceDefinitionGenerator(); - } - - @Test - void generate() { - var destination = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, "test-name") - .property(S3BucketSchema.REGION, Region.EU_WEST_2.id()) - .build(); - var asset = Asset.Builder.newInstance().build(); - var dr = DataRequest.Builder.newInstance().dataDestination(destination).assetId(asset.getId()).processId("process-id").build(); - var policy = Policy.Builder.newInstance().build(); - - var definition = generator.generate(dr, policy); - - assertThat(definition).isInstanceOf(S3BucketResourceDefinition.class); - var objectDef = (S3BucketResourceDefinition) definition; - assertThat(objectDef.getBucketName()).isEqualTo("test-name"); - assertThat(objectDef.getRegionId()).isEqualTo(Region.EU_WEST_2.id()); - assertThat(objectDef.getId()).satisfies(UUID::fromString); - } - - @Test - void generate_noDataRequestAsParameter() { - var policy = Policy.Builder.newInstance().build(); - assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> generator.generate(null, policy)); - } - - @Test - void generate_noRegionSpecified() { - var destination = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, "test-name") - .build(); - var asset = Asset.Builder.newInstance().build(); - var dr = DataRequest.Builder.newInstance().dataDestination(destination).assetId(asset.getId()).processId("process-id").build(); - var policy = Policy.Builder.newInstance().build(); - - var definition = generator.generate(dr, policy); - - assertThat(definition).isInstanceOf(S3BucketResourceDefinition.class); - var objectDef = (S3BucketResourceDefinition) definition; - assertThat(objectDef.getBucketName()).isEqualTo("test-name"); - assertThat(objectDef.getRegionId()).isEqualTo(Region.US_EAST_1.id()); - assertThat(objectDef.getId()).satisfies(UUID::fromString); - } - - @Test - void generate_noBucketNameSpecified() { - var destination = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE) - .property(S3BucketSchema.REGION, Region.EU_WEST_2.id()) - .build(); - var asset = Asset.Builder.newInstance().build(); - var dr = DataRequest.Builder.newInstance().dataDestination(destination).assetId(asset.getId()).processId("process-id").build(); - var policy = Policy.Builder.newInstance().build(); - - assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> generator.generate(dr, policy)); - } - - @Test - void canGenerate() { - var destination = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, "test-name") - .property(S3BucketSchema.REGION, Region.US_EAST_1.id()) - .build(); - var asset = Asset.Builder.newInstance().build(); - var dr = DataRequest.Builder.newInstance().dataDestination(destination).assetId(asset.getId()).processId("process-id").build(); - var policy = Policy.Builder.newInstance().build(); - - var definition = generator.canGenerate(dr, policy); - - assertThat(definition).isTrue(); - } - - @Test - void canGenerateIsNotTypeS3BucketSchema() { - var destination = DataAddress.Builder.newInstance().type("aNonS3BucketSchema") - .property(S3BucketSchema.BUCKET_NAME, "test-name") - .property(S3BucketSchema.REGION, Region.US_EAST_1.id()) - .build(); - var asset = Asset.Builder.newInstance().build(); - var dataRequest = DataRequest.Builder.newInstance().dataDestination(destination).assetId(asset.getId()).build(); - var policy = Policy.Builder.newInstance().build(); - - var definition = generator.canGenerate(dataRequest, policy); - assertThat(definition).isFalse(); - } - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusCheckerIntegrationTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusCheckerIntegrationTest.java deleted file mode 100644 index 58d43643430..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3StatusCheckerIntegrationTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.connector.provision.aws.s3; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.aws.s3.testfixtures.AbstractS3Test; -import org.eclipse.edc.aws.s3.testfixtures.annotations.AwsS3IntegrationTest; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.UUID; -import java.util.function.Supplier; - -import static java.util.Collections.emptyList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.junit.testfixtures.TestUtils.getFileFromResourceName; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@AwsS3IntegrationTest -class S3StatusCheckerIntegrationTest extends AbstractS3Test { - - public static final int ONE_MINUTE_MILLIS = 1000 * 60; - private static final String PROCESS_ID = UUID.randomUUID().toString(); - - private S3StatusChecker checker; - - @BeforeEach - void setup() { - var retryPolicy = RetryPolicy.builder().withMaxRetries(3).withBackoff(200, 1000, ChronoUnit.MILLIS).build(); - var providerMock = mock(AwsClientProvider.class); - when(providerMock.s3AsyncClient(anyString())).thenReturn(s3AsyncClient); - checker = new S3StatusChecker(providerMock, retryPolicy); - } - - @Test - void isComplete_noResources_whenNotComplete() { - var complete = checker.isComplete(createTransferProcess(bucketName), emptyList()); - - assertThat(complete).isFalse(); - } - - @Test - void isComplete_noResources_whenComplete() throws InterruptedException { - putTestFile(PROCESS_ID + ".complete", getFileFromResourceName("hello.txt"), bucketName); - var transferProcess = createTransferProcess(bucketName); - - var hasCompleted = waitUntil(() -> checker.isComplete(transferProcess, emptyList()), ONE_MINUTE_MILLIS); - - assertThat(hasCompleted).isTrue(); - } - - @Test - void isComplete_noResources_whenBucketNotExist() { - var complete = checker.isComplete(createTransferProcess(bucketName), emptyList()); - - assertThat(complete).isFalse(); - } - - @Test - void isComplete_withResources_whenNotComplete() { - var transferProcess = createTransferProcess(bucketName); - var provisionedResource = createProvisionedResource(transferProcess); - - var complete = checker.isComplete(transferProcess, List.of(provisionedResource)); - - assertThat(complete).isFalse(); - } - - @Test - void isComplete_withResources_whenComplete() throws InterruptedException { - putTestFile(PROCESS_ID + ".complete", getFileFromResourceName("hello.txt"), bucketName); - TransferProcess tp = createTransferProcess(bucketName); - - var hasCompleted = waitUntil(() -> checker.isComplete(tp, emptyList()), ONE_MINUTE_MILLIS); - - assertThat(hasCompleted).isTrue(); - } - - @Test - void isComplete_withResources_whenBucketNotExist() { - var transferProcess = createTransferProcess(bucketName); - var provisionedResource = createProvisionedResource(transferProcess); - - boolean complete = checker.isComplete(transferProcess, List.of(provisionedResource)); - - assertThat(complete).isFalse(); - } - - private boolean waitUntil(Supplier conditionSupplier, long maxTimeMillis) throws InterruptedException { - var done = false; - var start = System.currentTimeMillis(); - var complete = false; - while (!done) { - complete = conditionSupplier.get(); - if (complete) { - done = true; - } else { - done = System.currentTimeMillis() - start > maxTimeMillis; - } - Thread.sleep(5); - } - return complete; - } - - private S3BucketProvisionedResource createProvisionedResource(TransferProcess transferProcess) { - return S3BucketProvisionedResource.Builder.newInstance() - .bucketName(bucketName) - .region(REGION) - .resourceDefinitionId(UUID.randomUUID().toString()) - .transferProcessId(transferProcess.getId()) - .id(UUID.randomUUID().toString()) - .resourceName(bucketName) - .build(); - } - - private TransferProcess createTransferProcess(String bucketName) { - return TransferProcess.Builder.newInstance() - .id(UUID.randomUUID().toString()) - .dataRequest(DataRequest.Builder.newInstance() - .destinationType(S3BucketSchema.TYPE) - .dataDestination(DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .property(S3BucketSchema.REGION, AbstractS3Test.REGION) - .property(S3BucketSchema.BUCKET_NAME, bucketName) - .build()) - .build()) - .build(); - } - - -} diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/resources/hello.txt b/extensions/control-plane/provision/provision-aws-s3/src/test/resources/hello.txt deleted file mode 100644 index bc7774a7b18..00000000000 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/resources/hello.txt +++ /dev/null @@ -1 +0,0 @@ -hello world! \ No newline at end of file diff --git a/extensions/data-plane/data-plane-aws-s3/README.md b/extensions/data-plane/data-plane-aws-s3/README.md deleted file mode 100644 index 194beb4a578..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Aws S3 Data Plane module - -### About this module - -This module contains a Data Plane extension to copy data to and from Aws S3. - -When used as a source, it currently only supports copying a single object. \ No newline at end of file diff --git a/extensions/data-plane/data-plane-aws-s3/build.gradle.kts b/extensions/data-plane/data-plane-aws-s3/build.gradle.kts deleted file mode 100644 index d4dc6e96e13..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/build.gradle.kts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - - -plugins { - `java-library` -} - -dependencies { - api(project(":spi:data-plane:data-plane-spi")) - implementation(project(":core:common:util")) - implementation(project(":core:data-plane:data-plane-util")) - implementation(project(":extensions:common:aws:aws-s3-core")) - - implementation(libs.failsafe.core) - - testImplementation(project(":core:data-plane:data-plane-core")) - testImplementation(testFixtures(project(":extensions:common:aws:aws-s3-test"))) - testImplementation(project(":core:common:junit")) -} - - diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java deleted file mode 100644 index 43e39ba931c..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.TypeManager; - -import java.util.concurrent.Executors; - -@Extension(value = DataPlaneS3Extension.NAME) -public class DataPlaneS3Extension implements ServiceExtension { - - public static final String NAME = "Data Plane S3 Storage"; - @Inject - private PipelineService pipelineService; - - @Inject - private AwsClientProvider awsClientProvider; - - @Inject - private Vault vault; - - @Inject - private TypeManager typeManager; - - @Override - public String name() { - return NAME; - } - - @Override - public void initialize(ServiceExtensionContext context) { - var executorService = Executors.newFixedThreadPool(10); // TODO make configurable - - var monitor = context.getMonitor(); - - var sourceFactory = new S3DataSourceFactory(awsClientProvider, vault, typeManager); - pipelineService.registerFactory(sourceFactory); - - var sinkFactory = new S3DataSinkFactory(awsClientProvider, executorService, monitor, vault, typeManager); - pipelineService.registerFactory(sinkFactory); - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSink.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSink.java deleted file mode 100644 index 92bcefaaf80..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSink.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; -import org.jetbrains.annotations.NotNull; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static java.lang.String.format; - -class S3DataSink extends ParallelSink { - - private S3Client client; - private String bucketName; - private String keyName; - private int chunkSize; - - private S3DataSink() {} - - @Override - protected StreamResult transferParts(List parts) { - for (var part : parts) { - try (var input = part.openStream()) { - - var partNumber = 1; - var completedParts = new ArrayList(); - - var uploadId = client.createMultipartUpload(CreateMultipartUploadRequest.builder() - .bucket(bucketName) - .key(keyName) - .build()).uploadId(); - - while (true) { - var bytesChunk = input.readNBytes(chunkSize); - - if (bytesChunk.length < 1) { - break; - } - - completedParts.add(CompletedPart.builder().partNumber(partNumber) - .eTag(client.uploadPart(UploadPartRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .partNumber(partNumber) - .build(), RequestBody.fromByteBuffer(ByteBuffer.wrap(bytesChunk))).eTag()).build()); - partNumber++; - } - - client.completeMultipartUpload(CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .multipartUpload(CompletedMultipartUpload.builder() - .parts(completedParts) - .build()) - .build()); - - } catch (Exception e) { - return uploadFailure(e, keyName); - } - } - - return StreamResult.success(); - } - - @Override - protected StreamResult complete() { - var completeKeyName = keyName + ".complete"; - var request = PutObjectRequest.builder().bucket(bucketName).key(completeKeyName).build(); - try { - client.putObject(request, RequestBody.empty()); - return super.complete(); - } catch (Exception e) { - return uploadFailure(e, completeKeyName); - } - - } - - @NotNull - private StreamResult uploadFailure(Exception e, String keyName) { - var message = format("Error writing the %s object on the %s bucket: %s", keyName, bucketName, e.getMessage()); - monitor.severe(message, e); - return StreamResult.error(message); - } - - public static class Builder extends ParallelSink.Builder { - - private Builder() { - super(new S3DataSink()); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder client(S3Client client) { - sink.client = client; - return this; - } - - public Builder bucketName(String bucketName) { - sink.bucketName = bucketName; - return this; - } - - public Builder keyName(String keyName) { - sink.keyName = keyName; - return this; - } - - public Builder chunkSizeBytes(int chunkSize) { - sink.chunkSize = chunkSize; - return this; - } - - @Override - protected void validate() {} - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java deleted file mode 100644 index 5eeecabdbda..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsSecretToken; -import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.dataplane.aws.s3.validation.S3DataAddressCredentialsValidationRule; -import org.eclipse.edc.connector.dataplane.aws.s3.validation.S3DataAddressValidationRule; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; -import org.eclipse.edc.connector.dataplane.util.validation.ValidationRule; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; -import software.amazon.awssdk.services.s3.S3Client; - -import java.util.concurrent.ExecutorService; - -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; - -public class S3DataSinkFactory implements DataSinkFactory { - - private static final int CHUNK_SIZE_IN_BYTES = 1024 * 1024 * 500; // 500MB chunk size - - private final ValidationRule validation = new S3DataAddressValidationRule(); - private final ValidationRule credentialsValidation = new S3DataAddressCredentialsValidationRule(); - private final AwsClientProvider clientProvider; - private final ExecutorService executorService; - private final Monitor monitor; - private Vault vault; - private TypeManager typeManager; - - public S3DataSinkFactory(AwsClientProvider clientProvider, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager) { - this.clientProvider = clientProvider; - this.executorService = executorService; - this.monitor = monitor; - this.vault = vault; - this.typeManager = typeManager; - } - - @Override - public boolean canHandle(DataFlowRequest request) { - return S3BucketSchema.TYPE.equals(request.getDestinationDataAddress().getType()); - } - - @Override - public @NotNull Result validate(DataFlowRequest request) { - return validateRequest(request).map(it -> true); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - var destination = request.getDestinationDataAddress(); - - return validation.apply(destination).map(it -> null); - } - - @Override - public DataSink createSink(DataFlowRequest request) { - var validationResult = validateRequest(request); - if (validationResult.failed()) { - throw new EdcException(String.join(", ", validationResult.getFailureMessages())); - } - - var destination = request.getDestinationDataAddress(); - - S3Client client; - var secret = vault.resolveSecret(destination.getKeyName()); - if (secret != null) { - var secretToken = typeManager.readValue(secret, AwsTemporarySecretToken.class); - client = clientProvider.s3Client(destination.getProperty(REGION), secretToken); - } else if (credentialsValidation.apply(destination).succeeded()) { - var secretToken = new AwsSecretToken(destination.getProperty(ACCESS_KEY_ID), destination.getProperty(SECRET_ACCESS_KEY)); - client = clientProvider.s3Client(destination.getProperty(REGION), secretToken); - } else { - client = clientProvider.s3Client(destination.getProperty(REGION)); - } - - return S3DataSink.Builder.newInstance() - .bucketName(destination.getProperty(BUCKET_NAME)) - .keyName(destination.getKeyName()) - .requestId(request.getId()) - .executorService(executorService) - .monitor(monitor) - .client(client) - .chunkSizeBytes(CHUNK_SIZE_IN_BYTES) - .build(); - } - -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSource.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSource.java deleted file mode 100644 index 4533130530a..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSource.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; - -import java.io.InputStream; -import java.util.stream.Stream; - -import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult.success; - -class S3DataSource implements DataSource { - - private String bucketName; - private String keyName; - private S3Client client; - - private S3DataSource() { } - - @Override - public StreamResult> openPartStream() { - return success(Stream.of(new S3Part(client, keyName, bucketName))); - } - - private static class S3Part implements Part { - private final S3Client client; - private final String keyName; - private final String bucketName; - - S3Part(S3Client client, String keyName, String bucketName) { - this.client = client; - this.keyName = keyName; - this.bucketName = bucketName; - } - - @Override - public String name() { - return keyName; - } - - @Override - public long size() { - var request = HeadObjectRequest.builder().key(keyName).bucket(bucketName).build(); - return client.headObject(request).contentLength(); - } - - @Override - public InputStream openStream() { - var request = GetObjectRequest.builder().key(keyName).bucket(bucketName).build(); - return client.getObject(request); - } - } - - public static class Builder { - private final S3DataSource source; - - private Builder() { - source = new S3DataSource(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder bucketName(String bucketName) { - source.bucketName = bucketName; - return this; - } - - public Builder keyName(String keyName) { - source.keyName = keyName; - return this; - } - - public Builder client(S3Client client) { - source.client = client; - return this; - } - - public S3DataSource build() { - return source; - } - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java deleted file mode 100644 index 30709805ceb..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsSecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.dataplane.aws.s3.validation.S3DataAddressCredentialsValidationRule; -import org.eclipse.edc.connector.dataplane.aws.s3.validation.S3DataAddressValidationRule; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; -import org.eclipse.edc.connector.dataplane.util.validation.ValidationRule; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; -import software.amazon.awssdk.services.s3.S3Client; - -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; - -public class S3DataSourceFactory implements DataSourceFactory { - - private final ValidationRule validation = new S3DataAddressValidationRule(); - private final ValidationRule credentialsValidation = new S3DataAddressCredentialsValidationRule(); - private final AwsClientProvider clientProvider; - private final Vault vault; - - private final TypeManager typeManager; - - public S3DataSourceFactory(AwsClientProvider clientProvider, Vault vault, TypeManager typeManager) { - this.clientProvider = clientProvider; - this.vault = vault; - this.typeManager = typeManager; - } - - @Override - public boolean canHandle(DataFlowRequest request) { - return S3BucketSchema.TYPE.equals(request.getSourceDataAddress().getType()); - } - - @Override - public @NotNull Result validate(DataFlowRequest request) { - return validateRequest(request).map(it -> true); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - var source = request.getSourceDataAddress(); - - return validation.apply(source).map(it -> null); - } - - @Override - public DataSource createSource(DataFlowRequest request) { - var validationResult = validateRequest(request); - if (validationResult.failed()) { - throw new EdcException(String.join(", ", validationResult.getFailureMessages())); - } - - var source = request.getSourceDataAddress(); - - S3Client client; - var secret = vault.resolveSecret(source.getKeyName()); - if (secret != null) { - var secretToken = typeManager.readValue(secret, AwsSecretToken.class); - client = clientProvider.s3Client(source.getProperty(REGION), secretToken); - } else if (credentialsValidation.apply(source).succeeded()) { - var secretToken = new AwsSecretToken(source.getProperty(ACCESS_KEY_ID), source.getProperty(SECRET_ACCESS_KEY)); - client = clientProvider.s3Client(source.getProperty(REGION), secretToken); - } else { - client = clientProvider.s3Client(source.getProperty(REGION)); - } - - return S3DataSource.Builder.newInstance() - .bucketName(source.getProperty(BUCKET_NAME)) - .keyName(source.getKeyName()) - .client(client) - .build(); - } - -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressCredentialsValidationRule.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressCredentialsValidationRule.java deleted file mode 100644 index 886bbceb943..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressCredentialsValidationRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - Initial implementation - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3.validation; - -import org.eclipse.edc.connector.dataplane.util.validation.CompositeValidationRule; -import org.eclipse.edc.connector.dataplane.util.validation.EmptyValueValidationRule; -import org.eclipse.edc.connector.dataplane.util.validation.ValidationRule; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.DataAddress; - -import java.util.List; - -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; - -public class S3DataAddressCredentialsValidationRule implements ValidationRule { - - @Override - public Result apply(DataAddress dataAddress) { - var composite = new CompositeValidationRule<>( - List.of( - new EmptyValueValidationRule(ACCESS_KEY_ID), - new EmptyValueValidationRule(SECRET_ACCESS_KEY) - ) - ); - - return composite.apply(dataAddress.getProperties()); - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressValidationRule.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressValidationRule.java deleted file mode 100644 index 046d9a28d93..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/validation/S3DataAddressValidationRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - Initial implementation - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3.validation; - -import org.eclipse.edc.connector.dataplane.util.validation.CompositeValidationRule; -import org.eclipse.edc.connector.dataplane.util.validation.EmptyValueValidationRule; -import org.eclipse.edc.connector.dataplane.util.validation.ValidationRule; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.DataAddress; - -import java.util.List; - -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; - -public class S3DataAddressValidationRule implements ValidationRule { - - @Override - public Result apply(DataAddress dataAddress) { - var composite = new CompositeValidationRule<>( - List.of( - new EmptyValueValidationRule(BUCKET_NAME), - new EmptyValueValidationRule(REGION) - ) - ); - - return composite.apply(dataAddress.getProperties()); - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/data-plane/data-plane-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 9e7acd284aa..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Bayerische Motoren Werke Aktiengesellschaft (BMW AG) -# -# - -org.eclipse.edc.connector.dataplane.aws.s3.DataPlaneS3Extension diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3ExtensionTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3ExtensionTest.java deleted file mode 100644 index 45d678b3877..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3ExtensionTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.junit.extensions.EdcExtension; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -import java.util.UUID; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -@ExtendWith(EdcExtension.class) -class DataPlaneS3ExtensionTest { - - @Test - void shouldProvidePipelineServices(PipelineService pipelineService) { - var request = DataFlowRequest.Builder.newInstance() - .processId(UUID.randomUUID().toString()) - .sourceDataAddress(TestFunctions.s3DataAddressWithCredentials()) - .destinationDataAddress(TestFunctions.s3DataAddressWithCredentials()) - .build(); - - var result = pipelineService.validate(request); - - assertThat(result.succeeded()).isTrue(); - } -} \ No newline at end of file diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java deleted file mode 100644 index 7b08d957f04..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.aws.s3.testfixtures.AbstractS3Test; -import org.eclipse.edc.aws.s3.testfixtures.annotations.AwsS3IntegrationTest; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.core.internal.async.ByteArrayAsyncResponseTransformer; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; - -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; -import static org.mockito.Mockito.mock; - -@AwsS3IntegrationTest -public class S3DataPlaneIntegrationTest extends AbstractS3Test { - - private final String sourceBucketName = "source-" + UUID.randomUUID(); - private final String destinationBucketName = "destination-" + UUID.randomUUID(); - - @BeforeEach - void setup() { - createBucket(sourceBucketName); - createBucket(destinationBucketName); - } - - @AfterEach - void tearDown() { - deleteBucket(sourceBucketName); - deleteBucket(destinationBucketName); - } - - @Test - void shouldCopyFromSourceToSink() { - var body = UUID.randomUUID().toString(); - var key = UUID.randomUUID().toString(); - putStringOnBucket(sourceBucketName, key, body); - - var vault = mock(Vault.class); - var typeManager = new TypeManager(); - - var sinkFactory = new S3DataSinkFactory(clientProvider, Executors.newSingleThreadExecutor(), mock(Monitor.class), vault, typeManager); - var sourceFactory = new S3DataSourceFactory(clientProvider, vault, typeManager); - var sourceAddress = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .keyName(key) - .property(BUCKET_NAME, sourceBucketName) - .property(S3BucketSchema.REGION, REGION) - .property(ACCESS_KEY_ID, getCredentials().accessKeyId()) - .property(SECRET_ACCESS_KEY, getCredentials().secretAccessKey()) - .build(); - - var destinationAddress = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .keyName(key) - .property(BUCKET_NAME, destinationBucketName) - .property(S3BucketSchema.REGION, REGION) - .property(ACCESS_KEY_ID, getCredentials().accessKeyId()) - .property(SECRET_ACCESS_KEY, getCredentials().secretAccessKey()) - .build(); - - var request = DataFlowRequest.Builder.newInstance() - .id(UUID.randomUUID().toString()) - .processId(UUID.randomUUID().toString()) - .sourceDataAddress(sourceAddress) - .destinationDataAddress(destinationAddress) - .build(); - - var sink = sinkFactory.createSink(request); - var source = sourceFactory.createSource(request); - - var transferResult = sink.transfer(source); - - assertThat(transferResult).succeedsWithin(5, SECONDS); - assertThat(getObject(key)).succeedsWithin(5, SECONDS) - .extracting(ResponseBytes::response) - .extracting(GetObjectResponse::contentLength) - .extracting(Long::intValue) - .isEqualTo(body.length()); - assertThat(getObject(key + ".complete")).succeedsWithin(5, SECONDS); - } - - private CompletableFuture> getObject(String key) { - var getObjectRequest = GetObjectRequest.builder().bucket(destinationBucketName).key(key).build(); - return clientProvider.s3AsyncClient(REGION) - .getObject(getObjectRequest, new ByteArrayAsyncResponseTransformer<>()); - } - -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java deleted file mode 100644 index ece1b1f4617..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsSecretToken; -import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; - -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class S3DataSinkFactoryTest { - - private final AwsClientProvider clientProvider = mock(AwsClientProvider.class); - private final Vault vault = mock(Vault.class); - private final TypeManager typeManager = new TypeManager(); - private final S3DataSinkFactory factory = new S3DataSinkFactory(clientProvider, mock(ExecutorService.class), mock(Monitor.class), vault, typeManager); - - @Test - void canHandle_returnsTrueWhenExpectedType() { - var dataAddress = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE).build(); - - var result = factory.canHandle(createRequest(dataAddress)); - - assertThat(result).isTrue(); - } - - @Test - void canHandle_returnsFalseWhenUnexpectedType() { - var dataAddress = DataAddress.Builder.newInstance().type("any").build(); - - var result = factory.canHandle(createRequest(dataAddress)); - - assertThat(result).isFalse(); - } - - @Test - void validate_ShouldSucceedIfPropertiesAreValid() { - var destination = TestFunctions.s3DataAddressWithCredentials(); - var request = createRequest(destination); - - var result = factory.validateRequest(request); - - assertThat(result.succeeded()).isTrue(); - } - - @ParameterizedTest - @ArgumentsSource(InvalidInputs.class) - void validate_shouldFailIfMandatoryPropertiesAreMissing(String bucketName, String region, String accessKeyId, String secretAccessKey) { - var destination = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, bucketName) - .property(REGION, region) - .property(S3BucketSchema.ACCESS_KEY_ID, accessKeyId) - .property(S3BucketSchema.SECRET_ACCESS_KEY, secretAccessKey) - .build(); - - var request = createRequest(destination); - - var result = factory.validateRequest(request); - - assertThat(result.failed()).isTrue(); - } - - @Test - void createSink_shouldGetTheTemporarySecretTokenFromTheVault() { - var destination = TestFunctions.s3DataAddressWithCredentials(); - var temporaryKey = new AwsTemporarySecretToken("temporaryId", "temporarySecret", "temporaryToken", 10); - when(vault.resolveSecret(destination.getKeyName())).thenReturn(typeManager.writeValueAsString(temporaryKey)); - var request = createRequest(destination); - - var sink = factory.createSink(request); - - assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - verify(clientProvider).s3Client(eq(TestFunctions.VALID_REGION), isA(AwsTemporarySecretToken.class)); - } - - @Test - void createSink_shouldCreateDataSinkWithCredentialsInDataAddressIfTheresNoSecret() { - when(vault.resolveSecret(any())).thenReturn(null); - var destination = TestFunctions.s3DataAddressWithCredentials(); - var request = createRequest(destination); - - var sink = factory.createSink(request); - - assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - verify(clientProvider).s3Client(TestFunctions.VALID_REGION, new AwsSecretToken(TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY)); - } - - @Test - void createSink_shouldLetTheProviderGetTheCredentialsAsFallback() { - when(vault.resolveSecret(any())).thenReturn(null); - var destination = TestFunctions.s3DataAddressWithoutCredentials(); - var request = createRequest(destination); - - var sink = factory.createSink(request); - - assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - verify(clientProvider).s3Client(TestFunctions.VALID_REGION); - } - - @Test - void createSink_shouldThrowExceptionIfValidationFails() { - var destination = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .build(); - - var request = createRequest(destination); - - assertThatThrownBy(() -> factory.createSink(request)).isInstanceOf(EdcException.class); - } - - private static class InvalidInputs implements ArgumentsProvider { - - @Override - public Stream provideArguments(ExtensionContext context) throws Exception { - return Stream.of( - Arguments.of(TestFunctions.VALID_BUCKET_NAME, " ", TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY), - Arguments.of(" ", TestFunctions.VALID_REGION, TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY) - ); - } - } - - private DataFlowRequest createRequest(DataAddress destination) { - return DataFlowRequest.Builder.newInstance() - .id(UUID.randomUUID().toString()) - .processId(UUID.randomUUID().toString()) - .sourceDataAddress(DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE).build()) - .destinationDataAddress(destination) - .build(); - } -} \ No newline at end of file diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java deleted file mode 100644 index d886114d728..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2022 ZF Friedrichshafen AG - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * ZF Friedrichshafen AG - Initial implementation - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.connector.dataplane.spi.pipeline.InputStreamDataSource; -import org.eclipse.edc.spi.monitor.Monitor; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; -import software.amazon.awssdk.services.s3.model.UploadPartResponse; - -import java.io.ByteArrayInputStream; -import java.util.List; -import java.util.concurrent.Executors; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class S3DataSinkTest { - - private static final String BUCKET_NAME = "bucketName"; - private static final String KEY_NAME = "keyName"; - private static final String ETAG = "eTag"; - private static final int CHUNK_SIZE_BYTES = 50; - - private S3Client s3ClientMock; - private S3DataSink dataSink; - - private ArgumentCaptor completeMultipartUploadRequestCaptor; - - @BeforeEach - void setup() { - s3ClientMock = mock(S3Client.class); - completeMultipartUploadRequestCaptor = ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class); - - dataSink = S3DataSink.Builder.newInstance() - .bucketName(BUCKET_NAME) - .keyName(KEY_NAME) - .client(s3ClientMock) - .requestId(TestFunctions.createRequest(S3BucketSchema.TYPE).build().getId()) - .executorService(Executors.newFixedThreadPool(2)) - .monitor(mock(Monitor.class)) - .chunkSizeBytes(CHUNK_SIZE_BYTES) - .build(); - - when(s3ClientMock.createMultipartUpload(any(CreateMultipartUploadRequest.class))) - .thenReturn(CreateMultipartUploadResponse.builder().uploadId("uploadId").build()); - when(s3ClientMock.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) - .thenReturn(UploadPartResponse.builder().eTag(ETAG).build()); - } - - @Test - void transferParts_singlePart_succeeds() { - var result = dataSink.transferParts( - List.of(new InputStreamDataSource(KEY_NAME, new ByteArrayInputStream("content smaller than a chunk size".getBytes(UTF_8))))); - assertThat(result.succeeded()).isTrue(); - verify(s3ClientMock).completeMultipartUpload(completeMultipartUploadRequestCaptor.capture()); - - var completeMultipartUploadRequest = completeMultipartUploadRequestCaptor.getValue(); - assertThat(completeMultipartUploadRequest.bucket()).isEqualTo(BUCKET_NAME); - assertThat(completeMultipartUploadRequest.key()).isEqualTo(KEY_NAME); - assertThat(completeMultipartUploadRequest.multipartUpload().parts()).hasSize(1); - } - - @Test - void transferParts_multiPart_succeeds() { - var result = dataSink.transferParts( - List.of(new InputStreamDataSource(KEY_NAME, - new ByteArrayInputStream("content bigger than 50 bytes chunk size so that it gets chunked and uploaded as a multipart upload" - .getBytes(UTF_8))))); - assertThat(result.succeeded()).isTrue(); - verify(s3ClientMock).completeMultipartUpload(completeMultipartUploadRequestCaptor.capture()); - - var completeMultipartUploadRequest = completeMultipartUploadRequestCaptor.getValue(); - assertThat(completeMultipartUploadRequest.bucket()).isEqualTo(BUCKET_NAME); - assertThat(completeMultipartUploadRequest.key()).isEqualTo(KEY_NAME); - - assertThat(completeMultipartUploadRequest.multipartUpload().parts()).hasSize(2); - } - - @Test - void complete_succeedIfPutObjectSucceeds() { - when(s3ClientMock.putObject(any(PutObjectRequest.class), any(RequestBody.class))) - .thenReturn(PutObjectResponse.builder().build()); - - var result = dataSink.complete(); - - assertThat(result.succeeded()).isTrue(); - } - - @Test - void complete_failsIfPutObjectFails() { - when(s3ClientMock.putObject(any(PutObjectRequest.class), any(RequestBody.class))) - .thenThrow(SdkException.builder().message("an error").build()); - - var result = dataSink.complete(); - - assertThat(result.failed()).isTrue(); - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java deleted file mode 100644 index af951e54b33..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.AwsClientProvider; -import org.eclipse.edc.aws.s3.AwsSecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; - -import java.util.UUID; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_ACCESS_KEY_ID; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_BUCKET_NAME; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_REGION; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_SECRET_ACCESS_KEY; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.s3DataAddressWithCredentials; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.s3DataAddressWithoutCredentials; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class S3DataSourceFactoryTest { - - private final AwsClientProvider clientProvider = mock(AwsClientProvider.class); - private final TypeManager typeManager = new TypeManager(); - private final Vault vault = mock(Vault.class); - - private final S3DataSourceFactory factory = new S3DataSourceFactory(clientProvider, vault, typeManager); - - @Test - void canHandle_returnsTrueWhenExpectedType() { - var dataAddress = DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE).build(); - - var result = factory.canHandle(createRequest(dataAddress)); - - assertThat(result).isTrue(); - } - - @Test - void canHandle_returnsFalseWhenUnexpectedType() { - var dataAddress = DataAddress.Builder.newInstance().type("any").build(); - - var result = factory.canHandle(createRequest(dataAddress)); - - assertThat(result).isFalse(); - } - - @Test - void validate_shouldSucceedIfPropertiesAreValid() { - var source = s3DataAddressWithCredentials(); - var request = createRequest(source); - - var result = factory.validateRequest(request); - - assertThat(result.succeeded()).isTrue(); - } - - @ParameterizedTest - @ArgumentsSource(InvalidInputs.class) - void validate_shouldFailIfMandatoryPropertiesAreMissing(String bucketName, String region, String accessKeyId, String secretAccessKey) { - var source = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, bucketName) - .property(S3BucketSchema.REGION, region) - .property(S3BucketSchema.ACCESS_KEY_ID, accessKeyId) - .property(S3BucketSchema.SECRET_ACCESS_KEY, secretAccessKey) - .build(); - - var request = createRequest(source); - - var result = factory.validateRequest(request); - - assertThat(result.failed()).isTrue(); - } - - @Test - void createSource_shouldCreateDataSource() { - DataAddress source = s3DataAddressWithCredentials(); - var request = createRequest(source); - - var dataSource = factory.createSource(request); - - assertThat(dataSource).isNotNull().isInstanceOf(S3DataSource.class); - } - - @Test - void createSink_shouldLetTheProviderGetTheCredentialsIfNotProvidedByTheAddress() { - var destination = s3DataAddressWithoutCredentials(); - var request = createRequest(destination); - - var sink = factory.createSource(request); - - assertThat(sink).isNotNull().isInstanceOf(S3DataSource.class); - verify(clientProvider).s3Client(VALID_REGION); - } - - @Test - void createSource_shouldThrowExceptionIfValidationFails() { - var source = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .build(); - - var request = createRequest(source); - - assertThatThrownBy(() -> factory.createSource(request)).isInstanceOf(EdcException.class); - } - - @Test - void createSource_shouldGetTheSecretTokenFromTheVault() { - var source = TestFunctions.s3DataAddressWithCredentials(); - var temporaryKey = new AwsSecretToken("temporaryId", "temporarySecret"); - when(vault.resolveSecret(source.getKeyName())).thenReturn(typeManager.writeValueAsString(temporaryKey)); - var request = createRequest(source); - - var s3Source = factory.createSource(request); - - assertThat(s3Source).isNotNull().isInstanceOf(S3DataSource.class); - verify(clientProvider).s3Client(eq(TestFunctions.VALID_REGION), isA(AwsSecretToken.class)); - } - - private DataFlowRequest createRequest(DataAddress source) { - return DataFlowRequest.Builder.newInstance() - .id(UUID.randomUUID().toString()) - .processId(UUID.randomUUID().toString()) - .sourceDataAddress(source) - .destinationDataAddress(DataAddress.Builder.newInstance().type(S3BucketSchema.TYPE).build()) - .build(); - } - - private static class InvalidInputs implements ArgumentsProvider { - - @Override - public Stream provideArguments(ExtensionContext context) { - return Stream.of( - Arguments.of(VALID_BUCKET_NAME, " ", VALID_ACCESS_KEY_ID, VALID_SECRET_ACCESS_KEY), - Arguments.of(" ", VALID_REGION, VALID_ACCESS_KEY_ID, VALID_SECRET_ACCESS_KEY) - ); - } - } -} diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java deleted file mode 100644 index 1357e9aa1c9..00000000000 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - Initial implementation - * - */ - -package org.eclipse.edc.connector.dataplane.aws.s3; - -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; - -import java.util.UUID; - -public class TestFunctions { - - public static final String VALID_REGION = "validRegion"; - public static final String VALID_BUCKET_NAME = "validBucketName"; - public static final String VALID_ACCESS_KEY_ID = "validAccessKeyId"; - public static final String VALID_SECRET_ACCESS_KEY = "validSecretAccessKey"; - - public static DataAddress s3DataAddressWithCredentials() { - return DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .keyName("aKey") - .property(S3BucketSchema.BUCKET_NAME, VALID_BUCKET_NAME) - .property(S3BucketSchema.REGION, VALID_REGION) - .property(S3BucketSchema.ACCESS_KEY_ID, VALID_ACCESS_KEY_ID) - .property(S3BucketSchema.SECRET_ACCESS_KEY, VALID_SECRET_ACCESS_KEY) - .build(); - } - - public static DataAddress s3DataAddressWithoutCredentials() { - return DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .keyName("aKey") - .property(S3BucketSchema.BUCKET_NAME, VALID_BUCKET_NAME) - .property(S3BucketSchema.REGION, VALID_REGION) - .build(); - } - - public static DataFlowRequest.Builder createRequest(String type) { - return DataFlowRequest.Builder.newInstance() - .id(UUID.randomUUID().toString()) - .processId(UUID.randomUUID().toString()) - .sourceDataAddress(createDataAddress(type).build()) - .destinationDataAddress(createDataAddress(type).build()); - } - - public static DataAddress.Builder createDataAddress(String type) { - return DataAddress.Builder.newInstance().type(type); - } - -} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1cedfbe0a1d..f7ead0519fb 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,6 @@ apacheCommonsPool2 = "2.11.1" assertj = "3.24.2" atomikos = "6.0.0" awaitility = "4.2.0" -aws = "2.20.75" bouncyCastle-jdk18on = "1.73" cloudEvents = "2.5.0" @@ -96,13 +95,6 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } titaniumJsonLd = { module = "com.apicatalog:titanium-json-ld", version.ref = "titanium" } - -# AWS dependencies -aws-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws" } -aws-iam = { module = "software.amazon.awssdk:iam", version.ref = "aws" } -aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } -aws-secretsmanager = { module = "software.amazon.awssdk:secretsmanager", version.ref = "aws" } - # Google dependencies googlecloud-iam-admin = { module = "com.google.cloud:google-iam-admin", version.ref = "googleCloudIamAdmin" } googlecloud-iam-credentials = { module = "com.google.cloud:google-cloud-iamcredentials", version.ref = "googleCloudIamCredentials" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 619d00f0bd6..5762ee48875 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -85,8 +85,7 @@ include(":extensions:common:api:api-core") include(":extensions:common:api:api-observability") include(":extensions:common:auth:auth-basic") include(":extensions:common:auth:auth-tokenbased") -include(":extensions:common:aws:aws-s3-test") -include(":extensions:common:aws:aws-s3-core") + include(":extensions:common:configuration:configuration-filesystem") include(":extensions:common:events:events-cloud-http") @@ -116,7 +115,7 @@ include(":extensions:common:sql:sql-pool:sql-pool-apache-commons") include(":extensions:common:transaction") include(":extensions:common:transaction:transaction-atomikos") include(":extensions:common:transaction:transaction-local") -include(":extensions:common:vault:vault-aws") + include(":extensions:common:vault:vault-filesystem") include(":extensions:common:vault:vault-hashicorp") @@ -138,7 +137,6 @@ include(":extensions:control-plane:transfer:transfer-pull-http-receiver") include(":extensions:control-plane:transfer:transfer-pull-http-dynamic-receiver") include(":extensions:control-plane:provision:provision-gcs") include(":extensions:control-plane:provision:provision-http") -include(":extensions:control-plane:provision:provision-aws-s3") include(":extensions:control-plane:provision:provision-oauth2:provision-oauth2-core") include(":extensions:control-plane:provision:provision-oauth2:provision-oauth2") @@ -159,7 +157,6 @@ include(":extensions:data-plane:data-plane-client") include(":extensions:data-plane:data-plane-http") include(":extensions:data-plane:data-plane-http-oauth2") include(":extensions:data-plane:data-plane-http-oauth2-core") -include(":extensions:data-plane:data-plane-aws-s3") include(":extensions:data-plane:data-plane-google-storage") include(":extensions:data-plane:data-plane-integration-tests") include(":extensions:data-plane:store:sql:data-plane-store-sql")