From d455c6e7d2d1ad13dd4a71efc06922b348396020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 22 Sep 2023 16:45:03 +0200 Subject: [PATCH] AWS: Remove deprecated APIs for 1.4.0 (#8612) --- .../aws/TestAssumeRoleAwsClientFactory.java | 3 +- .../aws/TestDefaultAwsClientFactory.java | 7 +- .../apache/iceberg/aws/glue/GlueTestBase.java | 23 +- .../iceberg/aws/glue/TestGlueCatalogLock.java | 3 + .../aws/glue/TestGlueCatalogTable.java | 23 +- .../lakeformation/LakeFormationTestBase.java | 4 +- .../TestLakeFormationAwsClientFactory.java | 4 +- .../aws/s3/TestS3FileIOIntegration.java | 3 +- .../aws/ApacheHttpClientConfigurations.java | 20 +- .../aws/AssumeRoleAwsClientFactory.java | 31 +- .../iceberg/aws/AwsClientFactories.java | 51 +- .../iceberg/aws/AwsClientProperties.java | 19 +- .../org/apache/iceberg/aws/AwsProperties.java | 1377 +---------------- ...UrlConnectionHttpClientConfigurations.java | 4 +- .../apache/iceberg/aws/glue/GlueCatalog.java | 25 +- .../LakeFormationAwsClientFactory.java | 10 +- .../iceberg/aws/s3/S3FileIOProperties.java | 2 +- .../iceberg/aws/TestAwsClientFactories.java | 6 +- .../apache/iceberg/aws/TestAwsProperties.java | 279 ---- .../aws/TestHttpClientConfigurations.java | 30 +- .../iceberg/aws/TestHttpClientProperties.java | 82 + .../iceberg/aws/TestS3FileIOProperties.java | 252 +++ .../iceberg/aws/glue/TestGlueCatalog.java | 21 +- 23 files changed, 545 insertions(+), 1734 deletions(-) create mode 100644 aws/src/test/java/org/apache/iceberg/aws/TestHttpClientProperties.java create mode 100644 aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java diff --git a/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java b/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java index f78ec5d8acd6..4f2b3b8a1f79 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java @@ -82,7 +82,8 @@ public void before() { assumeRoleProperties = Maps.newHashMap(); assumeRoleProperties.put( AwsProperties.CLIENT_FACTORY, AssumeRoleAwsClientFactory.class.getName()); - assumeRoleProperties.put(AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_APACHE); + assumeRoleProperties.put( + HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_APACHE); assumeRoleProperties.put( AwsProperties.CLIENT_ASSUME_ROLE_REGION, AwsIntegTestUtil.testRegion()); assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN, response.role().arn()); diff --git a/aws/src/integration/java/org/apache/iceberg/aws/TestDefaultAwsClientFactory.java b/aws/src/integration/java/org/apache/iceberg/aws/TestDefaultAwsClientFactory.java index fd3a19ffe478..b69b24583f4f 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/TestDefaultAwsClientFactory.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/TestDefaultAwsClientFactory.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.junit.Test; import software.amazon.awssdk.core.exception.SdkClientException; @@ -48,7 +49,7 @@ public void testGlueEndpointOverride() { @Test public void testS3FileIoEndpointOverride() { Map properties = Maps.newHashMap(); - properties.put(AwsProperties.S3FILEIO_ENDPOINT, "https://unknown:1234"); + properties.put(S3FileIOProperties.ENDPOINT, "https://unknown:1234"); AwsClientFactory factory = AwsClientFactories.from(properties); S3Client s3Client = factory.s3(); AssertHelpers.assertThrowsCause( @@ -61,8 +62,8 @@ public void testS3FileIoEndpointOverride() { @Test public void testS3FileIoCredentialsOverride() { Map properties = Maps.newHashMap(); - properties.put(AwsProperties.S3FILEIO_ACCESS_KEY_ID, "unknown"); - properties.put(AwsProperties.S3FILEIO_SECRET_ACCESS_KEY, "unknown"); + properties.put(S3FileIOProperties.ACCESS_KEY_ID, "unknown"); + properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, "unknown"); AwsClientFactory factory = AwsClientFactories.from(properties); S3Client s3Client = factory.s3(); AssertHelpers.assertThrows( diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java index 9be6e1e7ced4..d900f133bd4b 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java @@ -28,6 +28,7 @@ import org.apache.iceberg.aws.AwsClientFactory; import org.apache.iceberg.aws.AwsIntegTestUtil; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -77,15 +78,29 @@ public class GlueTestBase { @BeforeClass public static void beforeClass() { glueCatalog = new GlueCatalog(); - AwsProperties properties = new AwsProperties(); - properties.setS3FileIoDeleteBatchSize(10); - glueCatalog.initialize(catalogName, testBucketPath, properties, glue, null, ImmutableMap.of()); + AwsProperties awsProperties = new AwsProperties(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(); + s3FileIOProperties.setDeleteBatchSize(10); + glueCatalog.initialize( + catalogName, + testBucketPath, + awsProperties, + s3FileIOProperties, + glue, + null, + ImmutableMap.of()); glueCatalogWithSkipNameValidation = new GlueCatalog(); AwsProperties propertiesSkipNameValidation = new AwsProperties(); propertiesSkipNameValidation.setGlueCatalogSkipNameValidation(true); glueCatalogWithSkipNameValidation.initialize( - catalogName, testBucketPath, propertiesSkipNameValidation, glue, null, ImmutableMap.of()); + catalogName, + testBucketPath, + propertiesSkipNameValidation, + new S3FileIOProperties(), + glue, + null, + ImmutableMap.of()); } @AfterClass diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogLock.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogLock.java index 68ae8266c199..4c48a277192d 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogLock.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogLock.java @@ -33,6 +33,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.aws.AwsProperties; import org.apache.iceberg.aws.dynamodb.DynamoDbLockManager; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors; @@ -56,11 +57,13 @@ public static void beforeClass() { lockTableName = getRandomName(); glueCatalog = new GlueCatalog(); AwsProperties awsProperties = new AwsProperties(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(); dynamo = clientFactory.dynamo(); glueCatalog.initialize( catalogName, testBucketPath, awsProperties, + s3FileIOProperties, glue, new DynamoDbLockManager(dynamo, lockTableName), ImmutableMap.of()); diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java index 7a05f1acbcff..c51bc2a86509 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java @@ -38,6 +38,7 @@ import org.apache.iceberg.TableOperations; import org.apache.iceberg.Transaction; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; @@ -139,6 +140,7 @@ public void testCreateAndLoadTableWithoutWarehouseLocation() { catalogName, null, new AwsProperties(), + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -380,6 +382,7 @@ public void testCommitTableSkipArchive() { catalogName, testBucketPath, properties, + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -596,11 +599,17 @@ public void testTableLevelS3Tags() { String testBucketPath = "s3://" + testBucketName + "/" + testPathPrefix; Map properties = ImmutableMap.of( - AwsProperties.S3_WRITE_TABLE_TAG_ENABLED, + S3FileIOProperties.WRITE_TABLE_TAG_ENABLED, "true", - AwsProperties.S3_WRITE_NAMESPACE_TAG_ENABLED, + S3FileIOProperties.WRITE_NAMESPACE_TAG_ENABLED, "true"); - glueCatalog.initialize(catalogName, testBucketPath, new AwsProperties(properties), glue, null); + glueCatalog.initialize( + catalogName, + testBucketPath, + new AwsProperties(properties), + new S3FileIOProperties(properties), + glue, + null); String namespace = createNamespace(); String tableName = getRandomName(); createTable(namespace, tableName); @@ -617,9 +626,9 @@ public void testTableLevelS3Tags() { .tagSet(); Map tagMap = tags.stream().collect(Collectors.toMap(Tag::key, Tag::value)); - Assert.assertTrue(tagMap.containsKey(AwsProperties.S3_TAG_ICEBERG_TABLE)); - Assert.assertEquals(tableName, tagMap.get(AwsProperties.S3_TAG_ICEBERG_TABLE)); - Assert.assertTrue(tagMap.containsKey(AwsProperties.S3_TAG_ICEBERG_NAMESPACE)); - Assert.assertEquals(namespace, tagMap.get(AwsProperties.S3_TAG_ICEBERG_NAMESPACE)); + Assert.assertTrue(tagMap.containsKey(S3FileIOProperties.S3_TAG_ICEBERG_TABLE)); + Assert.assertEquals(tableName, tagMap.get(S3FileIOProperties.S3_TAG_ICEBERG_TABLE)); + Assert.assertTrue(tagMap.containsKey(S3FileIOProperties.S3_TAG_ICEBERG_NAMESPACE)); + Assert.assertEquals(namespace, tagMap.get(S3FileIOProperties.S3_TAG_ICEBERG_NAMESPACE)); } } diff --git a/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/LakeFormationTestBase.java b/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/LakeFormationTestBase.java index bce227ec631f..431ee91bd22f 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/LakeFormationTestBase.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/LakeFormationTestBase.java @@ -30,6 +30,7 @@ import org.apache.iceberg.aws.AssumeRoleAwsClientFactory; import org.apache.iceberg.aws.AwsIntegTestUtil; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.HttpClientProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -227,7 +228,8 @@ public static void beforeClass() throws Exception { AwsProperties.CLIENT_ASSUME_ROLE_REGION, AwsIntegTestUtil.testRegion()); assumeRoleProperties.put(AwsProperties.GLUE_LAKEFORMATION_ENABLED, "true"); assumeRoleProperties.put(AwsProperties.GLUE_ACCOUNT_ID, AwsIntegTestUtil.testAccountId()); - assumeRoleProperties.put(AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_APACHE); + assumeRoleProperties.put( + HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_APACHE); assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN, lfPrivilegedRoleArn); assumeRoleProperties.put( AwsProperties.CLIENT_ASSUME_ROLE_TAGS_PREFIX diff --git a/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/TestLakeFormationAwsClientFactory.java b/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/TestLakeFormationAwsClientFactory.java index 13c135ed19b3..9c59ba1aa19e 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/TestLakeFormationAwsClientFactory.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/lakeformation/TestLakeFormationAwsClientFactory.java @@ -22,6 +22,7 @@ import java.util.UUID; import org.apache.iceberg.aws.AwsIntegTestUtil; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.HttpClientProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -82,7 +83,8 @@ public void before() { assumeRoleProperties = Maps.newHashMap(); assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_REGION, "us-east-1"); assumeRoleProperties.put(AwsProperties.GLUE_LAKEFORMATION_ENABLED, "true"); - assumeRoleProperties.put(AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_APACHE); + assumeRoleProperties.put( + HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_APACHE); assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN, response.role().arn()); assumeRoleProperties.put( AwsProperties.CLIENT_ASSUME_ROLE_TAGS_PREFIX diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java index c12c3f58e0de..d244f5f374c0 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java @@ -37,7 +37,6 @@ import org.apache.iceberg.aws.AwsClientFactories; import org.apache.iceberg.aws.AwsClientFactory; import org.apache.iceberg.aws.AwsIntegTestUtil; -import org.apache.iceberg.aws.AwsProperties; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -377,7 +376,7 @@ public void testDeleteFilesMultipleBatchesWithAccessPoints() throws Exception { @Test public void testDeleteFilesMultipleBatchesWithCrossRegionAccessPoints() throws Exception { - clientFactory.initialize(ImmutableMap.of(AwsProperties.S3_USE_ARN_REGION_ENABLED, "true")); + clientFactory.initialize(ImmutableMap.of(S3FileIOProperties.USE_ARN_REGION_ENABLED, "true")); S3FileIO s3FileIO = new S3FileIO(clientFactory::s3, getDeletionTestProperties()); s3FileIO.initialize( ImmutableMap.of( diff --git a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java index e6cb1610d42a..04f5b911ff58 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java +++ b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java @@ -47,33 +47,31 @@ public void configureHttpClientBuilder(T awsCli private void initialize(Map httpClientProperties) { this.connectionTimeoutMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_TIMEOUT_MS); + httpClientProperties, HttpClientProperties.APACHE_CONNECTION_TIMEOUT_MS); this.socketTimeoutMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_SOCKET_TIMEOUT_MS); + httpClientProperties, HttpClientProperties.APACHE_SOCKET_TIMEOUT_MS); this.acquisitionTimeoutMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, - AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS); + httpClientProperties, HttpClientProperties.APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS); this.connectionMaxIdleTimeMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_MAX_IDLE_TIME_MS); + httpClientProperties, HttpClientProperties.APACHE_CONNECTION_MAX_IDLE_TIME_MS); this.connectionTimeToLiveMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_TIME_TO_LIVE_MS); + httpClientProperties, HttpClientProperties.APACHE_CONNECTION_TIME_TO_LIVE_MS); this.expectContinueEnabled = PropertyUtil.propertyAsNullableBoolean( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_EXPECT_CONTINUE_ENABLED); + httpClientProperties, HttpClientProperties.APACHE_EXPECT_CONTINUE_ENABLED); this.maxConnections = PropertyUtil.propertyAsNullableInt( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_MAX_CONNECTIONS); + httpClientProperties, HttpClientProperties.APACHE_MAX_CONNECTIONS); this.tcpKeepAliveEnabled = PropertyUtil.propertyAsNullableBoolean( - httpClientProperties, AwsProperties.HTTP_CLIENT_APACHE_TCP_KEEP_ALIVE_ENABLED); + httpClientProperties, HttpClientProperties.APACHE_TCP_KEEP_ALIVE_ENABLED); this.useIdleConnectionReaperEnabled = PropertyUtil.propertyAsNullableBoolean( - httpClientProperties, - AwsProperties.HTTP_CLIENT_APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED); + httpClientProperties, HttpClientProperties.APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED); } @VisibleForTesting diff --git a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java index c7e93879921a..d9ea511f9bcd 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java +++ b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.UUID; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder; @@ -34,16 +35,18 @@ public class AssumeRoleAwsClientFactory implements AwsClientFactory { private AwsProperties awsProperties; + private HttpClientProperties httpClientProperties; + private S3FileIOProperties s3FileIOProperties; private String roleSessionName; @Override public S3Client s3() { return S3Client.builder() .applyMutation(this::applyAssumeRoleConfigurations) - .applyMutation(awsProperties::applyHttpClientConfigurations) - .applyMutation(awsProperties::applyS3EndpointConfigurations) - .applyMutation(awsProperties::applyS3ServiceConfigurations) - .applyMutation(awsProperties::applyS3SignerConfiguration) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) + .applyMutation(s3FileIOProperties::applyEndpointConfigurations) + .applyMutation(s3FileIOProperties::applyServiceConfigurations) + .applyMutation(s3FileIOProperties::applySignerConfiguration) .build(); } @@ -51,7 +54,7 @@ public S3Client s3() { public GlueClient glue() { return GlueClient.builder() .applyMutation(this::applyAssumeRoleConfigurations) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .build(); } @@ -59,7 +62,7 @@ public GlueClient glue() { public KmsClient kms() { return KmsClient.builder() .applyMutation(this::applyAssumeRoleConfigurations) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .build(); } @@ -67,7 +70,7 @@ public KmsClient kms() { public DynamoDbClient dynamo() { return DynamoDbClient.builder() .applyMutation(this::applyAssumeRoleConfigurations) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .applyMutation(awsProperties::applyDynamoDbEndpointConfigurations) .build(); } @@ -75,6 +78,8 @@ public DynamoDbClient dynamo() { @Override public void initialize(Map properties) { this.awsProperties = new AwsProperties(properties); + this.s3FileIOProperties = new S3FileIOProperties(properties); + this.httpClientProperties = new HttpClientProperties(properties); this.roleSessionName = genSessionName(); Preconditions.checkNotNull( awsProperties.clientAssumeRoleArn(), @@ -112,8 +117,18 @@ protected AwsProperties awsProperties() { return awsProperties; } + protected HttpClientProperties httpClientProperties() { + return httpClientProperties; + } + + protected S3FileIOProperties s3FileIOProperties() { + return s3FileIOProperties; + } + private StsClient sts() { - return StsClient.builder().applyMutation(awsProperties::applyHttpClientConfigurations).build(); + return StsClient.builder() + .applyMutation(httpClientProperties::applyHttpClientConfigurations) + .build(); } private String genSessionName() { diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java index d62b01ed938c..1ec982225449 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java +++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java @@ -20,6 +20,7 @@ import java.net.URI; import java.util.Map; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.common.DynConstructors; import org.apache.iceberg.relocated.com.google.common.base.Strings; import org.apache.iceberg.util.PropertyUtil; @@ -90,28 +91,35 @@ private static AwsClientFactory loadClientFactory(String impl, Map s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, b)) + .applyMutation(s3FileIOProperties::applySignerConfiguration) .build(); } @Override public GlueClient glue() { return GlueClient.builder() - .applyMutation(awsProperties::applyClientRegionConfiguration) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .applyMutation(awsProperties::applyGlueEndpointConfigurations) .applyMutation(awsProperties::applyClientCredentialConfigurations) .build(); @@ -120,8 +128,8 @@ public GlueClient glue() { @Override public KmsClient kms() { return KmsClient.builder() - .applyMutation(awsProperties::applyClientRegionConfiguration) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .applyMutation(awsProperties::applyClientCredentialConfigurations) .build(); } @@ -129,8 +137,8 @@ public KmsClient kms() { @Override public DynamoDbClient dynamo() { return DynamoDbClient.builder() - .applyMutation(awsProperties::applyClientRegionConfiguration) - .applyMutation(awsProperties::applyHttpClientConfigurations) + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) .applyMutation(awsProperties::applyClientCredentialConfigurations) .applyMutation(awsProperties::applyDynamoDbEndpointConfigurations) .build(); @@ -139,6 +147,9 @@ public DynamoDbClient dynamo() { @Override public void initialize(Map properties) { this.awsProperties = new AwsProperties(properties); + this.awsClientProperties = new AwsClientProperties(properties); + this.s3FileIOProperties = new S3FileIOProperties(properties); + this.httpClientProperties = new HttpClientProperties(properties); } } @@ -146,19 +157,19 @@ public void initialize(Map properties) { * Build a httpClientBuilder object * * @deprecated Not for public use. To configure the httpClient for a client, please use {@link - * AwsProperties#applyHttpClientConfigurations(AwsSyncClientBuilder)}. It will be removed in - * 2.0.0 + * HttpClientProperties#applyHttpClientConfigurations(AwsSyncClientBuilder)}. It will be + * removed in 2.0.0 */ @Deprecated public static SdkHttpClient.Builder configureHttpClientBuilder(String httpClientType) { String clientType = httpClientType; if (Strings.isNullOrEmpty(clientType)) { - clientType = AwsProperties.HTTP_CLIENT_TYPE_DEFAULT; + clientType = HttpClientProperties.CLIENT_TYPE_DEFAULT; } switch (clientType) { - case AwsProperties.HTTP_CLIENT_TYPE_URLCONNECTION: + case HttpClientProperties.CLIENT_TYPE_URLCONNECTION: return UrlConnectionHttpClient.builder(); - case AwsProperties.HTTP_CLIENT_TYPE_APACHE: + case HttpClientProperties.CLIENT_TYPE_APACHE: return ApacheHttpClient.builder(); default: throw new IllegalArgumentException("Unrecognized HTTP client type " + httpClientType); @@ -169,7 +180,7 @@ public static SdkHttpClient.Builder configureHttpClientBuilder(String httpClient * Configure the endpoint setting for a client * * @deprecated Not for public use. To configure the endpoint for a client, please use {@link - * AwsProperties#applyS3EndpointConfigurations(S3ClientBuilder)}, {@link + * S3FileIOProperties#applyEndpointConfigurations(S3ClientBuilder)}, {@link * AwsProperties#applyGlueEndpointConfigurations(GlueClientBuilder)}, or {@link * AwsProperties#applyDynamoDbEndpointConfigurations(DynamoDbClientBuilder)} accordingly. It * will be removed in 2.0.0 @@ -200,8 +211,8 @@ public static S3Configuration s3Configuration( * Build an AwsBasicCredential object * * @deprecated Not for public use. To configure the credentials for a s3 client, please use {@link - * AwsProperties#applyS3CredentialConfigurations(S3ClientBuilder)} in AwsProperties. It will - * be removed in 2.0.0. + * S3FileIOProperties#applyCredentialConfigurations(AwsClientProperties, S3ClientBuilder)} in + * AwsProperties. It will be removed in 2.0.0. */ @Deprecated static AwsCredentialsProvider credentialsProvider( diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java index 33a22c662162..d85764c499cb 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java @@ -57,7 +57,7 @@ public class AwsClientProperties implements Serializable { * classes to pass provider-specific properties. Each property consists of a key name and an * associated value. */ - private static final String CLIENT_CREDENTIAL_PROVIDER_PREFIX = "client.credentials-provider."; + protected static final String CLIENT_CREDENTIAL_PROVIDER_PREFIX = "client.credentials-provider."; /** * Used by {@link org.apache.iceberg.aws.AwsClientFactories.DefaultAwsClientFactory} and also @@ -97,7 +97,7 @@ public void setClientRegion(String clientRegion) { *

Sample usage: * * 

-   *     S3Client.builder().applyMutation(awsProperties::applyClientRegionConfiguration)
+   *     S3Client.builder().applyMutation(awsClientProperties::applyClientRegionConfiguration)
    *
*/ public void applyClientRegionConfiguration(T builder) { @@ -106,6 +106,21 @@ public void applyClientRegionConfiguration(T builde } } + /** + * Configure the credential provider for AWS clients. + * + *

Sample usage: + * + * 

+   *     DynamoDbClient.builder().applyMutation(awsClientProperties::applyClientCredentialConfigurations)
+   *
+ */ + public void applyClientCredentialConfigurations(T builder) { + if (!Strings.isNullOrEmpty(this.clientCredentialsProvider)) { + builder.credentialsProvider(credentialsProvider(this.clientCredentialsProvider)); + } + } + /** * Returns a credentials provider instance. If params were set, we return a new credentials * instance. If none of the params are set, we try to dynamically load the provided credentials diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java b/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java index 9266c83f1bd4..7bc2ba70dc4f 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java @@ -25,13 +25,9 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.aws.dynamodb.DynamoDbCatalog; -import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.aws.lakeformation.LakeFormationAwsClientFactory; -import org.apache.iceberg.aws.s3.S3FileIO; -import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; import org.apache.iceberg.common.DynClasses; import org.apache.iceberg.common.DynMethods; -import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.base.Strings; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -40,99 +36,22 @@ import org.apache.iceberg.util.SerializableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; -import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder; import software.amazon.awssdk.core.client.builder.SdkClientBuilder; -import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; import software.amazon.awssdk.services.glue.GlueClientBuilder; -import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.ObjectCannedACL; -import software.amazon.awssdk.services.s3.model.Tag; public class AwsProperties implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(AwsProperties.class); - /** - * Type of S3 Server side encryption used, default to {@link - * AwsProperties#S3FILEIO_SSE_TYPE_NONE}. - * - *

For more details: https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_TYPE = "s3.sse.type"; - - /** - * No server side encryption. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_TYPE_NONE = "none"; - - /** - * S3 SSE-KMS encryption. - * - *

For more details: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_TYPE_KMS = "kms"; - - /** - * S3 SSE-S3 encryption. - * - *

For more details: - * https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_TYPE_S3 = "s3"; - - /** - * S3 SSE-C encryption. - * - *

For more details: - * https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_TYPE_CUSTOM = "custom"; - - /** - * If S3 encryption type is SSE-KMS, input is a KMS Key ID or ARN. In case this property is not - * set, default key "aws/s3" is used. If encryption type is SSE-C, input is a custom base-64 - * AES256 symmetric key. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_KEY = "s3.sse.key"; - - /** - * If S3 encryption type is SSE-C, input is the base-64 MD5 digest of the secret key. This MD5 - * must be explicitly passed in by the caller to ensure key integrity. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SSE_MD5 = "s3.sse.md5"; - /** * The ID of the Glue Data Catalog where the tables reside. If none is provided, Glue * automatically uses the caller's AWS account ID by default. @@ -187,214 +106,6 @@ public class AwsProperties implements Serializable { */ public static final String GLUE_CATALOG_ENDPOINT = "glue.endpoint"; - /** - * Number of threads to use for uploading parts to S3 (shared pool across all output streams), - * default to {@link Runtime#availableProcessors()} - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public static final String S3FILEIO_MULTIPART_UPLOAD_THREADS = "s3.multipart.num-threads"; - - /** - * The size of a single part for multipart upload requests in bytes (default: 32MB). based on S3 - * requirement, the part size must be at least 5MB. Too ensure performance of the reader and - * writer, the part size must be less than 2GB. - * - *

For more details, see https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_MULTIPART_SIZE = "s3.multipart.part-size-bytes"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final int S3FILEIO_MULTIPART_SIZE_DEFAULT = 32 * 1024 * 1024; - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final int S3FILEIO_MULTIPART_SIZE_MIN = 5 * 1024 * 1024; - - /** - * The threshold expressed as a factor times the multipart size at which to switch from uploading - * using a single put object request to uploading using multipart upload (default: 1.5). - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public static final String S3FILEIO_MULTIPART_THRESHOLD_FACTOR = "s3.multipart.threshold"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final double S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT = 1.5; - - /** - * Location to put staging files for upload to S3, default to temp directory set in - * java.io.tmpdir. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_STAGING_DIRECTORY = "s3.staging-dir"; - - /** - * Used to configure canned access control list (ACL) for S3 client to use during write. If not - * set, ACL will not be set for requests. - * - *

The input must be one of {@link software.amazon.awssdk.services.s3.model.ObjectCannedACL}, - * such as 'public-read-write' For more details: - * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_ACL = "s3.acl"; - - /** - * Configure an alternative endpoint of the S3 service for S3FileIO to access. - * - *

This could be used to use S3FileIO with any s3-compatible object storage service that has a - * different endpoint, or access a private S3 endpoint in a virtual private cloud. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_ENDPOINT = "s3.endpoint"; - - /** - * If set {@code true}, requests to S3FileIO will use Path-Style, otherwise, Virtual Hosted-Style - * will be used. - * - *

For more details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_PATH_STYLE_ACCESS = "s3.path-style-access"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3FILEIO_PATH_STYLE_ACCESS_DEFAULT = false; - - /** - * Configure the static access key ID used to access S3FileIO. - * - *

When set, the default client factory will use the basic or session credentials provided - * instead of reading the default credential chain to create S3 access credentials. If {@link - * #S3FILEIO_SESSION_TOKEN} is set, session credential is used, otherwise basic credential is - * used. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_ACCESS_KEY_ID = "s3.access-key-id"; - - /** - * Configure the static secret access key used to access S3FileIO. - * - *

When set, the default client factory will use the basic or session credentials provided - * instead of reading the default credential chain to create S3 access credentials. If {@link - * #S3FILEIO_SESSION_TOKEN} is set, session credential is used, otherwise basic credential is - * used. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SECRET_ACCESS_KEY = "s3.secret-access-key"; - - /** - * Configure the static session token used to access S3FileIO. - * - *

When set, the default client factory will use the session credentials provided instead of - * reading the default credential chain to create S3 access credentials. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_SESSION_TOKEN = "s3.session-token"; - - /** - * Enable to make S3FileIO, to make cross-region call to the region specified in the ARN of an - * access point. - * - *

By default, attempting to use an access point in a different region will throw an exception. - * When enabled, this property allows using access points in other regions. - * - *

For more details see: - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#useArnRegionEnabled-- - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_USE_ARN_REGION_ENABLED = "s3.use-arn-region-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_USE_ARN_REGION_ENABLED_DEFAULT = false; - - /** - * Enables eTag checks for S3 PUT and MULTIPART upload requests. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_CHECKSUM_ENABLED = "s3.checksum-enabled"; - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_CHECKSUM_ENABLED_DEFAULT = false; - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_REMOTE_SIGNING_ENABLED = "s3.remote-signing-enabled"; - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_REMOTE_SIGNING_ENABLED_DEFAULT = false; - - /** - * Configure the batch size used when deleting multiple files from a given S3 bucket - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_DELETE_BATCH_SIZE = "s3.delete.batch-size"; - - /** - * Default batch size used when deleting files. - * - *

Refer to https://github.com/apache/hadoop/commit/56dee667707926f3796c7757be1a133a362f05c9 - * for more details on why this value was chosen. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final int S3FILEIO_DELETE_BATCH_SIZE_DEFAULT = 250; - - /** - * Max possible batch size for deletion. Currently, a max of 1000 keys can be deleted in one - * batch. https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final int S3FILEIO_DELETE_BATCH_SIZE_MAX = 1000; - /** Configure an alternative endpoint of the DynamoDB service to access. */ public static final String DYNAMODB_ENDPOINT = "dynamodb.endpoint"; @@ -456,436 +167,6 @@ public class AwsProperties implements Serializable { */ public static final String CLIENT_ASSUME_ROLE_SESSION_NAME = "client.assume-role.session-name"; - /** - * Configure the AWS credentials provider used to create AWS clients. A fully qualified concrete - * class with package that implements the {@link AwsCredentialsProvider} interface is required. - * - *

Additionally, the implementation class must also have a create() or create(Map) method - * implemented, which returns an instance of the class that provides aws credentials provider. - * - *

Example: - * client.credentials-provider=software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider - * - *

When set, the default client factory {@link - * org.apache.iceberg.aws.AwsClientFactories.DefaultAwsClientFactory} and also other client - * factory classes will use this provider to get AWS credentials provided instead of reading the - * default credential chain to get AWS access credentials. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.AwsClientProperties} - * instead - */ - @Deprecated - public static final String CLIENT_CREDENTIALS_PROVIDER = "client.credentials-provider"; - - /** - * Used by the client.credentials-provider configured value that will be used by {@link - * org.apache.iceberg.aws.AwsClientFactories.DefaultAwsClientFactory} and also other client - * factory classes to pass provider-specific properties. Each property consists of a key name and - * an associated value. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.AwsClientProperties} - * instead - */ - @Deprecated - private static final String CLIENT_CREDENTIAL_PROVIDER_PREFIX = "client.credentials-provider."; - - /** - * Used by {@link org.apache.iceberg.aws.AwsClientFactories.DefaultAwsClientFactory} and also - * other client factory classes. If set, all AWS clients except STS client will use the given - * region instead of the default region chain. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.AwsClientProperties} - * instead - */ - @Deprecated public static final String CLIENT_REGION = "client.region"; - - /** - * The type of {@link software.amazon.awssdk.http.SdkHttpClient} implementation used by {@link - * AwsClientFactory} If set, all AWS clients will use this specified HTTP client. If not set, - * {@link #HTTP_CLIENT_TYPE_DEFAULT} will be used. For specific types supported, see - * HTTP_CLIENT_TYPE_* defined below. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated public static final String HTTP_CLIENT_TYPE = "http-client.type"; - - /** - * If this is set under {@link #HTTP_CLIENT_TYPE}, {@link - * software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient} will be used as the HTTP - * Client in {@link AwsClientFactory} - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated public static final String HTTP_CLIENT_TYPE_URLCONNECTION = "urlconnection"; - - /** - * If this is set under {@link #HTTP_CLIENT_TYPE}, {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient} will be used as the HTTP Client in {@link - * AwsClientFactory} - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated public static final String HTTP_CLIENT_TYPE_APACHE = "apache"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated public static final String HTTP_CLIENT_TYPE_DEFAULT = HTTP_CLIENT_TYPE_APACHE; - - /** - * Used to configure the connection timeout in milliseconds for {@link - * software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder}. This flag only - * works when {@link #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_URLCONNECTION} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/UrlConnectionHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS = - "http-client.urlconnection.connection-timeout-ms"; - - /** - * Used to configure the socket timeout in milliseconds for {@link - * software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder}. This flag only - * works when {@link #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_URLCONNECTION} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/UrlConnectionHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS = - "http-client.urlconnection.socket-timeout-ms"; - - /** - * Used to configure the connection timeout in milliseconds for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_CONNECTION_TIMEOUT_MS = - "http-client.apache.connection-timeout-ms"; - - /** - * Used to configure the socket timeout in milliseconds for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_SOCKET_TIMEOUT_MS = - "http-client.apache.socket-timeout-ms"; - - /** - * Used to configure the connection acquisition timeout in milliseconds for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS = - "http-client.apache.connection-acquisition-timeout-ms"; - - /** - * Used to configure the connection max idle time in milliseconds for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_CONNECTION_MAX_IDLE_TIME_MS = - "http-client.apache.connection-max-idle-time-ms"; - - /** - * Used to configure the connection time to live in milliseconds for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_CONNECTION_TIME_TO_LIVE_MS = - "http-client.apache.connection-time-to-live-ms"; - - /** - * Used to configure whether to enable the expect continue setting for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

In default, this is disabled. - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_EXPECT_CONTINUE_ENABLED = - "http-client.apache.expect-continue-enabled"; - - /** - * Used to configure the max connections number for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE} - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_MAX_CONNECTIONS = - "http-client.apache.max-connections"; - - /** - * Used to configure whether to enable the tcp keep alive setting for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE}. - * - *

In default, this is disabled. - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_TCP_KEEP_ALIVE_ENABLED = - "http-client.apache.tcp-keep-alive-enabled"; - - /** - * Used to configure whether to use idle connection reaper for {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}. This flag only works when {@link - * #HTTP_CLIENT_TYPE} is set to {@link #HTTP_CLIENT_TYPE_APACHE}. - * - *

In default, this is enabled. - * - *

For more details, see - * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public static final String HTTP_CLIENT_APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED = - "http-client.apache.use-idle-connection-reaper-enabled"; - /** - * Used by {@link S3FileIO} to tag objects when writing. To set, we can pass a catalog property. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html - * - *

Example: s3.write.tags.my_key=my_val - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_WRITE_TAGS_PREFIX = "s3.write.tags."; - - /** - * Used by {@link GlueCatalog} to tag objects when writing. To set, we can pass a catalog - * property. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html - * - *

Example: s3.write.table-tag-enabled=true - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_WRITE_TABLE_TAG_ENABLED = "s3.write.table-tag-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_WRITE_TABLE_TAG_ENABLED_DEFAULT = false; - - /** - * Used by {@link GlueCatalog} to tag objects when writing. To set, we can pass a catalog - * property. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html - * - *

Example: s3.write.namespace-tag-enabled=true - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public static final String S3_WRITE_NAMESPACE_TAG_ENABLED = "s3.write.namespace-tag-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_WRITE_NAMESPACE_TAG_ENABLED_DEFAULT = false; - - /** - * Tag name that will be used by {@link #S3_WRITE_TAGS_PREFIX} when {@link - * #S3_WRITE_TABLE_TAG_ENABLED} is enabled - * - *

Example: iceberg.table=tableName - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_TAG_ICEBERG_TABLE = "iceberg.table"; - - /** - * Tag name that will be used by {@link #S3_WRITE_TAGS_PREFIX} when {@link - * #S3_WRITE_NAMESPACE_TAG_ENABLED} is enabled - * - *

Example: iceberg.namespace=namespaceName - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_TAG_ICEBERG_NAMESPACE = "iceberg.namespace"; - - /** - * Used by {@link S3FileIO} to tag objects when deleting. When this config is set, objects are - * tagged with the configured key-value pairs before deletion. This is considered a soft-delete, - * because users are able to configure tag-based object lifecycle policy at bucket level to - * transition objects to different tiers. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lifecycle-mgmt.html - * - *

Example: s3.delete.tags.my_key=my_val - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_DELETE_TAGS_PREFIX = "s3.delete.tags."; - - /** - * Number of threads to use for adding delete tags to S3 objects, default to {@link - * Runtime#availableProcessors()} - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3FILEIO_DELETE_THREADS = "s3.delete.num-threads"; - - /** - * Determines if {@link S3FileIO} deletes the object when io.delete() is called, default to true. - * Once disabled, users are expected to set tags through {@link #S3_DELETE_TAGS_PREFIX} and manage - * deleted files through S3 lifecycle policy. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_DELETE_ENABLED = "s3.delete-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_DELETE_ENABLED_DEFAULT = true; - - /** - * Determines if S3 client will use the Acceleration Mode, default to false. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/transfer-acceleration.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_ACCELERATION_ENABLED = "s3.acceleration-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_ACCELERATION_ENABLED_DEFAULT = false; - - /** - * Determines if S3 client will use the Dualstack Mode, default to false. - * - *

For more details, see - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_DUALSTACK_ENABLED = "s3.dualstack-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_DUALSTACK_ENABLED_DEFAULT = false; - - /** - * Used by {@link S3FileIO}, prefix used for bucket access point configuration. To set, we can - * pass a catalog property. - * - *

For more details, see https://aws.amazon.com/s3/features/access-points/ - * - *

Example: s3.access-points.my-bucket=access-point - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_ACCESS_POINTS_PREFIX = "s3.access-points."; - - /** - * This flag controls whether the S3 client will be initialized during the S3FileIO - * initialization, instead of default lazy initialization upon use. This is needed for cases that - * the credentials to use might change and needs to be preloaded. - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final String S3_PRELOAD_CLIENT_ENABLED = "s3.preload-client-enabled"; - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated public static final boolean S3_PRELOAD_CLIENT_ENABLED_DEFAULT = false; - /** * Used by {@link LakeFormationAwsClientFactory}. The table name used as part of lake formation * credentials request. @@ -934,7 +215,6 @@ public class AwsProperties implements Serializable { public static final String REST_SESSION_TOKEN = "rest.session-token"; private static final String HTTP_CLIENT_PREFIX = "http-client."; - private String httpClientType; private final Map httpClientProperties; private final Set stsClientAssumeRoleTags; @@ -947,33 +227,6 @@ public class AwsProperties implements Serializable { private String clientCredentialsProvider; private final Map clientCredentialsProviderProperties; - private String s3FileIoSseType; - private String s3FileIoSseKey; - private String s3FileIoSseMd5; - private String s3AccessKeyId; - private String s3SecretAccessKey; - private String s3SessionToken; - private int s3FileIoMultipartUploadThreads; - private int s3FileIoMultiPartSize; - private int s3FileIoDeleteBatchSize; - private double s3FileIoMultipartThresholdFactor; - private String s3fileIoStagingDirectory; - private ObjectCannedACL s3FileIoAcl; - private boolean isS3ChecksumEnabled; - private final Set s3WriteTags; - private boolean s3WriteTableTagEnabled; - private boolean s3WriteNamespaceTagEnabled; - private final Set s3DeleteTags; - private int s3FileIoDeleteThreads; - private boolean isS3DeleteEnabled; - private final Map s3BucketToAccessPointMapping; - private boolean s3PreloadClientEnabled; - private boolean s3DualStackEnabled; - private boolean s3PathStyleAccess; - private boolean s3UseArnRegionEnabled; - private boolean s3AccelerationEnabled; - private String s3Endpoint; - private String glueEndpoint; private String glueCatalogId; private boolean glueCatalogSkipArchive; @@ -982,8 +235,6 @@ public class AwsProperties implements Serializable { private String dynamoDbTableName; private String dynamoDbEndpoint; - - private final boolean s3RemoteSigningEnabled; private final Map allProperties; private String restSigningRegion; @@ -993,7 +244,6 @@ public class AwsProperties implements Serializable { private String restSessionToken; public AwsProperties() { - this.httpClientType = HTTP_CLIENT_TYPE_DEFAULT; this.httpClientProperties = Collections.emptyMap(); this.stsClientAssumeRoleTags = Sets.newHashSet(); @@ -1006,33 +256,6 @@ public AwsProperties() { this.clientCredentialsProvider = null; this.clientCredentialsProviderProperties = null; - this.s3FileIoSseType = S3FILEIO_SSE_TYPE_NONE; - this.s3FileIoSseKey = null; - this.s3FileIoSseMd5 = null; - this.s3AccessKeyId = null; - this.s3SecretAccessKey = null; - this.s3SessionToken = null; - this.s3FileIoAcl = null; - this.s3Endpoint = null; - this.s3FileIoMultipartUploadThreads = Runtime.getRuntime().availableProcessors(); - this.s3FileIoMultiPartSize = S3FILEIO_MULTIPART_SIZE_DEFAULT; - this.s3FileIoMultipartThresholdFactor = S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT; - this.s3FileIoDeleteBatchSize = S3FILEIO_DELETE_BATCH_SIZE_DEFAULT; - this.s3fileIoStagingDirectory = System.getProperty("java.io.tmpdir"); - this.isS3ChecksumEnabled = S3_CHECKSUM_ENABLED_DEFAULT; - this.s3WriteTags = Sets.newHashSet(); - this.s3WriteTableTagEnabled = S3_WRITE_TABLE_TAG_ENABLED_DEFAULT; - this.s3WriteNamespaceTagEnabled = S3_WRITE_NAMESPACE_TAG_ENABLED_DEFAULT; - this.s3DeleteTags = Sets.newHashSet(); - this.s3FileIoDeleteThreads = Runtime.getRuntime().availableProcessors(); - this.isS3DeleteEnabled = S3_DELETE_ENABLED_DEFAULT; - this.s3BucketToAccessPointMapping = Collections.emptyMap(); - this.s3PreloadClientEnabled = S3_PRELOAD_CLIENT_ENABLED_DEFAULT; - this.s3DualStackEnabled = S3_DUALSTACK_ENABLED_DEFAULT; - this.s3PathStyleAccess = S3FILEIO_PATH_STYLE_ACCESS_DEFAULT; - this.s3UseArnRegionEnabled = S3_USE_ARN_REGION_ENABLED_DEFAULT; - this.s3AccelerationEnabled = S3_ACCELERATION_ENABLED_DEFAULT; - this.glueCatalogId = null; this.glueEndpoint = null; this.glueCatalogSkipArchive = GLUE_CATALOG_SKIP_ARCHIVE_DEFAULT; @@ -1042,20 +265,13 @@ public AwsProperties() { this.dynamoDbEndpoint = null; this.dynamoDbTableName = DYNAMODB_TABLE_NAME_DEFAULT; - this.s3RemoteSigningEnabled = S3_REMOTE_SIGNING_ENABLED_DEFAULT; this.allProperties = Maps.newHashMap(); this.restSigningName = REST_SIGNING_NAME_DEFAULT; - - ValidationException.check( - s3KeyIdAccessKeyBothConfigured(), - "S3 client access key ID and secret access key must be set at the same time"); } @SuppressWarnings("MethodLength") public AwsProperties(Map properties) { - this.httpClientType = - PropertyUtil.propertyAsString(properties, HTTP_CLIENT_TYPE, HTTP_CLIENT_TYPE_DEFAULT); this.httpClientProperties = PropertyUtil.filterProperties(properties, key -> key.startsWith(HTTP_CLIENT_PREFIX)); this.stsClientAssumeRoleTags = toStsTags(properties, CLIENT_ASSUME_ROLE_TAGS_PREFIX); @@ -1066,24 +282,12 @@ public AwsProperties(Map properties) { this.clientAssumeRoleExternalId = properties.get(CLIENT_ASSUME_ROLE_EXTERNAL_ID); this.clientAssumeRoleRegion = properties.get(CLIENT_ASSUME_ROLE_REGION); this.clientAssumeRoleSessionName = properties.get(CLIENT_ASSUME_ROLE_SESSION_NAME); - this.clientRegion = properties.get(CLIENT_REGION); - this.clientCredentialsProvider = properties.get(CLIENT_CREDENTIALS_PROVIDER); + this.clientRegion = properties.get(AwsClientProperties.CLIENT_REGION); + this.clientCredentialsProvider = + properties.get(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER); this.clientCredentialsProviderProperties = - PropertyUtil.propertiesWithPrefix(properties, CLIENT_CREDENTIAL_PROVIDER_PREFIX); - - this.s3FileIoSseType = properties.getOrDefault(S3FILEIO_SSE_TYPE, S3FILEIO_SSE_TYPE_NONE); - this.s3FileIoSseKey = properties.get(S3FILEIO_SSE_KEY); - this.s3FileIoSseMd5 = properties.get(S3FILEIO_SSE_MD5); - this.s3AccessKeyId = properties.get(S3FILEIO_ACCESS_KEY_ID); - this.s3SecretAccessKey = properties.get(S3FILEIO_SECRET_ACCESS_KEY); - this.s3SessionToken = properties.get(S3FILEIO_SESSION_TOKEN); - if (S3FILEIO_SSE_TYPE_CUSTOM.equals(s3FileIoSseType)) { - Preconditions.checkNotNull( - s3FileIoSseKey, "Cannot initialize SSE-C S3FileIO with null encryption key"); - Preconditions.checkNotNull( - s3FileIoSseMd5, "Cannot initialize SSE-C S3FileIO with null encryption key MD5"); - } - this.s3Endpoint = properties.get(S3FILEIO_ENDPOINT); + PropertyUtil.propertiesWithPrefix( + properties, AwsClientProperties.CLIENT_CREDENTIAL_PROVIDER_PREFIX); this.glueEndpoint = properties.get(GLUE_CATALOG_ENDPOINT); this.glueCatalogId = properties.get(GLUE_CATALOG_ID); @@ -1098,87 +302,11 @@ public AwsProperties(Map properties) { this.glueLakeFormationEnabled = PropertyUtil.propertyAsBoolean( properties, GLUE_LAKEFORMATION_ENABLED, GLUE_LAKEFORMATION_ENABLED_DEFAULT); - this.s3FileIoMultipartUploadThreads = - PropertyUtil.propertyAsInt( - properties, - S3FILEIO_MULTIPART_UPLOAD_THREADS, - Runtime.getRuntime().availableProcessors()); - this.s3PathStyleAccess = - PropertyUtil.propertyAsBoolean( - properties, S3FILEIO_PATH_STYLE_ACCESS, S3FILEIO_PATH_STYLE_ACCESS_DEFAULT); - this.s3UseArnRegionEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_USE_ARN_REGION_ENABLED, S3_USE_ARN_REGION_ENABLED_DEFAULT); - this.s3AccelerationEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_ACCELERATION_ENABLED, S3_ACCELERATION_ENABLED_DEFAULT); - this.s3DualStackEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_DUALSTACK_ENABLED, S3_DUALSTACK_ENABLED_DEFAULT); - try { - this.s3FileIoMultiPartSize = - PropertyUtil.propertyAsInt( - properties, S3FILEIO_MULTIPART_SIZE, S3FILEIO_MULTIPART_SIZE_DEFAULT); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - String.format( - "Input malformed or exceeded maximum multipart upload size 5GB: %s", - properties.get(S3FILEIO_MULTIPART_SIZE))); - } - this.s3FileIoMultipartThresholdFactor = - PropertyUtil.propertyAsDouble( - properties, - S3FILEIO_MULTIPART_THRESHOLD_FACTOR, - S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT); - Preconditions.checkArgument( - s3FileIoMultipartThresholdFactor >= 1.0, "Multipart threshold factor must be >= to 1.0"); - Preconditions.checkArgument( - s3FileIoMultiPartSize >= S3FILEIO_MULTIPART_SIZE_MIN, - "Minimum multipart upload object size must be larger than 5 MB."); - this.s3fileIoStagingDirectory = - PropertyUtil.propertyAsString( - properties, S3FILEIO_STAGING_DIRECTORY, System.getProperty("java.io.tmpdir")); - String aclType = properties.get(S3FILEIO_ACL); - this.s3FileIoAcl = ObjectCannedACL.fromValue(aclType); - Preconditions.checkArgument( - s3FileIoAcl == null || !s3FileIoAcl.equals(ObjectCannedACL.UNKNOWN_TO_SDK_VERSION), - "Cannot support S3 CannedACL " + aclType); - this.isS3ChecksumEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_CHECKSUM_ENABLED, S3_CHECKSUM_ENABLED_DEFAULT); - this.s3FileIoDeleteBatchSize = - PropertyUtil.propertyAsInt( - properties, S3FILEIO_DELETE_BATCH_SIZE, S3FILEIO_DELETE_BATCH_SIZE_DEFAULT); - Preconditions.checkArgument( - s3FileIoDeleteBatchSize > 0 && s3FileIoDeleteBatchSize <= S3FILEIO_DELETE_BATCH_SIZE_MAX, - String.format( - "Deletion batch size must be between 1 and %s", S3FILEIO_DELETE_BATCH_SIZE_MAX)); - this.s3WriteTags = toS3Tags(properties, S3_WRITE_TAGS_PREFIX); - this.s3WriteTableTagEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_WRITE_TABLE_TAG_ENABLED, S3_WRITE_TABLE_TAG_ENABLED_DEFAULT); - this.s3WriteNamespaceTagEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_WRITE_NAMESPACE_TAG_ENABLED, S3_WRITE_NAMESPACE_TAG_ENABLED_DEFAULT); - this.s3DeleteTags = toS3Tags(properties, S3_DELETE_TAGS_PREFIX); - this.s3FileIoDeleteThreads = - PropertyUtil.propertyAsInt( - properties, S3FILEIO_DELETE_THREADS, Runtime.getRuntime().availableProcessors()); - this.isS3DeleteEnabled = - PropertyUtil.propertyAsBoolean(properties, S3_DELETE_ENABLED, S3_DELETE_ENABLED_DEFAULT); - this.s3BucketToAccessPointMapping = - PropertyUtil.propertiesWithPrefix(properties, S3_ACCESS_POINTS_PREFIX); - this.s3PreloadClientEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_PRELOAD_CLIENT_ENABLED, S3_PRELOAD_CLIENT_ENABLED_DEFAULT); this.dynamoDbEndpoint = properties.get(DYNAMODB_ENDPOINT); this.dynamoDbTableName = PropertyUtil.propertyAsString(properties, DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT); - this.s3RemoteSigningEnabled = - PropertyUtil.propertyAsBoolean( - properties, S3_REMOTE_SIGNING_ENABLED, S3_REMOTE_SIGNING_ENABLED_DEFAULT); this.allProperties = SerializableMap.copyOf(properties); this.restSigningRegion = properties.get(REST_SIGNER_REGION); @@ -1186,10 +314,6 @@ public AwsProperties(Map properties) { this.restAccessKeyId = properties.get(REST_ACCESS_KEY_ID); this.restSecretAccessKey = properties.get(REST_SECRET_ACCESS_KEY); this.restSessionToken = properties.get(REST_SESSION_TOKEN); - - ValidationException.check( - s3KeyIdAccessKeyBothConfigured(), - "S3 client access key ID and secret access key must be set at the same time"); } public Set stsClientAssumeRoleTags() { @@ -1208,84 +332,12 @@ public String clientAssumeRoleExternalId() { return clientAssumeRoleExternalId; } - public String clientAssumeRoleRegion() { - return clientAssumeRoleRegion; - } - - public String clientAssumeRoleSessionName() { - return clientAssumeRoleSessionName; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public String s3FileIoSseType() { - return s3FileIoSseType; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoSseType(String sseType) { - this.s3FileIoSseType = sseType; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public String s3FileIoSseKey() { - return s3FileIoSseKey; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public int s3FileIoDeleteBatchSize() { - return s3FileIoDeleteBatchSize; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoDeleteBatchSize(int deleteBatchSize) { - this.s3FileIoDeleteBatchSize = deleteBatchSize; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoSseKey(String sseKey) { - this.s3FileIoSseKey = sseKey; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public String s3FileIoSseMd5() { - return s3FileIoSseMd5; + public String clientAssumeRoleRegion() { + return clientAssumeRoleRegion; } - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoSseMd5(String sseMd5) { - this.s3FileIoSseMd5 = sseMd5; + public String clientAssumeRoleSessionName() { + return clientAssumeRoleSessionName; } public String glueCatalogId() { @@ -1320,114 +372,6 @@ public void setGlueLakeFormationEnabled(boolean glueLakeFormationEnabled) { this.glueLakeFormationEnabled = glueLakeFormationEnabled; } - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public int s3FileIoMultipartUploadThreads() { - return s3FileIoMultipartUploadThreads; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoMultipartUploadThreads(int threads) { - this.s3FileIoMultipartUploadThreads = threads; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public int s3FileIoMultiPartSize() { - return s3FileIoMultiPartSize; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoMultiPartSize(int size) { - this.s3FileIoMultiPartSize = size; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public double s3FileIOMultipartThresholdFactor() { - return s3FileIoMultipartThresholdFactor; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoMultipartThresholdFactor(double factor) { - this.s3FileIoMultipartThresholdFactor = factor; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public String s3fileIoStagingDirectory() { - return s3fileIoStagingDirectory; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3fileIoStagingDirectory(String directory) { - this.s3fileIoStagingDirectory = directory; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public ObjectCannedACL s3FileIoAcl() { - return this.s3FileIoAcl; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoAcl(ObjectCannedACL acl) { - this.s3FileIoAcl = acl; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3PreloadClientEnabled(boolean s3PreloadClientEnabled) { - this.s3PreloadClientEnabled = s3PreloadClientEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public boolean s3PreloadClientEnabled() { - return s3PreloadClientEnabled; - } - public String dynamoDbTableName() { return dynamoDbTableName; } @@ -1436,287 +380,40 @@ public void setDynamoDbTableName(String name) { this.dynamoDbTableName = name; } - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public boolean isS3ChecksumEnabled() { - return this.isS3ChecksumEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3ChecksumEnabled(boolean eTagCheckEnabled) { - this.isS3ChecksumEnabled = eTagCheckEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public Set s3WriteTags() { - return s3WriteTags; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public boolean s3WriteTableTagEnabled() { - return s3WriteTableTagEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3WriteTableTagEnabled(boolean s3WriteTableNameTagEnabled) { - this.s3WriteTableTagEnabled = s3WriteTableNameTagEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public boolean s3WriteNamespaceTagEnabled() { - return s3WriteNamespaceTagEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3WriteNamespaceTagEnabled(boolean s3WriteNamespaceTagEnabled) { - this.s3WriteNamespaceTagEnabled = s3WriteNamespaceTagEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public Set s3DeleteTags() { - return s3DeleteTags; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public int s3FileIoDeleteThreads() { - return s3FileIoDeleteThreads; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void setS3FileIoDeleteThreads(int threads) { - this.s3FileIoDeleteThreads = threads; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public boolean isS3DeleteEnabled() { - return isS3DeleteEnabled; - } - - /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ + /** @deprecated will be removed in 1.5.0, use {@link HttpClientProperties} instead */ @Deprecated - public void setS3DeleteEnabled(boolean s3DeleteEnabled) { - this.isS3DeleteEnabled = s3DeleteEnabled; + public Map httpClientProperties() { + return httpClientProperties; } /** - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead + * @deprecated will be removed in 1.5.0, use {@link AwsClientProperties#clientRegion()} instead */ @Deprecated - public Map s3BucketToAccessPointMapping() { - return s3BucketToAccessPointMapping; - } - - public Map httpClientProperties() { - return httpClientProperties; - } - public String clientRegion() { return clientRegion; } - public void setClientRegion(String clientRegion) { - this.clientRegion = clientRegion; - } - /** - * Configure the credentials for an S3 client. - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyS3CredentialConfigurations)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} + * @deprecated will be removed in 1.5.0, use {@link AwsClientProperties#setClientRegion(String)} * instead */ @Deprecated - public void applyS3CredentialConfigurations(T builder) { - builder.credentialsProvider( - s3RemoteSigningEnabled - ? AnonymousCredentialsProvider.create() - : credentialsProvider(s3AccessKeyId, s3SecretAccessKey, s3SessionToken)); + public void setClientRegion(String clientRegion) { + this.clientRegion = clientRegion; } /** - * Configure a client AWS region. - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyClientRegionConfiguration)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.AwsClientProperties} - * instead + * @deprecated will be removed in 1.5.0, use {@link + * AwsClientProperties#applyClientCredentialConfigurations(AwsClientBuilder)} instead */ @Deprecated - public void applyClientRegionConfiguration(T builder) { - if (clientRegion != null) { - builder.region(Region.of(clientRegion)); - } - } - - /** - * Configure the credential provider for AWS clients. - * - *

Sample usage: - * - * 

-   *     DynamoDbClient.builder().applyMutation(awsProperties::applyClientCredentialConfigurations)
-   *
- */ public void applyClientCredentialConfigurations(T builder) { if (!Strings.isNullOrEmpty(this.clientCredentialsProvider)) { builder.credentialsProvider(credentialsProvider(this.clientCredentialsProvider)); } } - /** - * Configure services settings for an S3 client. The settings include: s3DualStack, - * s3UseArnRegion, s3PathStyleAccess, and s3Acceleration - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyS3ServiceConfigurations)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void applyS3ServiceConfigurations(T builder) { - builder - .dualstackEnabled(s3DualStackEnabled) - .serviceConfiguration( - S3Configuration.builder() - .pathStyleAccessEnabled(s3PathStyleAccess) - .useArnRegionEnabled(s3UseArnRegionEnabled) - .accelerateModeEnabled(s3AccelerationEnabled) - .build()); - } - - /** - * Configure a signer for an S3 client. - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyS3SignerConfiguration)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void applyS3SignerConfiguration(T builder) { - if (s3RemoteSigningEnabled) { - builder.overrideConfiguration( - c -> - c.putAdvancedOption( - SdkAdvancedClientOption.SIGNER, S3V4RestSignerClient.create(allProperties))); - } - } - - /** - * Configure the httpClient for a client according to the HttpClientType. The two supported - * HttpClientTypes are urlconnection and apache - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyHttpClientConfigurations)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - public void applyHttpClientConfigurations(T builder) { - if (Strings.isNullOrEmpty(httpClientType)) { - httpClientType = HTTP_CLIENT_TYPE_DEFAULT; - } - switch (httpClientType) { - case HTTP_CLIENT_TYPE_URLCONNECTION: - UrlConnectionHttpClientConfigurations urlConnectionHttpClientConfigurations = - loadHttpClientConfigurations(UrlConnectionHttpClientConfigurations.class.getName()); - urlConnectionHttpClientConfigurations.configureHttpClientBuilder(builder); - break; - case HTTP_CLIENT_TYPE_APACHE: - ApacheHttpClientConfigurations apacheHttpClientConfigurations = - loadHttpClientConfigurations(ApacheHttpClientConfigurations.class.getName()); - apacheHttpClientConfigurations.configureHttpClientBuilder(builder); - break; - default: - throw new IllegalArgumentException("Unrecognized HTTP client type " + httpClientType); - } - } - - /** - * Override the endpoint for an S3 client. - * - *

Sample usage: - * - * 

-   *     S3Client.builder().applyMutation(awsProperties::applyS3EndpointConfigurations)
-   *
- * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties} - * instead - */ - @Deprecated - public void applyS3EndpointConfigurations(T builder) { - configureEndpoint(builder, s3Endpoint); - } - /** * Override the endpoint for a glue client. * @@ -1736,7 +433,7 @@ public void applyGlueEndpointConfigurations(T buil *

Sample usage: * * 

-   *     DynamoDbClient.builder().applyMutation(awsProperties::applyS3EndpointConfigurations)
+   *     DynamoDbClient.builder().applyMutation(awsProperties::applyDynamoDbEndpointConfigurations)
    *
*/ public void applyDynamoDbEndpointConfigurations(T builder) { @@ -1760,12 +457,6 @@ public AwsCredentialsProvider restCredentialsProvider() { this.restAccessKeyId, this.restSecretAccessKey, this.restSessionToken); } - private Set toS3Tags(Map properties, String prefix) { - return PropertyUtil.propertiesWithPrefix(properties, prefix).entrySet().stream() - .map(e -> Tag.builder().key(e.getKey()).value(e.getValue()).build()) - .collect(Collectors.toSet()); - } - private Set toStsTags( Map properties, String prefix) { return PropertyUtil.propertiesWithPrefix(properties, prefix).entrySet().stream() @@ -1778,10 +469,6 @@ private Set toStsTags( .collect(Collectors.toSet()); } - private boolean s3KeyIdAccessKeyBothConfigured() { - return (s3AccessKeyId == null) == (s3SecretAccessKey == null); - } - private AwsCredentialsProvider credentialsProvider( String accessKeyId, String secretAccessKey, String sessionToken) { if (accessKeyId != null) { @@ -1846,30 +533,4 @@ private void configureEndpoint(T builder, String en builder.endpointOverride(URI.create(endpoint)); } } - - /** - * Dynamically load the http client builder to avoid runtime deps requirements of both {@link - * software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient} and {@link - * software.amazon.awssdk.http.apache.ApacheHttpClient}, since including both will cause error - * described in issue#6715 - * - * @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties} - * instead - */ - @Deprecated - private T loadHttpClientConfigurations(String impl) { - Object httpClientConfigurations; - try { - httpClientConfigurations = - DynMethods.builder("create") - .hiddenImpl(impl, Map.class) - .buildStaticChecked() - .invoke(httpClientProperties); - return (T) httpClientConfigurations; - } catch (NoSuchMethodException e) { - throw new IllegalArgumentException( - String.format("Cannot create %s to generate and configure the http client builder", impl), - e); - } - } } diff --git a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java index 72e1eeb65bcf..e71b02caef7d 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java +++ b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java @@ -42,10 +42,10 @@ public void configureHttpClientBuilder(T awsCli private void initialize(Map httpClientProperties) { this.httpClientUrlConnectionConnectionTimeoutMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS); + httpClientProperties, HttpClientProperties.URLCONNECTION_CONNECTION_TIMEOUT_MS); this.httpClientUrlConnectionSocketTimeoutMs = PropertyUtil.propertyAsNullableLong( - httpClientProperties, AwsProperties.HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS); + httpClientProperties, HttpClientProperties.URLCONNECTION_SOCKET_TIMEOUT_MS); } @VisibleForTesting diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java index 0338d1da5bd9..dea7926729ef 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java @@ -39,6 +39,7 @@ import org.apache.iceberg.aws.AwsClientFactory; import org.apache.iceberg.aws.AwsProperties; import org.apache.iceberg.aws.lakeformation.LakeFormationAwsClientFactory; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; @@ -92,6 +93,7 @@ public class GlueCatalog extends BaseMetastoreCatalog private String catalogName; private String warehousePath; private AwsProperties awsProperties; + private S3FileIOProperties s3FileIOProperties; private LockManager lockManager; private CloseableGroup closeableGroup; private Map catalogProperties; @@ -143,6 +145,7 @@ public void initialize(String name, Map properties) { name, properties.get(CatalogProperties.WAREHOUSE_LOCATION), new AwsProperties(properties), + new S3FileIOProperties(properties), awsClientFactory.glue(), initializeLockManager(properties)); } @@ -167,18 +170,25 @@ void initialize( String name, String path, AwsProperties properties, + S3FileIOProperties s3Properties, GlueClient client, LockManager lock, Map catalogProps) { this.catalogProperties = catalogProps; - initialize(name, path, properties, client, lock); + initialize(name, path, properties, s3Properties, client, lock); } @VisibleForTesting void initialize( - String name, String path, AwsProperties properties, GlueClient client, LockManager lock) { + String name, + String path, + AwsProperties properties, + S3FileIOProperties s3Properties, + GlueClient client, + LockManager lock) { this.catalogName = name; this.awsProperties = properties; + this.s3FileIOProperties = s3Properties; this.warehousePath = (path != null && path.length() > 0) ? LocationUtil.stripTrailingSlash(path) : null; this.glue = client; @@ -198,15 +208,16 @@ protected TableOperations newTableOps(TableIdentifier tableIdentifier) { ImmutableMap.builder().putAll(catalogProperties); boolean skipNameValidation = awsProperties.glueCatalogSkipNameValidation(); - if (awsProperties.s3WriteTableTagEnabled()) { + if (s3FileIOProperties.writeTableTagEnabled()) { tableSpecificCatalogPropertiesBuilder.put( - AwsProperties.S3_WRITE_TAGS_PREFIX.concat(AwsProperties.S3_TAG_ICEBERG_TABLE), + S3FileIOProperties.WRITE_TAGS_PREFIX.concat(S3FileIOProperties.S3_TAG_ICEBERG_TABLE), IcebergToGlueConverter.getTableName(tableIdentifier, skipNameValidation)); } - if (awsProperties.s3WriteNamespaceTagEnabled()) { + if (s3FileIOProperties.isWriteNamespaceTagEnabled()) { tableSpecificCatalogPropertiesBuilder.put( - AwsProperties.S3_WRITE_TAGS_PREFIX.concat(AwsProperties.S3_TAG_ICEBERG_NAMESPACE), + S3FileIOProperties.WRITE_TAGS_PREFIX.concat( + S3FileIOProperties.S3_TAG_ICEBERG_NAMESPACE), IcebergToGlueConverter.getDatabaseName(tableIdentifier, skipNameValidation)); } @@ -218,7 +229,7 @@ protected TableOperations newTableOps(TableIdentifier tableIdentifier) { .put( AwsProperties.LAKE_FORMATION_TABLE_NAME, IcebergToGlueConverter.getTableName(tableIdentifier, skipNameValidation)) - .put(AwsProperties.S3_PRELOAD_CLIENT_ENABLED, String.valueOf(true)); + .put(S3FileIOProperties.PRELOAD_CLIENT_ENABLED, String.valueOf(true)); } // FileIO initialization depends on tableSpecificCatalogProperties, so a new FileIO is diff --git a/aws/src/main/java/org/apache/iceberg/aws/lakeformation/LakeFormationAwsClientFactory.java b/aws/src/main/java/org/apache/iceberg/aws/lakeformation/LakeFormationAwsClientFactory.java index 7bd19f5a85a2..fb0b1c23efc4 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/lakeformation/LakeFormationAwsClientFactory.java +++ b/aws/src/main/java/org/apache/iceberg/aws/lakeformation/LakeFormationAwsClientFactory.java @@ -78,9 +78,9 @@ public void initialize(Map catalogProperties) { public S3Client s3() { if (isTableRegisteredWithLakeFormation()) { return S3Client.builder() - .applyMutation(awsProperties()::applyHttpClientConfigurations) - .applyMutation(awsProperties()::applyS3EndpointConfigurations) - .applyMutation(awsProperties()::applyS3ServiceConfigurations) + .applyMutation(httpClientProperties()::applyHttpClientConfigurations) + .applyMutation(s3FileIOProperties()::applyEndpointConfigurations) + .applyMutation(s3FileIOProperties()::applyServiceConfigurations) .credentialsProvider( new LakeFormationCredentialsProvider(lakeFormation(), buildTableArn())) .region(Region.of(region())) @@ -94,7 +94,7 @@ public S3Client s3() { public KmsClient kms() { if (isTableRegisteredWithLakeFormation()) { return KmsClient.builder() - .applyMutation(awsProperties()::applyHttpClientConfigurations) + .applyMutation(httpClientProperties()::applyHttpClientConfigurations) .credentialsProvider( new LakeFormationCredentialsProvider(lakeFormation(), buildTableArn())) .region(Region.of(region())) @@ -134,7 +134,7 @@ private String buildTableArn() { private LakeFormationClient lakeFormation() { return LakeFormationClient.builder() .applyMutation(this::applyAssumeRoleConfigurations) - .applyMutation(awsProperties()::applyHttpClientConfigurations) + .applyMutation(httpClientProperties()::applyHttpClientConfigurations) .build(); } diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index 901840205a76..af3d5a8472dd 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -741,7 +741,7 @@ public void applySignerConfiguration(T builder) { *

Sample usage: * * 

-   *     S3Client.builder().applyMutation(awsProperties::applyS3EndpointConfigurations)
+   *     S3Client.builder().applyMutation(s3FileIOProperties::applyEndpointConfigurations)
    *
*/ public void applyEndpointConfigurations(T builder) { diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java index 8b19d339a75b..01c14790a34e 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java +++ b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java @@ -230,9 +230,9 @@ private AwsClientFactory getAwsClientFactoryByCredentialsProvider(String provide private Map getDefaultClientFactoryProperties(String providerClass) { Map properties = Maps.newHashMap(); - properties.put(AwsProperties.CLIENT_CREDENTIALS_PROVIDER + ".param1", "value1"); - properties.put(AwsProperties.CLIENT_REGION, Region.AWS_GLOBAL.toString()); - properties.put(AwsProperties.CLIENT_CREDENTIALS_PROVIDER, providerClass); + properties.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER + ".param1", "value1"); + properties.put(AwsClientProperties.CLIENT_REGION, Region.AWS_GLOBAL.toString()); + properties.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER, providerClass); return properties; } diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestAwsProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestAwsProperties.java index 556968ec22eb..c4b9bd4d5bc1 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/TestAwsProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/TestAwsProperties.java @@ -20,310 +20,31 @@ import java.io.IOException; import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.TestHelpers; -import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; -import software.amazon.awssdk.core.signer.Signer; -import software.amazon.awssdk.http.SdkHttpClient; -import software.amazon.awssdk.http.apache.ApacheHttpClient; -import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.model.ObjectCannedACL; public class TestAwsProperties { - @Test - public void testS3FileIoSseCustom_mustHaveCustomKey() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_SSE_TYPE, AwsProperties.S3FILEIO_SSE_TYPE_CUSTOM); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(NullPointerException.class) - .hasMessage("Cannot initialize SSE-C S3FileIO with null encryption key"); - } - - @Test - public void testS3FileIoSseCustom_mustHaveCustomMd5() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_SSE_TYPE, AwsProperties.S3FILEIO_SSE_TYPE_CUSTOM); - map.put(AwsProperties.S3FILEIO_SSE_KEY, "something"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(NullPointerException.class) - .hasMessage("Cannot initialize SSE-C S3FileIO with null encryption key MD5"); - } - - @Test - public void testS3FileIoAcl() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_ACL, ObjectCannedACL.AUTHENTICATED_READ.toString()); - AwsProperties properties = new AwsProperties(map); - Assertions.assertThat(properties.s3FileIoAcl()).isEqualTo(ObjectCannedACL.AUTHENTICATED_READ); - } - - @Test - public void testS3FileIoAcl_unknownType() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_ACL, "bad-input"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Cannot support S3 CannedACL bad-input"); - } - - @Test - public void testS3MultipartSizeTooSmall() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_MULTIPART_SIZE, "1"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Minimum multipart upload object size must be larger than 5 MB."); - } - - @Test - public void testS3MultipartSizeTooLarge() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_MULTIPART_SIZE, "5368709120"); // 5GB - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Input malformed or exceeded maximum multipart upload size 5GB: 5368709120"); - } - - @Test - public void testS3MultipartThresholdFactorLessThanOne() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_MULTIPART_THRESHOLD_FACTOR, "0.9"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Multipart threshold factor must be >= to 1.0"); - } - - @Test - public void testS3FileIoDeleteBatchSizeTooLarge() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_DELETE_BATCH_SIZE, "2000"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Deletion batch size must be between 1 and 1000"); - } - - @Test - public void testS3FileIoDeleteBatchSizeTooSmall() { - Map map = Maps.newHashMap(); - map.put(AwsProperties.S3FILEIO_DELETE_BATCH_SIZE, "0"); - - Assertions.assertThatThrownBy(() -> new AwsProperties(map)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Deletion batch size must be between 1 and 1000"); - } - - @Test - public void testS3FileIoDefaultCredentialsConfiguration() { - // set nothing - Map properties = Maps.newHashMap(); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); - ArgumentCaptor awsCredentialsProviderCaptor = - ArgumentCaptor.forClass(AwsCredentialsProvider.class); - - awsProperties.applyS3CredentialConfigurations(mockS3ClientBuilder); - Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); - AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); - - Assertions.assertThat(capturedAwsCredentialsProvider) - .as("Should use default credentials if nothing is set") - .isInstanceOf(DefaultCredentialsProvider.class); - } - - @Test - public void testS3FileIoBasicCredentialsConfiguration() { - // set access key id and secret access key - Map properties = Maps.newHashMap(); - properties.put(AwsProperties.S3FILEIO_ACCESS_KEY_ID, "key"); - properties.put(AwsProperties.S3FILEIO_SECRET_ACCESS_KEY, "secret"); - AwsProperties awsPropertiesTwoSet = new AwsProperties(properties); - S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); - ArgumentCaptor awsCredentialsProviderCaptor = - ArgumentCaptor.forClass(AwsCredentialsProvider.class); - - awsPropertiesTwoSet.applyS3CredentialConfigurations(mockS3ClientBuilder); - Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); - AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); - - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials()) - .as("Should use basic credentials if access key ID and secret access key are set") - .isInstanceOf(AwsBasicCredentials.class); - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().accessKeyId()) - .as("The access key id should be the same as the one set by tag S3FILEIO_ACCESS_KEY_ID") - .isEqualTo("key"); - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().secretAccessKey()) - .as( - "The secret access key should be the same as the one set by tag S3FILEIO_SECRET_ACCESS_KEY") - .isEqualTo("secret"); - } - - @Test - public void testS3FileIoSessionCredentialsConfiguration() { - // set access key id, secret access key, and session token - Map properties = Maps.newHashMap(); - properties.put(AwsProperties.S3FILEIO_ACCESS_KEY_ID, "key"); - properties.put(AwsProperties.S3FILEIO_SECRET_ACCESS_KEY, "secret"); - properties.put(AwsProperties.S3FILEIO_SESSION_TOKEN, "token"); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); - ArgumentCaptor awsCredentialsProviderCaptor = - ArgumentCaptor.forClass(AwsCredentialsProvider.class); - - awsProperties.applyS3CredentialConfigurations(mockS3ClientBuilder); - Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); - AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); - - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials()) - .as("Should use session credentials if session token is set") - .isInstanceOf(AwsSessionCredentials.class); - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().accessKeyId()) - .as("The access key id should be the same as the one set by tag S3FILEIO_ACCESS_KEY_ID") - .isEqualTo("key"); - Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().secretAccessKey()) - .as( - "The secret access key should be the same as the one set by tag S3FILEIO_SECRET_ACCESS_KEY") - .isEqualTo("secret"); - } - - @Test - public void testUrlHttpClientConfiguration() { - Map properties = Maps.newHashMap(); - properties.put(AwsProperties.HTTP_CLIENT_TYPE, "urlconnection"); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); - ArgumentCaptor httpClientBuilderCaptor = - ArgumentCaptor.forClass(SdkHttpClient.Builder.class); - - awsProperties.applyHttpClientConfigurations(mockS3ClientBuilder); - Mockito.verify(mockS3ClientBuilder).httpClientBuilder(httpClientBuilderCaptor.capture()); - SdkHttpClient.Builder capturedHttpClientBuilder = httpClientBuilderCaptor.getValue(); - - Assertions.assertThat(capturedHttpClientBuilder) - .as("Should use url connection http client") - .isInstanceOf(UrlConnectionHttpClient.Builder.class); - } - - @Test - public void testApacheHttpClientConfiguration() { - Map properties = Maps.newHashMap(); - properties.put(AwsProperties.HTTP_CLIENT_TYPE, "apache"); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); - ArgumentCaptor httpClientBuilderCaptor = - ArgumentCaptor.forClass(SdkHttpClient.Builder.class); - - awsProperties.applyHttpClientConfigurations(mockS3ClientBuilder); - Mockito.verify(mockS3ClientBuilder).httpClientBuilder(httpClientBuilderCaptor.capture()); - SdkHttpClient.Builder capturedHttpClientBuilder = httpClientBuilderCaptor.getValue(); - Assertions.assertThat(capturedHttpClientBuilder) - .as("Should use apache http client") - .isInstanceOf(ApacheHttpClient.Builder.class); - } - - @Test - public void testInvalidHttpClientType() { - Map properties = Maps.newHashMap(); - properties.put(AwsProperties.HTTP_CLIENT_TYPE, "test"); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder s3ClientBuilder = S3Client.builder(); - - Assertions.assertThatThrownBy( - () -> awsProperties.applyHttpClientConfigurations(s3ClientBuilder)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unrecognized HTTP client type test"); - } @Test public void testKryoSerialization() throws IOException { AwsProperties awsProperties = new AwsProperties(); AwsProperties deSerializedAwsProperties = TestHelpers.KryoHelpers.roundTripSerialize(awsProperties); - Assertions.assertThat(deSerializedAwsProperties.s3BucketToAccessPointMapping()) - .isEqualTo(awsProperties.s3BucketToAccessPointMapping()); Assertions.assertThat(deSerializedAwsProperties.httpClientProperties()) .isEqualTo(awsProperties.httpClientProperties()); AwsProperties awsPropertiesWithProps = new AwsProperties(ImmutableMap.of("a", "b")); AwsProperties deSerializedAwsPropertiesWithProps = TestHelpers.KryoHelpers.roundTripSerialize(awsPropertiesWithProps); - Assertions.assertThat(deSerializedAwsPropertiesWithProps.s3BucketToAccessPointMapping()) - .isEqualTo(awsPropertiesWithProps.s3BucketToAccessPointMapping()); Assertions.assertThat(deSerializedAwsPropertiesWithProps.httpClientProperties()) .isEqualTo(awsProperties.httpClientProperties()); AwsProperties awsPropertiesWithEmptyProps = new AwsProperties(Collections.emptyMap()); AwsProperties deSerializedAwsPropertiesWithEmptyProps = TestHelpers.KryoHelpers.roundTripSerialize(awsPropertiesWithProps); - Assertions.assertThat(deSerializedAwsPropertiesWithEmptyProps.s3BucketToAccessPointMapping()) - .isEqualTo(awsPropertiesWithEmptyProps.s3BucketToAccessPointMapping()); Assertions.assertThat(deSerializedAwsPropertiesWithEmptyProps.httpClientProperties()) .isEqualTo(awsProperties.httpClientProperties()); } - - @Test - public void testS3RemoteSignerWithoutUri() { - Map properties = - ImmutableMap.of(AwsProperties.S3_REMOTE_SIGNING_ENABLED, "true"); - AwsProperties awsProperties = new AwsProperties(properties); - - Assertions.assertThatThrownBy( - () -> awsProperties.applyS3SignerConfiguration(S3Client.builder())) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("S3 signer service URI is required"); - } - - @Test - public void testS3RemoteSigningEnabled() { - String uri = "http://localhost:12345"; - Map properties = - ImmutableMap.of( - AwsProperties.S3_REMOTE_SIGNING_ENABLED, "true", CatalogProperties.URI, uri); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder builder = S3Client.builder(); - - awsProperties.applyS3SignerConfiguration(builder); - - Optional signer = - builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.SIGNER); - Assertions.assertThat(signer).isPresent().get().isInstanceOf(S3V4RestSignerClient.class); - S3V4RestSignerClient signerClient = (S3V4RestSignerClient) signer.get(); - Assertions.assertThat(signerClient.baseSignerUri()).isEqualTo(uri); - Assertions.assertThat(signerClient.properties()).isEqualTo(properties); - } - - @Test - public void testS3RemoteSigningDisabled() { - Map properties = - ImmutableMap.of(AwsProperties.S3_REMOTE_SIGNING_ENABLED, "false"); - AwsProperties awsProperties = new AwsProperties(properties); - S3ClientBuilder builder = S3Client.builder(); - - awsProperties.applyS3SignerConfiguration(builder); - - Optional signer = - builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.SIGNER); - Assertions.assertThat(signer).isNotPresent(); - } } diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java index 2c6f35c3a048..8ad4cc9538eb 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java +++ b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java @@ -30,10 +30,10 @@ public class TestHttpClientConfigurations { @Test public void testUrlConnectionOverrideConfigurations() { Map properties = Maps.newHashMap(); - properties.put(AwsProperties.HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS, "90"); - properties.put(AwsProperties.HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS, "80"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_SOCKET_TIMEOUT_MS, "100"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_TIMEOUT_MS, "200"); + properties.put(HttpClientProperties.URLCONNECTION_SOCKET_TIMEOUT_MS, "90"); + properties.put(HttpClientProperties.URLCONNECTION_CONNECTION_TIMEOUT_MS, "80"); + properties.put(HttpClientProperties.APACHE_SOCKET_TIMEOUT_MS, "100"); + properties.put(HttpClientProperties.APACHE_CONNECTION_TIMEOUT_MS, "200"); AwsProperties awsProperties = new AwsProperties(properties); UrlConnectionHttpClientConfigurations urlConnectionHttpClientConfigurations = UrlConnectionHttpClientConfigurations.create(awsProperties.httpClientProperties()); @@ -72,17 +72,17 @@ public void testUrlConnectionDefaultConfigurations() { @Test public void testApacheOverrideConfigurations() { Map properties = Maps.newHashMap(); - properties.put(AwsProperties.HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS, "90"); - properties.put(AwsProperties.HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS, "80"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_SOCKET_TIMEOUT_MS, "100"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_TIMEOUT_MS, "200"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS, "101"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_MAX_IDLE_TIME_MS, "102"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_CONNECTION_TIME_TO_LIVE_MS, "103"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_EXPECT_CONTINUE_ENABLED, "true"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_MAX_CONNECTIONS, "104"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_TCP_KEEP_ALIVE_ENABLED, "true"); - properties.put(AwsProperties.HTTP_CLIENT_APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED, "false"); + properties.put(HttpClientProperties.URLCONNECTION_SOCKET_TIMEOUT_MS, "90"); + properties.put(HttpClientProperties.URLCONNECTION_CONNECTION_TIMEOUT_MS, "80"); + properties.put(HttpClientProperties.APACHE_SOCKET_TIMEOUT_MS, "100"); + properties.put(HttpClientProperties.APACHE_CONNECTION_TIMEOUT_MS, "200"); + properties.put(HttpClientProperties.APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS, "101"); + properties.put(HttpClientProperties.APACHE_CONNECTION_MAX_IDLE_TIME_MS, "102"); + properties.put(HttpClientProperties.APACHE_CONNECTION_TIME_TO_LIVE_MS, "103"); + properties.put(HttpClientProperties.APACHE_EXPECT_CONTINUE_ENABLED, "true"); + properties.put(HttpClientProperties.APACHE_MAX_CONNECTIONS, "104"); + properties.put(HttpClientProperties.APACHE_TCP_KEEP_ALIVE_ENABLED, "true"); + properties.put(HttpClientProperties.APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED, "false"); AwsProperties awsProperties = new AwsProperties(properties); ApacheHttpClientConfigurations apacheHttpClientConfigurations = ApacheHttpClientConfigurations.create(awsProperties.httpClientProperties()); diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientProperties.java new file mode 100644 index 000000000000..df338a5d2aea --- /dev/null +++ b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientProperties.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.aws; + +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +public class TestHttpClientProperties { + + @Test + public void testUrlHttpClientConfiguration() { + Map properties = Maps.newHashMap(); + properties.put(HttpClientProperties.CLIENT_TYPE, "urlconnection"); + HttpClientProperties httpProperties = new HttpClientProperties(properties); + S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); + ArgumentCaptor httpClientBuilderCaptor = + ArgumentCaptor.forClass(SdkHttpClient.Builder.class); + + httpProperties.applyHttpClientConfigurations(mockS3ClientBuilder); + Mockito.verify(mockS3ClientBuilder).httpClientBuilder(httpClientBuilderCaptor.capture()); + SdkHttpClient.Builder capturedHttpClientBuilder = httpClientBuilderCaptor.getValue(); + + Assertions.assertThat(capturedHttpClientBuilder) + .as("Should use url connection http client") + .isInstanceOf(UrlConnectionHttpClient.Builder.class); + } + + @Test + public void testApacheHttpClientConfiguration() { + Map properties = Maps.newHashMap(); + properties.put(HttpClientProperties.CLIENT_TYPE, "apache"); + HttpClientProperties httpProperties = new HttpClientProperties(properties); + S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); + ArgumentCaptor httpClientBuilderCaptor = + ArgumentCaptor.forClass(SdkHttpClient.Builder.class); + + httpProperties.applyHttpClientConfigurations(mockS3ClientBuilder); + Mockito.verify(mockS3ClientBuilder).httpClientBuilder(httpClientBuilderCaptor.capture()); + SdkHttpClient.Builder capturedHttpClientBuilder = httpClientBuilderCaptor.getValue(); + Assertions.assertThat(capturedHttpClientBuilder) + .as("Should use apache http client") + .isInstanceOf(ApacheHttpClient.Builder.class); + } + + @Test + public void testInvalidHttpClientType() { + Map properties = Maps.newHashMap(); + properties.put(HttpClientProperties.CLIENT_TYPE, "test"); + HttpClientProperties httpProperties = new HttpClientProperties(properties); + S3ClientBuilder s3ClientBuilder = S3Client.builder(); + + Assertions.assertThatThrownBy( + () -> httpProperties.applyHttpClientConfigurations(s3ClientBuilder)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unrecognized HTTP client type test"); + } +} diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java new file mode 100644 index 000000000000..4f9d879a7e10 --- /dev/null +++ b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.aws; + +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.ObjectCannedACL; + +public class TestS3FileIOProperties { + + @Test + public void testS3FileIoSseCustom_mustHaveCustomKey() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.SSE_TYPE, S3FileIOProperties.SSE_TYPE_CUSTOM); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot initialize SSE-C S3FileIO with null encryption key"); + } + + @Test + public void testS3FileIoSseCustom_mustHaveCustomMd5() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.SSE_TYPE, S3FileIOProperties.SSE_TYPE_CUSTOM); + map.put(S3FileIOProperties.SSE_KEY, "something"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot initialize SSE-C S3FileIO with null encryption key MD5"); + } + + @Test + public void testS3FileIoAcl() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.ACL, ObjectCannedACL.AUTHENTICATED_READ.toString()); + S3FileIOProperties properties = new S3FileIOProperties(map); + Assertions.assertThat(properties.acl()).isEqualTo(ObjectCannedACL.AUTHENTICATED_READ); + } + + @Test + public void testS3FileIoAcl_unknownType() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.ACL, "bad-input"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot support S3 CannedACL bad-input"); + } + + @Test + public void testS3MultipartSizeTooSmall() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.MULTIPART_SIZE, "1"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Minimum multipart upload object size must be larger than 5 MB."); + } + + @Test + public void testS3MultipartSizeTooLarge() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.MULTIPART_SIZE, "5368709120"); // 5GB + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Input malformed or exceeded maximum multipart upload size 5GB: 5368709120"); + } + + @Test + public void testS3MultipartThresholdFactorLessThanOne() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.MULTIPART_THRESHOLD_FACTOR, "0.9"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Multipart threshold factor must be >= to 1.0"); + } + + @Test + public void testS3FileIoDeleteBatchSizeTooLarge() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.DELETE_BATCH_SIZE, "2000"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Deletion batch size must be between 1 and 1000"); + } + + @Test + public void testS3FileIoDeleteBatchSizeTooSmall() { + Map map = Maps.newHashMap(); + map.put(S3FileIOProperties.DELETE_BATCH_SIZE, "0"); + + Assertions.assertThatThrownBy(() -> new S3FileIOProperties(map)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Deletion batch size must be between 1 and 1000"); + } + + @Test + public void testS3FileIoDefaultCredentialsConfiguration() { + // set nothing + Map properties = Maps.newHashMap(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); + AwsClientProperties awsClientProperties = new AwsClientProperties(properties); + S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); + ArgumentCaptor awsCredentialsProviderCaptor = + ArgumentCaptor.forClass(AwsCredentialsProvider.class); + + s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, mockS3ClientBuilder); + Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); + AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); + + Assertions.assertThat(capturedAwsCredentialsProvider) + .as("Should use default credentials if nothing is set") + .isInstanceOf(DefaultCredentialsProvider.class); + } + + @Test + public void testS3FileIoBasicCredentialsConfiguration() { + // set access key id and secret access key + Map properties = Maps.newHashMap(); + properties.put(S3FileIOProperties.ACCESS_KEY_ID, "key"); + properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, "secret"); + S3FileIOProperties s3PropertiesTwoSet = new S3FileIOProperties(properties); + AwsClientProperties awsClientProperties = new AwsClientProperties(properties); + S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); + ArgumentCaptor awsCredentialsProviderCaptor = + ArgumentCaptor.forClass(AwsCredentialsProvider.class); + + s3PropertiesTwoSet.applyCredentialConfigurations(awsClientProperties, mockS3ClientBuilder); + Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); + AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); + + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials()) + .as("Should use basic credentials if access key ID and secret access key are set") + .isInstanceOf(AwsBasicCredentials.class); + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().accessKeyId()) + .as("The access key id should be the same as the one set by tag S3FILEIO_ACCESS_KEY_ID") + .isEqualTo("key"); + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().secretAccessKey()) + .as( + "The secret access key should be the same as the one set by tag S3FILEIO_SECRET_ACCESS_KEY") + .isEqualTo("secret"); + } + + @Test + public void testS3FileIoSessionCredentialsConfiguration() { + // set access key id, secret access key, and session token + Map properties = Maps.newHashMap(); + properties.put(S3FileIOProperties.ACCESS_KEY_ID, "key"); + properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, "secret"); + properties.put(S3FileIOProperties.SESSION_TOKEN, "token"); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + AwsClientProperties awsClientProperties = new AwsClientProperties(properties); + S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); + ArgumentCaptor awsCredentialsProviderCaptor = + ArgumentCaptor.forClass(AwsCredentialsProvider.class); + + s3Properties.applyCredentialConfigurations(awsClientProperties, mockS3ClientBuilder); + Mockito.verify(mockS3ClientBuilder).credentialsProvider(awsCredentialsProviderCaptor.capture()); + AwsCredentialsProvider capturedAwsCredentialsProvider = awsCredentialsProviderCaptor.getValue(); + + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials()) + .as("Should use session credentials if session token is set") + .isInstanceOf(AwsSessionCredentials.class); + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().accessKeyId()) + .as("The access key id should be the same as the one set by tag S3FILEIO_ACCESS_KEY_ID") + .isEqualTo("key"); + Assertions.assertThat(capturedAwsCredentialsProvider.resolveCredentials().secretAccessKey()) + .as( + "The secret access key should be the same as the one set by tag S3FILEIO_SECRET_ACCESS_KEY") + .isEqualTo("secret"); + } + + @Test + public void testS3RemoteSignerWithoutUri() { + Map properties = + ImmutableMap.of(S3FileIOProperties.REMOTE_SIGNING_ENABLED, "true"); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + + Assertions.assertThatThrownBy(() -> s3Properties.applySignerConfiguration(S3Client.builder())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("S3 signer service URI is required"); + } + + @Test + public void testS3RemoteSigningEnabled() { + String uri = "http://localhost:12345"; + Map properties = + ImmutableMap.of( + S3FileIOProperties.REMOTE_SIGNING_ENABLED, "true", CatalogProperties.URI, uri); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + S3ClientBuilder builder = S3Client.builder(); + + s3Properties.applySignerConfiguration(builder); + + Optional signer = + builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.SIGNER); + Assertions.assertThat(signer).isPresent().get().isInstanceOf(S3V4RestSignerClient.class); + S3V4RestSignerClient signerClient = (S3V4RestSignerClient) signer.get(); + Assertions.assertThat(signerClient.baseSignerUri()).isEqualTo(uri); + Assertions.assertThat(signerClient.properties()).isEqualTo(properties); + } + + @Test + public void testS3RemoteSigningDisabled() { + Map properties = + ImmutableMap.of(S3FileIOProperties.REMOTE_SIGNING_ENABLED, "false"); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + S3ClientBuilder builder = S3Client.builder(); + + s3Properties.applySignerConfiguration(builder); + + Optional signer = + builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.SIGNER); + Assertions.assertThat(signer).isNotPresent(); + } +} diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java index 79c4d8994c3a..47aa2ad3d733 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java @@ -25,6 +25,7 @@ import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.Schema; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; @@ -79,6 +80,7 @@ public void before() { CATALOG_NAME, WAREHOUSE_PATH, new AwsProperties(), + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -91,6 +93,7 @@ public void testConstructorEmptyWarehousePath() { CATALOG_NAME, null, new AwsProperties(), + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -115,6 +118,7 @@ public void testConstructorWarehousePathWithEndSlash() { CATALOG_NAME, WAREHOUSE_PATH + "/", new AwsProperties(), + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -153,11 +157,13 @@ public void testDefaultWarehouseLocationCustomCatalogId() { GlueCatalog catalogWithCustomCatalogId = new GlueCatalog(); String catalogId = "myCatalogId"; AwsProperties awsProperties = new AwsProperties(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(); awsProperties.setGlueCatalogId(catalogId); catalogWithCustomCatalogId.initialize( CATALOG_NAME, WAREHOUSE_PATH + "/", awsProperties, + s3FileIOProperties, glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -592,6 +598,7 @@ public void testTablePropsDefinedAtCatalogLevel() { CATALOG_NAME, WAREHOUSE_PATH, new AwsProperties(), + new S3FileIOProperties(), glue, LockManagers.defaultLockManager(), catalogProps); @@ -608,11 +615,13 @@ public void testTablePropsDefinedAtCatalogLevel() { @Test public void testValidateIdentifierSkipNameValidation() { AwsProperties props = new AwsProperties(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(); props.setGlueCatalogSkipNameValidation(true); glueCatalog.initialize( CATALOG_NAME, WAREHOUSE_PATH, props, + s3FileIOProperties, glue, LockManagers.defaultLockManager(), ImmutableMap.of()); @@ -624,15 +633,17 @@ public void testValidateIdentifierSkipNameValidation() { public void testTableLevelS3TagProperties() { Map properties = ImmutableMap.of( - AwsProperties.S3_WRITE_TABLE_TAG_ENABLED, + S3FileIOProperties.WRITE_TABLE_TAG_ENABLED, "true", - AwsProperties.S3_WRITE_NAMESPACE_TAG_ENABLED, + S3FileIOProperties.WRITE_NAMESPACE_TAG_ENABLED, "true"); AwsProperties awsProperties = new AwsProperties(properties); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); glueCatalog.initialize( CATALOG_NAME, WAREHOUSE_PATH, awsProperties, + s3FileIOProperties, glue, LockManagers.defaultLockManager(), properties); @@ -643,9 +654,11 @@ public void testTableLevelS3TagProperties() { Assertions.assertThat(tableCatalogProperties) .containsEntry( - AwsProperties.S3_WRITE_TAGS_PREFIX.concat(AwsProperties.S3_TAG_ICEBERG_TABLE), "table") + S3FileIOProperties.WRITE_TAGS_PREFIX.concat(S3FileIOProperties.S3_TAG_ICEBERG_TABLE), + "table") .containsEntry( - AwsProperties.S3_WRITE_TAGS_PREFIX.concat(AwsProperties.S3_TAG_ICEBERG_NAMESPACE), + S3FileIOProperties.WRITE_TAGS_PREFIX.concat( + S3FileIOProperties.S3_TAG_ICEBERG_NAMESPACE), "db"); } }