From c905c3f324c9392950f86d88b0bc53c18927bc14 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 2 Jan 2024 22:53:55 +0800 Subject: [PATCH] [#1293] improvement(IT): Change to use api method instead of DTOs (#1294) ### What changes were proposed in this pull request? Remove the usage of DTOs of partitioning, sort ordering and bucketing. ### Why are the changes needed? We should not use DTOs when using Gravition java client. Fix: #1293 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing changes can cover it. --- .../test/catalog/hive/CatalogHiveIT.java | 47 +++++++-------- .../catalog/jdbc/mysql/CatalogMysqlIT.java | 3 +- .../jdbc/postgresql/CatalogPostgreSqlIT.java | 4 +- .../lakehouse/iceberg/CatalogIcebergIT.java | 57 ++++++++----------- 4 files changed, 46 insertions(+), 65 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index 06ae0509897..5582698aeff 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -36,8 +36,6 @@ import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.dto.rel.DistributionDTO; -import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO; import com.datastrato.gravitino.dto.rel.partitions.IdentityPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; @@ -53,12 +51,14 @@ import com.datastrato.gravitino.rel.SchemaChange; import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.distributions.Strategy; import com.datastrato.gravitino.rel.expressions.sorts.NullOrdering; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; @@ -337,20 +337,16 @@ public void testCreateHiveTableWithDistributionAndSortOrder() NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); - DistributionDTO distribution = - new DistributionDTO.Builder() - .withNumber(10) - .withArgs(FieldReferenceDTO.of(HIVE_COL_NAME1)) - .withStrategy(Strategy.EVEN) - .build(); - final SortOrderDTO[] sortOrders = - new SortOrderDTO[] { - new SortOrderDTO.Builder() - .withNullOrder(NullOrdering.NULLS_FIRST) - .withDirection(SortDirection.DESCENDING) - .withSortTerm(FieldReferenceDTO.of(HIVE_COL_NAME2)) - .build() + Distribution distribution = + Distributions.of(Strategy.EVEN, 10, NamedReference.field(HIVE_COL_NAME1)); + + final SortOrder[] sortOrders = + new SortOrder[] { + SortOrders.of( + NamedReference.field(HIVE_COL_NAME2), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST) }; Map properties = createProperties(); @@ -396,12 +392,8 @@ public void testCreateHiveTableWithDistributionAndSortOrder() // Test bad request // Bad name in distribution - final DistributionDTO badDistribution = - new DistributionDTO.Builder() - .withNumber(10) - .withArgs(FieldReferenceDTO.of(HIVE_COL_NAME1 + "bad_name")) - .withStrategy(Strategy.EVEN) - .build(); + final Distribution badDistribution = + Distributions.of(Strategy.EVEN, 10, NamedReference.field(HIVE_COL_NAME1 + "bad_name")); Assertions.assertThrows( Exception.class, () -> { @@ -417,13 +409,12 @@ public void testCreateHiveTableWithDistributionAndSortOrder() sortOrders); }); - final SortOrderDTO[] badSortOrders = - new SortOrderDTO[] { - new SortOrderDTO.Builder() - .withNullOrder(NullOrdering.NULLS_FIRST) - .withDirection(SortDirection.DESCENDING) - .withSortTerm(FieldReferenceDTO.of(HIVE_COL_NAME2 + "bad_name")) - .build() + final SortOrder[] badSortOrders = + new SortOrder[] { + SortOrders.of( + NamedReference.field(HIVE_COL_NAME2 + "bad_name"), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST) }; Assertions.assertThrows( diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index 1f0c2261989..2d22ff13ec8 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NotFoundException; @@ -278,7 +277,7 @@ void testCreateAndLoadMysqlTable() { NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); Distribution distribution = Distributions.NONE; - final SortOrderDTO[] sortOrders = SortOrderDTO.EMPTY_SORT; + final SortOrder[] sortOrders = new SortOrder[0]; Partitioning[] partitioning = Partitioning.EMPTY_PARTITIONING; diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index 7459882da13..e0d4e75a46b 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -11,7 +11,6 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; @@ -281,8 +280,7 @@ void testCreateAndLoadPostgreSqlTable() { NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); Distribution distribution = Distributions.NONE; - final SortOrderDTO[] sortOrders = SortOrderDTO.EMPTY_SORT; - + SortOrder[] sortOrders = new SortOrder[0]; Partitioning[] partitioning = Partitioning.EMPTY_PARTITIONING; Map properties = createProperties(); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index 1f9ad9058e7..789a81ab56f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -17,8 +17,6 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.dto.rel.SortOrderDTO; -import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO; import com.datastrato.gravitino.dto.rel.partitions.DayPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.IdentityPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; @@ -36,11 +34,15 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.sorts.NullOrdering; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -335,13 +337,12 @@ void testCreateAndLoadIcebergTable() { NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); Distribution distribution = Distributions.NONE; - final SortOrderDTO[] sortOrders = - new SortOrderDTO[] { - new SortOrderDTO.Builder() - .withNullOrder(NullOrdering.NULLS_FIRST) - .withDirection(SortDirection.DESCENDING) - .withSortTerm(FieldReferenceDTO.of(ICEBERG_COL_NAME2)) - .build() + final SortOrder[] sortOrders = + new SortOrder[] { + SortOrders.of( + NamedReference.field(ICEBERG_COL_NAME2), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST) }; Partitioning[] partitioning = new Partitioning[] {DayPartitioningDTO.of(columns[1].name())}; @@ -640,21 +641,15 @@ public void testAlterIcebergTable() { void testPartitionAndSortOrderIcebergTable() { ColumnDTO[] columns = createColumns(); String testTableName = GravitinoITUtils.genRandomName("test_table"); - SortOrderDTO[] sortOrders = { - new SortOrderDTO.Builder() - .withSortTerm(FieldReferenceDTO.of(columns[0].name())) - .withDirection(SortDirection.ASCENDING) - .withNullOrder(NullOrdering.NULLS_FIRST) - .build(), - new SortOrderDTO.Builder() - .withSortTerm(FieldReferenceDTO.of(columns[2].name())) - .withDirection(SortDirection.DESCENDING) - .withNullOrder(NullOrdering.NULLS_LAST) - .build() + SortOrder[] sortOrders = { + SortOrders.ascending(NamedReference.field(columns[0].name())), + SortOrders.descending(NamedReference.field(columns[2].name())) }; - Partitioning[] partitioning = { - DayPartitioningDTO.of(columns[1].name()), IdentityPartitioningDTO.of(columns[2].name()) + + Transform[] partitioning = { + Transforms.day(columns[1].name()), Transforms.identity(columns[2].name()) }; + catalog .asTableCatalog() .createTable( @@ -692,16 +687,14 @@ void testOperationDataIcebergTable() { ColumnDTO[] columns = createColumns(); String testTableName = GravitinoITUtils.genRandomName("test_table"); SortOrder[] sortOrders = { - new SortOrderDTO.Builder() - .withSortTerm(FieldReferenceDTO.of(columns[0].name())) - .withDirection(SortDirection.ASCENDING) - .withNullOrder(NullOrdering.NULLS_FIRST) - .build(), - new SortOrderDTO.Builder() - .withSortTerm(FieldReferenceDTO.of(columns[2].name())) - .withDirection(SortDirection.DESCENDING) - .withNullOrder(NullOrdering.NULLS_LAST) - .build() + SortOrders.of( + NamedReference.field(columns[0].name()), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST), + SortOrders.of( + NamedReference.field(columns[2].name()), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST), }; Partitioning[] transforms = { DayPartitioningDTO.of(columns[1].name()), IdentityPartitioningDTO.of(columns[2].name())