diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java index 8f64b678cef..60eddebe5d5 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.client; import static com.datastrato.gravitino.dto.rel.partitioning.Partitioning.EMPTY_PARTITIONING; +import static com.datastrato.gravitino.dto.util.DTOConverters.fromDTOs; import static com.datastrato.gravitino.rel.expressions.sorts.SortDirection.DESCENDING; import static org.apache.hc.core5.http.HttpStatus.SC_BAD_REQUEST; import static org.apache.hc.core5.http.HttpStatus.SC_CONFLICT; @@ -50,6 +51,7 @@ import com.datastrato.gravitino.exceptions.RESTException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.SupportsSchemas; import com.datastrato.gravitino.rel.Table; @@ -372,7 +374,8 @@ public void testCreateTable() throws JsonProcessingException { Table table = catalog .asTableCatalog() - .createTable(tableId, columns, "comment", Collections.emptyMap(), sortOrderDTOs); + .createTable( + tableId, fromDTOs(columns), "comment", Collections.emptyMap(), sortOrderDTOs); Assertions.assertEquals(expectedTable.name(), table.name()); Assertions.assertEquals(expectedTable.comment(), table.comment()); Assertions.assertEquals(expectedTable.properties(), table.properties()); @@ -388,14 +391,15 @@ public void testCreateTable() throws JsonProcessingException { assertTableEquals(expectedTable, table); // test validate column default value - ColumnDTO[] errorColumns = - new ColumnDTO[] { - createMockColumn("col1", Types.ByteType.get(), "comment1"), - createMockColumn( + Column[] errorColumns = + new Column[] { + Column.of("col1", Types.ByteType.get(), "comment1"), + Column.of( "col2", Types.StringType.get(), "comment2", false, + false, new LiteralDTO.Builder().withValue(null).withDataType(Types.NullType.get()).build()) }; @@ -420,7 +424,9 @@ public void testCreateTable() throws JsonProcessingException { Throwable ex = Assertions.assertThrows( NoSuchSchemaException.class, - () -> tableCatalog.createTable(tableId, columns, "comment", emptyMap, sortOrder)); + () -> + tableCatalog.createTable( + tableId, fromDTOs(columns), "comment", emptyMap, sortOrder)); Assertions.assertTrue(ex.getMessage().contains("schema not found")); // Test throw TableAlreadyExistsException @@ -432,7 +438,9 @@ public void testCreateTable() throws JsonProcessingException { Throwable ex1 = Assertions.assertThrows( TableAlreadyExistsException.class, - () -> tableCatalog.createTable(tableId, columns, "comment", emptyMap, sortOrder)); + () -> + tableCatalog.createTable( + tableId, fromDTOs(columns), "comment", emptyMap, sortOrder)); Assertions.assertTrue(ex1.getMessage().contains("table already exists")); } @@ -474,7 +482,8 @@ public void testCreatePartitionedTable() throws JsonProcessingException { Table table = catalog .asTableCatalog() - .createTable(tableId, columns, "comment", Collections.emptyMap(), EMPTY_PARTITIONING); + .createTable( + tableId, fromDTOs(columns), "comment", Collections.emptyMap(), EMPTY_PARTITIONING); assertTableEquals(expectedTable, table); // Test partitioning @@ -507,7 +516,8 @@ public void testCreatePartitionedTable() throws JsonProcessingException { table = catalog .asTableCatalog() - .createTable(tableId, columns, "comment", Collections.emptyMap(), partitioning); + .createTable( + tableId, fromDTOs(columns), "comment", Collections.emptyMap(), partitioning); assertTableEquals(expectedTable, table); // Test throw TableAlreadyExistsException @@ -521,7 +531,9 @@ public void testCreatePartitionedTable() throws JsonProcessingException { Throwable ex1 = Assertions.assertThrows( TableAlreadyExistsException.class, - () -> tableCatalog.createTable(tableId, columns, "comment", emptyMap, partitioning)); + () -> + tableCatalog.createTable( + tableId, fromDTOs(columns), "comment", emptyMap, partitioning)); Assertions.assertTrue(ex1.getMessage().contains("table already exists")); // Test partitioning field not exist in table @@ -530,7 +542,8 @@ public void testCreatePartitionedTable() throws JsonProcessingException { Assertions.assertThrows( IllegalArgumentException.class, () -> - tableCatalog.createTable(tableId, columns, "comment", emptyMap, errorPartitioning)); + tableCatalog.createTable( + tableId, fromDTOs(columns), "comment", emptyMap, errorPartitioning)); Assertions.assertTrue(ex2.getMessage().contains("not found in table")); // Test empty columns @@ -539,7 +552,7 @@ public void testCreatePartitionedTable() throws JsonProcessingException { IllegalArgumentException.class, () -> tableCatalog.createTable( - tableId, new ColumnDTO[0], "comment", emptyMap, errorPartitioning)); + tableId, new Column[0], "comment", emptyMap, errorPartitioning)); Assertions.assertTrue( ex3.getMessage().contains("\"columns\" field is required and cannot be empty")); } @@ -598,7 +611,7 @@ public void testCreateIndexTable() throws JsonProcessingException { .asTableCatalog() .createTable( tableId, - columns, + fromDTOs(columns), "comment", Collections.emptyMap(), EMPTY_PARTITIONING, @@ -621,7 +634,7 @@ public void testCreateIndexTable() throws JsonProcessingException { () -> tableCatalog.createTable( tableId, - columns, + fromDTOs(columns), "comment", emptyMap, EMPTY_PARTITIONING, diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java index 2e659390e13..4db834a9d70 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.client; +import static com.datastrato.gravitino.dto.util.DTOConverters.fromDTOs; import static com.datastrato.gravitino.dto.util.DTOConverters.toDTO; import static org.apache.hc.core5.http.HttpStatus.SC_OK; import static org.apache.http.HttpStatus.SC_CONFLICT; @@ -115,7 +116,7 @@ public static void setUp() throws Exception { .asTableCatalog() .createTable( tableId, - columns, + fromDTOs(columns), "comment", Collections.emptyMap(), partitioning, diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index 6613b285b9a..f3aeda4f4e7 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -37,7 +37,6 @@ import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO; import com.datastrato.gravitino.dto.rel.partitioning.IdentityPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; @@ -49,6 +48,7 @@ import com.datastrato.gravitino.integration.test.container.HiveContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.SchemaChange; import com.datastrato.gravitino.rel.SupportsSchemas; @@ -275,26 +275,11 @@ private static void createSchema() throws TException, InterruptedException { Assertions.assertEquals("val2", database.getParameters().get("key2")); } - private ColumnDTO[] createColumns() { - ColumnDTO col1 = - new ColumnDTO.Builder<>() - .withName(HIVE_COL_NAME1) - .withDataType(Types.ByteType.get()) - .withComment("col_1_comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder<>() - .withName(HIVE_COL_NAME2) - .withDataType(Types.DateType.get()) - .withComment("col_2_comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder<>() - .withName(HIVE_COL_NAME3) - .withDataType(Types.StringType.get()) - .withComment("col_3_comment") - .build(); - return new ColumnDTO[] {col1, col2, col3}; + private Column[] createColumns() { + Column col1 = Column.of(HIVE_COL_NAME1, Types.ByteType.get(), "col_1_comment"); + Column col2 = Column.of(HIVE_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(HIVE_COL_NAME3, Types.StringType.get(), "col_3_comment"); + return new Column[] {col1, col2, col3}; } private void checkTableReadWrite(org.apache.hadoop.hive.metastore.api.Table table) { @@ -345,7 +330,7 @@ private Map createProperties() { public void testCreateHiveTableWithDistributionAndSortOrder() throws TException, InterruptedException { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -445,7 +430,7 @@ public void testCreateHiveTableWithDistributionAndSortOrder() @Test public void testCreateHiveTable() throws TException, InterruptedException { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -498,7 +483,7 @@ public void testCreateHiveTable() throws TException, InterruptedException { @Test public void testHiveTableProperties() throws TException, InterruptedException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); // test default properties @@ -616,7 +601,7 @@ public void testHiveSchemaProperties() throws TException, InterruptedException { @Test public void testCreatePartitionedHiveTable() throws TException, InterruptedException { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -667,7 +652,7 @@ public void testCreatePartitionedHiveTable() throws TException, InterruptedExcep @Test public void testListPartitionNames() throws TException, InterruptedException { // test empty partitions - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); Table nonPartitionedTable = @@ -694,7 +679,7 @@ public void testListPartitionNames() throws TException, InterruptedException { @Test public void testListPartitions() throws TException, InterruptedException { // test empty partitions - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); Table nonPartitionedTable = @@ -821,7 +806,7 @@ public void testAddPartition() throws TException, InterruptedException { } private Table preparePartitionedTable() throws TException, InterruptedException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier nameIdentifier = NameIdentifier.of( @@ -913,7 +898,7 @@ void testAlterUnknownTable() { @Test public void testAlterHiveTable() throws TException, InterruptedException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); Table createdTable = catalog .asTableCatalog() @@ -983,25 +968,11 @@ public void testAlterHiveTable() throws TException, InterruptedException { Assertions.assertTrue(exception.getMessage().contains("Cannot alter partition column")); // test updateColumnPosition exception - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("name") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName("address") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName("date_of_birth") - .withDataType(Types.DateType.get()) - .withComment("comment") - .build(); - ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; + Column col1 = Column.of("name", Types.StringType.get(), "comment"); + Column col2 = Column.of("address", Types.StringType.get(), "comment"); + Column col3 = Column.of("date_of_birth", Types.DateType.get(), "comment"); + + Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = NameIdentifier.of( metalakeName, @@ -1218,7 +1189,7 @@ void testAlterEntityName() { // Now try to rename table final String tableName = GravitinoITUtils.genRandomName("CatalogHiveIT_table"); final String newTableName = GravitinoITUtils.genRandomName("CatalogHiveIT_table_new"); - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( @@ -1278,7 +1249,7 @@ void testDropAndRename() { @Test public void testDropHiveManagedTable() throws TException, InterruptedException, IOException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( @@ -1303,7 +1274,7 @@ public void testDropHiveManagedTable() throws TException, InterruptedException, @Test public void testDropHiveExternalTable() throws TException, InterruptedException, IOException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( @@ -1330,7 +1301,7 @@ public void testDropHiveExternalTable() throws TException, InterruptedException, @Test public void testPurgeHiveManagedTable() throws TException, InterruptedException, IOException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( @@ -1357,7 +1328,7 @@ public void testPurgeHiveManagedTable() throws TException, InterruptedException, @Test public void testPurgeHiveExternalTable() throws TException, InterruptedException, IOException { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestMultipleJdbcLoad.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestMultipleJdbcLoad.java index c726dfc8c6b..e1beffc1651 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestMultipleJdbcLoad.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestMultipleJdbcLoad.java @@ -13,13 +13,13 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.integration.test.catalog.jdbc.mysql.CatalogMysqlIT; import com.datastrato.gravitino.integration.test.catalog.jdbc.postgresql.CatalogPostgreSqlIT; import com.datastrato.gravitino.integration.test.catalog.jdbc.utils.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; import java.io.IOException; @@ -142,18 +142,13 @@ public void testCreateMultipleJdbc() throws URISyntaxException { String tableName = GravitinoITUtils.genRandomName("it_table"); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("col_1") - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); + Column col1 = Column.of("col_1", Types.IntegerType.get(), "col_1_comment"); String comment = "test"; mysqlCatalog .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName), - new ColumnDTO[] {col1}, + new Column[] {col1}, comment, Collections.emptyMap()); @@ -161,7 +156,7 @@ public void testCreateMultipleJdbc() throws URISyntaxException { .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName), - new ColumnDTO[] {col1}, + new Column[] {col1}, comment, Collections.emptyMap()); @@ -243,18 +238,13 @@ public void testCreateMultipleJdbcInIceberg() throws URISyntaxException { String tableName = GravitinoITUtils.genRandomName("it_table"); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("col_1") - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); + Column col1 = Column.of("col_1", Types.IntegerType.get(), "col_1_comment"); String comment = "test"; mysqlCatalog .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName), - new ColumnDTO[] {col1}, + new Column[] {col1}, comment, Collections.emptyMap()); @@ -262,7 +252,7 @@ public void testCreateMultipleJdbcInIceberg() throws URISyntaxException { .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName), - new ColumnDTO[] {col1}, + new Column[] {col1}, comment, Collections.emptyMap()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java index e854732469a..ea2c50c9c37 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java @@ -12,12 +12,12 @@ import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.integration.test.catalog.jdbc.mysql.service.MysqlService; import com.datastrato.gravitino.integration.test.catalog.jdbc.utils.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableChange; @@ -117,12 +117,7 @@ public void testAuditTable() throws Exception { Catalog catalog = createCatalog(catalogName); Map properties = Maps.newHashMap(); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("col_1") - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); + Column col1 = Column.of("col_1", Types.IntegerType.get(), "col_1_comment"); catalog .asSchemas() @@ -132,7 +127,7 @@ public void testAuditTable() throws Exception { .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - new ColumnDTO[] {col1}, + new Column[] {col1}, "comment", properties); Assertions.assertEquals(expectUser, table.auditInfo().creator()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index d09e4d84be8..7301346118f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -15,7 +15,6 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NotFoundException; @@ -198,26 +197,12 @@ private void createSchema() { prop.forEach((key, value) -> Assertions.assertEquals(loadSchema.properties().get(key), value)); } - private ColumnDTO[] createColumns() { - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName(MYSQL_COL_NAME1) - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName(MYSQL_COL_NAME2) - .withDataType(Types.DateType.get()) - .withComment("col_2_comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName(MYSQL_COL_NAME3) - .withDataType(Types.StringType.get()) - .withComment("col_3_comment") - .build(); - return new ColumnDTO[] {col1, col2, col3}; + private Column[] createColumns() { + Column col1 = Column.of(MYSQL_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(MYSQL_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(MYSQL_COL_NAME3, Types.StringType.get(), "col_3_comment"); + + return new Column[] {col1, col2, col3}; } private Map createProperties() { @@ -303,7 +288,7 @@ void testOperationMysqlSchema() { @Test void testCreateAndLoadMysqlTable() { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -353,44 +338,15 @@ void testCreateAndLoadMysqlTable() { @Test void testColumnNameWithKeyWords() { // Create table from Gravitino API - ColumnDTO[] columns = - new ColumnDTO[] { - new ColumnDTO.Builder() - .withName("integer") - .withDataType(Types.IntegerType.get()) - .withComment("integer") - .build(), - new ColumnDTO.Builder() - .withName("long") - .withDataType(Types.LongType.get()) - .withComment("long") - .build(), - new ColumnDTO.Builder() - .withName("float") - .withDataType(Types.FloatType.get()) - .withComment("float") - .build(), - new ColumnDTO.Builder() - .withName("double") - .withDataType(Types.DoubleType.get()) - .withComment("double") - .build(), - new ColumnDTO.Builder() - .withName("decimal") - .withDataType(Types.DecimalType.of(10, 3)) - .withComment("decimal") - .build(), - new ColumnDTO.Builder() - .withName("date") - .withDataType(Types.DateType.get()) - .withComment("date") - .build(), - new ColumnDTO.Builder() - .withName("time") - .withDataType(Types.TimeType.get()) - .withComment("time") - .build() - }; + Column[] columns = { + Column.of("integer", Types.IntegerType.get(), "integer"), + Column.of("long", Types.LongType.get(), "long"), + Column.of("float", Types.FloatType.get(), "float"), + Column.of("double", Types.DoubleType.get(), "double"), + Column.of("decimal", Types.DecimalType.of(10, 3), "decimal"), + Column.of("date", Types.DateType.get(), "date"), + Column.of("time", Types.TimeType.get(), "time") + }; String name = GravitinoITUtils.genRandomName("table") + "_keyword"; NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, name); @@ -603,7 +559,7 @@ void testColumnDefaultValueConverter() { @Test void testAlterAndDropMysqlTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); catalog .asTableCatalog() .createTable( @@ -663,25 +619,11 @@ void testAlterAndDropMysqlTable() { Assertions.assertNotNull(table.auditInfo().lastModifiedTime()); Assertions.assertNotNull(table.auditInfo().lastModifier()); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("name") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName("address") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName("date_of_birth") - .withDataType(Types.DateType.get()) - .withComment("comment") - .build(); - ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; + Column col1 = Column.of("name", Types.StringType.get(), "comment"); + Column col2 = Column.of("address", Types.StringType.get(), "comment"); + Column col3 = Column.of("date_of_birth", Types.DateType.get(), "comment"); + + Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = NameIdentifier.of( metalakeName, diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index 862ba387434..37c0fbedc74 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -15,7 +15,6 @@ import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.integration.test.catalog.jdbc.postgresql.service.PostgreSqlService; @@ -192,77 +191,31 @@ private void createSchema() { Assertions.assertEquals(createdSchema.comment(), loadSchema.comment()); } - private ColumnDTO[] createColumns() { - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName(POSTGRESQL_COL_NAME1) - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName(POSTGRESQL_COL_NAME2) - .withDataType(Types.DateType.get()) - .withComment("col_2_comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName(POSTGRESQL_COL_NAME3) - .withDataType(Types.StringType.get()) - .withComment("col_3_comment") - .build(); - return new ColumnDTO[] {col1, col2, col3}; + private Column[] createColumns() { + Column col1 = Column.of(POSTGRESQL_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(POSTGRESQL_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(POSTGRESQL_COL_NAME3, Types.StringType.get(), "col_3_comment"); + + return new Column[] {col1, col2, col3}; } - private ColumnDTO[] columnsWithSpecialNames() { - return new ColumnDTO[] { - new ColumnDTO.Builder() - .withName("integer") - .withDataType(Types.IntegerType.get()) - .withComment("integer") - .build(), - new ColumnDTO.Builder() - .withName("long") - .withDataType(Types.LongType.get()) - .withComment("long") - .build(), - new ColumnDTO.Builder() - .withName("float") - .withDataType(Types.FloatType.get()) - .withComment("float") - .build(), - new ColumnDTO.Builder() - .withName("double") - .withDataType(Types.DoubleType.get()) - .withComment("double") - .build(), - new ColumnDTO.Builder() - .withName("decimal") - .withDataType(Types.DecimalType.of(10, 3)) - .withComment("decimal") - .build(), - new ColumnDTO.Builder() - .withName("date") - .withDataType(Types.DateType.get()) - .withComment("date") - .build(), - new ColumnDTO.Builder() - .withName("time") - .withDataType(Types.TimeType.get()) - .withComment("time") - .build(), - new ColumnDTO.Builder() - .withName("binary") - .withDataType(Types.TimestampType.withoutTimeZone()) - .withComment("binary") - .build() + private Column[] columnsWithSpecialNames() { + return new Column[] { + Column.of("integer", Types.IntegerType.get(), "integer"), + Column.of("long", Types.LongType.get(), "long"), + Column.of("float", Types.FloatType.get(), "float"), + Column.of("double", Types.DoubleType.get(), "double"), + Column.of("decimal", Types.DecimalType.of(10, 3), "decimal"), + Column.of("date", Types.DateType.get(), "date"), + Column.of("time", Types.TimeType.get(), "time"), + Column.of("binary", Types.TimestampType.withoutTimeZone(), "binary") }; } @Test void testCreateTableWithSpecialColumnNames() { // Create table from Gravitino API - ColumnDTO[] columns = columnsWithSpecialNames(); + Column[] columns = columnsWithSpecialNames(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -370,7 +323,7 @@ void testOperationPostgreSqlSchema() { @Test void testCreateAndLoadPostgreSqlTable() { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -418,7 +371,7 @@ void testCreateAndLoadPostgreSqlTable() { @Test void testAlterAndDropPostgreSqlTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); Table table = catalog .asTableCatalog() @@ -491,25 +444,11 @@ void testAlterAndDropPostgreSqlTable() { Assertions.assertNotNull(table.auditInfo().lastModifiedTime()); Assertions.assertNotNull(table.auditInfo().lastModifier()); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("name") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName("address") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName("date_of_birth") - .withDataType(Types.DateType.get()) - .withComment("comment") - .build(); - ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; + Column col1 = Column.of("name", Types.StringType.get(), "comment"); + Column col2 = Column.of("address", Types.StringType.get(), "comment"); + Column col3 = Column.of("date_of_birth", Types.DateType.get(), "comment"); + + Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = NameIdentifier.of( metalakeName, catalogName, schemaName, GravitinoITUtils.genRandomName("jdbc_it_table")); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index 78a0ad950bc..63a8ab3acb4 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -16,7 +16,6 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper; import com.datastrato.gravitino.client.GravitinoMetaLake; -import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; @@ -215,25 +214,10 @@ private static void createSchema() { prop.forEach((key, value) -> Assertions.assertEquals(loadSchema.properties().get(key), value)); } - private ColumnDTO[] createColumns() { - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName(ICEBERG_COL_NAME1) - .withDataType(Types.IntegerType.get()) - .withComment("col_1_comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName(ICEBERG_COL_NAME2) - .withDataType(Types.DateType.get()) - .withComment("col_2_comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName(ICEBERG_COL_NAME3) - .withDataType(Types.StringType.get()) - .withComment("col_3_comment") - .build(); + private Column[] createColumns() { + Column col1 = Column.of(ICEBERG_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(ICEBERG_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(ICEBERG_COL_NAME3, Types.StringType.get(), "col_3_comment"); Types.StructType structTypeInside = Types.StructType.of( Types.StructType.Field.notNullField("integer_field_inside", Types.IntegerType.get()), @@ -245,13 +229,8 @@ private ColumnDTO[] createColumns() { Types.StructType.Field.notNullField( "string_field", Types.StringType.get(), "string field"), Types.StructType.Field.nullableField("struct_field", structTypeInside, "struct field")); - ColumnDTO col4 = - new ColumnDTO.Builder() - .withName(ICEBERG_COL_NAME4) - .withDataType(structType) - .withComment("col_4_comment") - .build(); - return new ColumnDTO[] {col1, col2, col3, col4}; + Column col4 = Column.of(ICEBERG_COL_NAME4, structType, "col_4_comment"); + return new Column[] {col1, col2, col3, col4}; } private Map createProperties() { @@ -352,7 +331,7 @@ void testOperationIcebergSchema() { @Test void testCreateTableWithNullComment() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -368,7 +347,7 @@ void testCreateTableWithNullComment() { @Test void testCreateAndLoadIcebergTable() { // Create table from Gravitino API - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); @@ -403,7 +382,7 @@ void testCreateAndLoadIcebergTable() { Assertions.assertEquals(createdTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - Assertions.assertEquals(createdTable.columns()[i], columns[i]); + Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]); } // TODO add partitioning and sort order check @@ -420,7 +399,7 @@ void testCreateAndLoadIcebergTable() { } Assertions.assertEquals(loadTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - Assertions.assertEquals(columns[i], loadTable.columns()[i]); + Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]); } Assertions.assertEquals(partitioning.length, loadTable.partitioning().length); @@ -462,7 +441,7 @@ void testCreateAndLoadIcebergTable() { @Test void testListAndDropIcebergTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier table1 = NameIdentifier.of(metalakeName, catalogName, schemaName, "table_1"); @@ -524,7 +503,7 @@ void testListAndDropIcebergTable() { @Test public void testAlterIcebergTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); Table table = catalog .asTableCatalog() @@ -601,25 +580,11 @@ public void testAlterIcebergTable() { columns[0].name(), ((Partitioning.SingleFieldPartitioning) table.partitioning()[0]).fieldName()[0]); - ColumnDTO col1 = - new ColumnDTO.Builder() - .withName("name") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col2 = - new ColumnDTO.Builder() - .withName("address") - .withDataType(Types.StringType.get()) - .withComment("comment") - .build(); - ColumnDTO col3 = - new ColumnDTO.Builder() - .withName("date_of_birth") - .withDataType(Types.DateType.get()) - .withComment("comment") - .build(); - ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; + Column col1 = Column.of("name", Types.StringType.get(), "comment"); + Column col2 = Column.of("address", Types.StringType.get(), "comment"); + Column col3 = Column.of("date_of_birth", Types.DateType.get(), "comment"); + + Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = NameIdentifier.of( metalakeName, @@ -676,7 +641,7 @@ public void testAlterIcebergTable() { @Test void testPartitionAndSortOrderIcebergTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); String testTableName = GravitinoITUtils.genRandomName("test_table"); SortOrder[] sortOrders = { SortOrders.ascending(NamedReference.field(columns[0].name())), @@ -721,7 +686,7 @@ void testPartitionAndSortOrderIcebergTable() { @Test void testOperationDataIcebergTable() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); String testTableName = GravitinoITUtils.genRandomName("test_table"); SortOrder[] sortOrders = { SortOrders.of( @@ -867,7 +832,7 @@ public void testOperatorSchemeProperties() { @Test public void testTableDistribution() { - ColumnDTO[] columns = createColumns(); + Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index bc0cd6fcc69..5fead123172 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -526,30 +526,30 @@ private static boolean checkTrinoHasLoaded(String sql, long maxWaitTimeSec) } private Column[] createHiveFullTypeColumns() { - Column[] columnDTO = createFullTypeColumns(); + Column[] columns = createFullTypeColumns(); Set unsupportedType = Sets.newHashSet("FixedType", "StringType", "TimeType"); // MySQL doesn't support timestamp time zone - return Arrays.stream(columnDTO) + return Arrays.stream(columns) .filter(c -> !unsupportedType.contains(c.name())) .toArray(Column[]::new); } private Column[] createMySQLFullTypeColumns() { - Column[] columnDTO = createFullTypeColumns(); + Column[] columns = createFullTypeColumns(); Set unsupportedType = Sets.newHashSet("FixedType", "StringType", "TimestampType", "BooleanType"); // MySQL doesn't support timestamp time zone - return Arrays.stream(columnDTO) + return Arrays.stream(columns) .filter(c -> !unsupportedType.contains(c.name())) .toArray(Column[]::new); } private Column[] createIcebergFullTypeColumns() { - Column[] columnDTO = createFullTypeColumns(); + Column[] columns = createFullTypeColumns(); Set unsupportedType = Sets.newHashSet("ByteType", "ShortType", "VarCharType", "FixedCharType"); - return Arrays.stream(columnDTO) + return Arrays.stream(columns) .filter(c -> !unsupportedType.contains(c.name())) .toArray(Column[]::new); } @@ -1220,9 +1220,9 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { } // Create a table with primary key - Column[] columnDTOS = createMySQLFullTypeColumns(); - columnDTOS = - Arrays.stream(columnDTOS) + Column[] columns = createMySQLFullTypeColumns(); + columns = + Arrays.stream(columns) .map( c -> { if ("IntegerType".equals(c.name())) { @@ -1237,7 +1237,7 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { .asTableCatalog() .createTable( NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - columnDTOS, + columns, "Created by gravitino client", ImmutableMap.builder().build(), new Transform[0],