From c7eed597473ae8e7bff3251d0229bd3734804692 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Mon, 13 May 2024 20:52:24 +0800 Subject: [PATCH 01/12] pre-assign partition --- .../doris/operation/DorisTableOperations.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 32b1133523f..6e2574a40ff 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -11,6 +11,7 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.exceptions.NoSuchColumnException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; @@ -18,10 +19,12 @@ import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Strategy; +import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.partitions.RangePartition; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -216,8 +219,19 @@ private static void appendPartitionSql( "The partition field must be one of the columns"); String partitionColumn = BACK_QUOTE + rangePartition.fieldName()[0] + BACK_QUOTE; - // TODO we currently do not support pre-assign partition when creating range partitioning - partitionSqlBuilder.append(partitionColumn).append(") () "); + partitionSqlBuilder.append(partitionColumn).append(") "); + + // pre-assign partition when creating range partitioning + partitionSqlBuilder.append("(").append(NEW_LINE); + RangePartition[] assignments = rangePartition.assignments(); + if (!ArrayUtils.isEmpty(assignments)) { + for (RangePartition part : assignments) { + partitionSqlBuilder.append("PARTITION ").append(part.name()).append(" VALUES "); + Literal lower = part.lower(); + if (LiteralDTO.NULL.equals(lower)) {} + } + } + partitionSqlBuilder.append(")"); } else if (partitioning[0] instanceof Transforms.ListTransform) { Transforms.ListTransform listPartition = (Transforms.ListTransform) partitioning[0]; partitionSqlBuilder.append(" PARTITION BY LIST("); From 519ac38b1d53508ec0cbcbdaf307cc422855cfd7 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 14 May 2024 15:25:39 +0800 Subject: [PATCH 02/12] feat: assign range partition --- .../doris/operation/DorisTableOperations.java | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 6e2574a40ff..4162b4c41e6 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -19,11 +19,11 @@ import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Strategy; -import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.partitions.ListPartition; import com.datastrato.gravitino.rel.partitions.RangePartition; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -50,7 +50,7 @@ public class DorisTableOperations extends JdbcTableOperations { private static final String BACK_QUOTE = "`"; private static final String DORIS_AUTO_INCREMENT = "AUTO_INCREMENT"; - + private static final String DOUBLE_QUOTE = "\""; private static final String NEW_LINE = "\n"; @Override @@ -219,19 +219,50 @@ private static void appendPartitionSql( "The partition field must be one of the columns"); String partitionColumn = BACK_QUOTE + rangePartition.fieldName()[0] + BACK_QUOTE; - partitionSqlBuilder.append(partitionColumn).append(") "); - + partitionSqlBuilder.append(partitionColumn).append(") ").append(NEW_LINE).append("(").append(NEW_LINE); // pre-assign partition when creating range partitioning - partitionSqlBuilder.append("(").append(NEW_LINE); RangePartition[] assignments = rangePartition.assignments(); if (!ArrayUtils.isEmpty(assignments)) { + ImmutableList.Builder partitions = ImmutableList.builder(); for (RangePartition part : assignments) { - partitionSqlBuilder.append("PARTITION ").append(part.name()).append(" VALUES "); - Literal lower = part.lower(); - if (LiteralDTO.NULL.equals(lower)) {} + StringBuilder partitionAssignSqlBuilder = new StringBuilder(); + partitionAssignSqlBuilder.append("PARTITION ").append(part.name()).append(" VALUES "); + LiteralDTO upper = (LiteralDTO) part.upper(); + LiteralDTO lower = (LiteralDTO) part.lower(); + if (LiteralDTO.NULL.equals(upper) && LiteralDTO.NULL.equals(lower)) { + partitionAssignSqlBuilder.append("LESS THAN MAXVALUE"); + } else if (LiteralDTO.NULL.equals(lower)) { + partitionAssignSqlBuilder + .append("LESS THAN (") + .append(DOUBLE_QUOTE) + .append(upper.value()) + .append(DOUBLE_QUOTE) + .append(")"); + } else if (LiteralDTO.NULL.equals(upper)) { + partitionAssignSqlBuilder + .append("[(") + .append(DOUBLE_QUOTE) + .append(lower.value()) + .append(DOUBLE_QUOTE) + .append("), (MAXVALUE))"); + } else { + partitionAssignSqlBuilder + .append("[(") + .append(DOUBLE_QUOTE) + .append(lower.value()) + .append(DOUBLE_QUOTE) + .append("), (") + .append(DOUBLE_QUOTE) + .append(upper.value()) + .append(DOUBLE_QUOTE) + .append("))"); + } + partitions.add(partitionAssignSqlBuilder.toString()); } + partitionSqlBuilder.append( + partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); } - partitionSqlBuilder.append(")"); + partitionSqlBuilder.append(NEW_LINE).append(")"); } else if (partitioning[0] instanceof Transforms.ListTransform) { Transforms.ListTransform listPartition = (Transforms.ListTransform) partitioning[0]; partitionSqlBuilder.append(" PARTITION BY LIST("); @@ -248,8 +279,16 @@ private static void appendPartitionSql( } String partitionColumns = partitionColumnsBuilder.build().stream().collect(Collectors.joining(",")); - // TODO we currently do not support pre-assign partition when creating list partitioning table - partitionSqlBuilder.append(partitionColumns).append(") () "); + partitionSqlBuilder.append(partitionColumns).append(") ").append(NEW_LINE).append("(").append(NEW_LINE); + // pre-assign partition when creating range partitioning + ListPartition[] assignments = listPartition.assignments(); + if (!ArrayUtils.isEmpty(assignments)) { + ImmutableList.Builder partitions = ImmutableList.builder(); + for (ListPartition part : assignments) { + + } + } + partitionSqlBuilder.append(NEW_LINE).append(")"); } else { throw new IllegalArgumentException("Unsupported partition type of Doris"); } From ee9f0bc1f04a4c62cd3d4cff6055f130cd95e626 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Mon, 17 Jun 2024 19:47:36 +0800 Subject: [PATCH 03/12] assign list partition --- .../doris/operation/DorisTableOperations.java | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 4162b4c41e6..3648bf0ee62 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -19,6 +19,7 @@ import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Strategy; +import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; @@ -219,14 +220,19 @@ private static void appendPartitionSql( "The partition field must be one of the columns"); String partitionColumn = BACK_QUOTE + rangePartition.fieldName()[0] + BACK_QUOTE; - partitionSqlBuilder.append(partitionColumn).append(") ").append(NEW_LINE).append("(").append(NEW_LINE); - // pre-assign partition when creating range partitioning + partitionSqlBuilder.append(partitionColumn).append(") ").append(NEW_LINE).append("("); + RangePartition[] assignments = rangePartition.assignments(); if (!ArrayUtils.isEmpty(assignments)) { ImmutableList.Builder partitions = ImmutableList.builder(); for (RangePartition part : assignments) { StringBuilder partitionAssignSqlBuilder = new StringBuilder(); - partitionAssignSqlBuilder.append("PARTITION ").append(part.name()).append(" VALUES "); + partitionAssignSqlBuilder + .append(" PARTITION ") + .append(BACK_QUOTE) + .append(part.name()) + .append(BACK_QUOTE) + .append(" VALUES "); LiteralDTO upper = (LiteralDTO) part.upper(); LiteralDTO lower = (LiteralDTO) part.lower(); if (LiteralDTO.NULL.equals(upper) && LiteralDTO.NULL.equals(lower)) { @@ -259,9 +265,11 @@ private static void appendPartitionSql( } partitions.add(partitionAssignSqlBuilder.toString()); } - partitionSqlBuilder.append( - partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); + partitionSqlBuilder + .append(NEW_LINE) + .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); } + partitionSqlBuilder.append(NEW_LINE).append(")"); } else if (partitioning[0] instanceof Transforms.ListTransform) { Transforms.ListTransform listPartition = (Transforms.ListTransform) partitioning[0]; @@ -279,15 +287,52 @@ private static void appendPartitionSql( } String partitionColumns = partitionColumnsBuilder.build().stream().collect(Collectors.joining(",")); - partitionSqlBuilder.append(partitionColumns).append(") ").append(NEW_LINE).append("(").append(NEW_LINE); - // pre-assign partition when creating range partitioning + partitionSqlBuilder.append(partitionColumns).append(") ").append(NEW_LINE).append("("); + ListPartition[] assignments = listPartition.assignments(); if (!ArrayUtils.isEmpty(assignments)) { ImmutableList.Builder partitions = ImmutableList.builder(); - for (ListPartition part : assignments) { - + for (ListPartition parts : assignments) { + StringBuilder partitionAssignSqlBuilder = new StringBuilder(); + partitionAssignSqlBuilder + .append(" PARTITION ") + .append(BACK_QUOTE) + .append(parts.name()) + .append(BACK_QUOTE) + .append(" VALUES IN ") + .append("("); + ImmutableList.Builder partitionValues = ImmutableList.builder(); + for (Literal[] part : parts.lists()) { + Preconditions.checkArgument( + part.length == filedNames.length, + "The number of partitioning columns must be consistent."); + StringBuilder partitionValuesSqlBuilder = new StringBuilder(); + if (part.length > 1) { + partitionValuesSqlBuilder + .append("(") + .append( + Arrays.stream(part) + .map(p -> DOUBLE_QUOTE + p.value() + DOUBLE_QUOTE) + .collect(Collectors.joining(","))) + .append(")"); + } else { + partitionValuesSqlBuilder + .append(DOUBLE_QUOTE) + .append(part[0].value()) + .append(DOUBLE_QUOTE); + } + partitionValues.add(partitionValuesSqlBuilder.toString()); + } + partitionAssignSqlBuilder + .append(partitionValues.build().stream().collect(Collectors.joining(","))) + .append(")"); + partitions.add(partitionAssignSqlBuilder.toString()); } + partitionSqlBuilder + .append(NEW_LINE) + .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); } + partitionSqlBuilder.append(NEW_LINE).append(")"); } else { throw new IllegalArgumentException("Unsupported partition type of Doris"); From d663e9f81f9c0e47d0622e89726589a38fe7e764 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 18 Jun 2024 11:16:55 +0800 Subject: [PATCH 04/12] add UT --- .../operation/TestDorisTableOperations.java | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java index 5c1c94c137b..bebe635191e 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -6,19 +6,25 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; +import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.partitions.ListPartition; +import com.datastrato.gravitino.rel.partitions.Partitions; +import com.datastrato.gravitino.rel.partitions.RangePartition; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.utils.RandomNameUtils; import com.google.common.collect.Maps; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -439,7 +445,27 @@ public void testCreateTableWithPartition() { // create table with range partition String rangePartitionTableName = GravitinoITUtils.genRandomName("range_partition_table"); - Transform[] rangePartition = new Transform[] {Transforms.range(new String[] {col4.name()})}; + LocalDate today = LocalDate.now(); + LocalDate tomorrow = today.plusDays(1); + LiteralDTO todayLiteral = + LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); + LiteralDTO tomorrowLiteral = + LiteralDTO.builder() + .withDataType(Types.DateType.get()) + .withValue(tomorrow.toString()) + .build(); + RangePartition rangePartition1 = + (RangePartition) Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + RangePartition rangePartition2 = + (RangePartition) Partitions.range("p2", tomorrowLiteral, todayLiteral, null); + RangePartition rangePartition3 = + (RangePartition) Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + Transform[] rangePartition = + new Transform[] { + Transforms.range( + new String[] {col4.name()}, + new RangePartition[] {rangePartition1, rangePartition2, rangePartition3}) + }; TABLE_OPERATIONS.create( databaseName, rangePartitionTableName, @@ -460,8 +486,28 @@ public void testCreateTableWithPartition() { // create table with list partition String listPartitionTableName = GravitinoITUtils.genRandomName("list_partition_table"); + LiteralDTO intLiteral1 = + LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("1").build(); + LiteralDTO intLiteral2 = + LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); + ListPartition listPartition1 = + (ListPartition) + Partitions.list( + "p1", + new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + null); + ListPartition listPartition2 = + (ListPartition) + Partitions.list( + "p2", + new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + null); Transform[] listPartition = - new Transform[] {Transforms.list(new String[] {col1.name()}, new String[] {col4.name()})}; + new Transform[] { + Transforms.list( + new String[][] {{col1.name()}, {col4.name()}}, + new ListPartition[] {listPartition1, listPartition2}) + }; TABLE_OPERATIONS.create( databaseName, listPartitionTableName, From fc9b2181f9deb3d55be06bc0ebcd9271ecb64778 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 18 Jun 2024 14:53:40 +0800 Subject: [PATCH 05/12] add IT --- .../integration/test/CatalogDorisIT.java | 85 +++++++++++++++++-- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 26a87ada3c2..570222c1d53 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.SupportsSchemas; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.integration.test.container.ContainerSuite; @@ -27,14 +28,20 @@ import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.literals.Literal; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.partitions.ListPartition; +import com.datastrato.gravitino.rel.partitions.Partitions; +import com.datastrato.gravitino.rel.partitions.RangePartition; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.utils.RandomNameUtils; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; +import java.time.LocalDate; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -72,6 +79,7 @@ public class CatalogDorisIT extends AbstractIT { public String DORIS_COL_NAME1 = "doris_col_name1"; public String DORIS_COL_NAME2 = "doris_col_name2"; public String DORIS_COL_NAME3 = "doris_col_name3"; + public String DORIS_COL_NAME4 = "doris_col_name4"; // Because the creation of Schema Change is an asynchronous process, we need to wait for a while // For more information, you can refer to the comment in @@ -169,8 +177,65 @@ private Column[] createColumns() { Column col1 = Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10), "col_2_comment"); Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10), "col_3_comment"); + Column col4 = Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment"); - return new Column[] {col1, col2, col3}; + return new Column[] {col1, col2, col3, col4}; + } + + private Transform[] createRangePartition() { + LocalDate today = LocalDate.now(); + LocalDate tomorrow = today.plusDays(1); + LiteralDTO todayLiteral = + LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); + LiteralDTO tomorrowLiteral = + LiteralDTO.builder() + .withDataType(Types.DateType.get()) + .withValue(tomorrow.toString()) + .build(); + RangePartition rangePartition1 = + (RangePartition) Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + RangePartition rangePartition2 = + (RangePartition) Partitions.range("p2", tomorrowLiteral, todayLiteral, null); + RangePartition rangePartition3 = + (RangePartition) Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + return new Transform[] { + Transforms.range( + new String[] {DORIS_COL_NAME4}, + new RangePartition[] {rangePartition1, rangePartition2, rangePartition3}) + }; + } + + private Transform[] createListPartition() { + LocalDate today = LocalDate.now(); + LocalDate tomorrow = today.plusDays(1); + LiteralDTO todayLiteral = + LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); + LiteralDTO tomorrowLiteral = + LiteralDTO.builder() + .withDataType(Types.DateType.get()) + .withValue(tomorrow.toString()) + .build(); + LiteralDTO intLiteral1 = + LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("1").build(); + LiteralDTO intLiteral2 = + LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); + ListPartition listPartition1 = + (ListPartition) + Partitions.list( + "p1", + new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + null); + ListPartition listPartition2 = + (ListPartition) + Partitions.list( + "p2", + new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + null); + return new Transform[] { + Transforms.list( + new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}}, + new ListPartition[] {listPartition1, listPartition2}) + }; } private Map createTableProperties() { @@ -334,6 +399,7 @@ void testDorisTableBasicOperation() { }; Map properties = createTableProperties(); + Transform[] partitions = createRangePartition(); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = tableCatalog.createTable( @@ -341,7 +407,7 @@ void testDorisTableBasicOperation() { columns, table_comment, properties, - Transforms.EMPTY_TRANSFORM, + partitions, distribution, null, indexes); @@ -491,6 +557,7 @@ void testAlterDorisTable() { }; Map properties = createTableProperties(); + Transform[] partitions = createListPartition(); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = tableCatalog.createTable( @@ -498,7 +565,7 @@ void testAlterDorisTable() { columns, table_comment, properties, - Transforms.EMPTY_TRANSFORM, + partitions, distribution, null, indexes); @@ -539,25 +606,25 @@ void testAlterDorisTable() { tableCatalog.alterTable( tableIdentifier, TableChange.addColumn( - new String[] {"col_4"}, Types.VarCharType.of(255), "col_4_comment", true)); + new String[] {"col_5"}, Types.VarCharType.of(255), "col_5_comment", true)); Awaitility.await() .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertEquals( - 4, tableCatalog.loadTable(tableIdentifier).columns().length)); + 5, tableCatalog.loadTable(tableIdentifier).columns().length)); ITUtils.assertColumn( - Column.of("col_4", Types.VarCharType.of(255), "col_4_comment"), - tableCatalog.loadTable(tableIdentifier).columns()[3]); + Column.of("col_5", Types.VarCharType.of(255), "col_5_comment"), + tableCatalog.loadTable(tableIdentifier).columns()[4]); // change column position // TODO: change column position is unstable, add it later // drop column tableCatalog.alterTable( - tableIdentifier, TableChange.deleteColumn(new String[] {"col_4"}, true)); + tableIdentifier, TableChange.deleteColumn(new String[] {"col_5"}, true)); Awaitility.await() .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) @@ -565,7 +632,7 @@ void testAlterDorisTable() { .untilAsserted( () -> Assertions.assertEquals( - 3, tableCatalog.loadTable(tableIdentifier).columns().length)); + 4, tableCatalog.loadTable(tableIdentifier).columns().length)); } @Test From 85371b53009e7c0932bdf03c0c73afe2995d54df Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 18 Jun 2024 15:54:55 +0800 Subject: [PATCH 06/12] fix IT --- .../catalog/doris/operation/DorisTableOperations.java | 2 +- .../catalog/doris/integration/test/CatalogDorisIT.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 3648bf0ee62..f1863198887 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -273,7 +273,7 @@ private static void appendPartitionSql( partitionSqlBuilder.append(NEW_LINE).append(")"); } else if (partitioning[0] instanceof Transforms.ListTransform) { Transforms.ListTransform listPartition = (Transforms.ListTransform) partitioning[0]; - partitionSqlBuilder.append(" PARTITION BY LIST("); + partitionSqlBuilder.append(NEW_LINE).append(" PARTITION BY LIST("); ImmutableList.Builder partitionColumnsBuilder = ImmutableList.builder(); String[][] filedNames = listPartition.fieldNames(); diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 570222c1d53..27a209f3a8a 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -174,10 +174,12 @@ private void createSchema() { } private Column[] createColumns() { - Column col1 = Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col1 = + Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment", false, false, null); Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10), "col_2_comment"); Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10), "col_3_comment"); - Column col4 = Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment"); + Column col4 = + Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment", false, false, null); return new Column[] {col1, col2, col3, col4}; } From 87fadeb72cbce03c62eb765c59183924994708fd Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 18 Jun 2024 19:57:55 +0800 Subject: [PATCH 07/12] code style --- .../integration/test/CatalogDorisIT.java | 27 ++++++++----------- .../operation/TestDorisTableOperations.java | 27 ++++++++----------- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 27a209f3a8a..f641705700d 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -194,12 +194,9 @@ private Transform[] createRangePartition() { .withDataType(Types.DateType.get()) .withValue(tomorrow.toString()) .build(); - RangePartition rangePartition1 = - (RangePartition) Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); - RangePartition rangePartition2 = - (RangePartition) Partitions.range("p2", tomorrowLiteral, todayLiteral, null); - RangePartition rangePartition3 = - (RangePartition) Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); + RangePartition rangePartition3 = Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); return new Transform[] { Transforms.range( new String[] {DORIS_COL_NAME4}, @@ -222,17 +219,15 @@ private Transform[] createListPartition() { LiteralDTO intLiteral2 = LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); ListPartition listPartition1 = - (ListPartition) - Partitions.list( - "p1", - new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, - null); + Partitions.list( + "p1", + new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + null); ListPartition listPartition2 = - (ListPartition) - Partitions.list( - "p2", - new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, - null); + Partitions.list( + "p2", + new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + null); return new Transform[] { Transforms.list( new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}}, diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java index bebe635191e..598c111f0f3 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -454,12 +454,9 @@ public void testCreateTableWithPartition() { .withDataType(Types.DateType.get()) .withValue(tomorrow.toString()) .build(); - RangePartition rangePartition1 = - (RangePartition) Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); - RangePartition rangePartition2 = - (RangePartition) Partitions.range("p2", tomorrowLiteral, todayLiteral, null); - RangePartition rangePartition3 = - (RangePartition) Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); + RangePartition rangePartition3 = Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); Transform[] rangePartition = new Transform[] { Transforms.range( @@ -491,17 +488,15 @@ public void testCreateTableWithPartition() { LiteralDTO intLiteral2 = LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); ListPartition listPartition1 = - (ListPartition) - Partitions.list( - "p1", - new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, - null); + Partitions.list( + "p1", + new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + null); ListPartition listPartition2 = - (ListPartition) - Partitions.list( - "p2", - new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, - null); + Partitions.list( + "p2", + new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + null); Transform[] listPartition = new Transform[] { Transforms.list( From ee2c22acb07508592e7108c99bc80ca21875039d Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Tue, 18 Jun 2024 20:22:50 +0800 Subject: [PATCH 08/12] code style --- .../integration/test/CatalogDorisIT.java | 152 +++++++----------- 1 file changed, 55 insertions(+), 97 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index f641705700d..ca328504c6e 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -27,8 +27,8 @@ import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; -import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.literals.Literal; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; @@ -263,9 +263,7 @@ void testDorisSchemaBasicOperation() { Assertions.assertThrows( SchemaAlreadyExistsException.class, - () -> { - schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap()); - }); + () -> schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap())); // test drop schema Assertions.assertTrue(schemas.dropSchema(schemaIdent.name(), false)); @@ -315,11 +313,7 @@ void testDropDorisSchema() { // Check database has been dropped SupportsSchemas schemas = catalog.asSchemas(); - Assertions.assertThrows( - NoSuchSchemaException.class, - () -> { - schemas.loadSchema(schemaName); - }); + Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(schemaName)); } @Test @@ -333,52 +327,32 @@ void testSchemaWithIllegalName() { String sqlInjection = databaseName + "`; DROP TABLE important_table; -- "; Assertions.assertThrows( IllegalArgumentException.class, - () -> { - schemas.createSchema(sqlInjection, comment, properties); - }); + () -> schemas.createSchema(sqlInjection, comment, properties)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - schemas.dropSchema(sqlInjection, false); - }); + IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection, false)); String sqlInjection1 = databaseName + "`; SLEEP(10); -- "; Assertions.assertThrows( IllegalArgumentException.class, - () -> { - schemas.createSchema(sqlInjection1, comment, properties); - }); + () -> schemas.createSchema(sqlInjection1, comment, properties)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - schemas.dropSchema(sqlInjection1, false); - }); + IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection1, false)); String sqlInjection2 = databaseName + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; Assertions.assertThrows( IllegalArgumentException.class, - () -> { - schemas.createSchema(sqlInjection2, comment, properties); - }); + () -> schemas.createSchema(sqlInjection2, comment, properties)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - schemas.dropSchema(sqlInjection2, false); - }); + IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection2, false)); // should throw an exception with input that has more than 64 characters String invalidInput = StringUtils.repeat("a", 65); Assertions.assertThrows( IllegalArgumentException.class, - () -> { - schemas.createSchema(invalidInput, comment, properties); - }); + () -> schemas.createSchema(invalidInput, comment, properties)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - schemas.dropSchema(invalidInput, false); - }); + IllegalArgumentException.class, () -> schemas.dropSchema(invalidInput, false)); } @Test @@ -442,22 +416,18 @@ void testDorisIllegalTableName() { Assertions.assertThrows( IllegalArgumentException.class, - () -> { - tableCatalog.createTable( - tableIdentifier, - columns, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t1_indexes); - }); + () -> + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t1_indexes)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - catalog.asTableCatalog().dropTable(tableIdentifier); - }); + IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier)); String t2_name = table_name + "`; SLEEP(10); -- "; Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); @@ -468,22 +438,18 @@ void testDorisIllegalTableName() { Assertions.assertThrows( IllegalArgumentException.class, - () -> { - tableCatalog.createTable( - tableIdentifier2, - columns2, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t2_indexes); - }); + () -> + tableCatalog.createTable( + tableIdentifier2, + columns2, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t2_indexes)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - catalog.asTableCatalog().dropTable(tableIdentifier2); - }); + IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier2)); String t3_name = table_name + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; @@ -495,22 +461,18 @@ void testDorisIllegalTableName() { Assertions.assertThrows( IllegalArgumentException.class, - () -> { - tableCatalog.createTable( - tableIdentifier3, - columns3, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t3_indexes); - }); + () -> + tableCatalog.createTable( + tableIdentifier3, + columns3, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t3_indexes)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - catalog.asTableCatalog().dropTable(tableIdentifier3); - }); + IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier3)); String invalidInput = StringUtils.repeat("a", 65); Column t4_col = Column.of(invalidInput, Types.LongType.get(), "id", false, false, null); @@ -521,22 +483,18 @@ void testDorisIllegalTableName() { Assertions.assertThrows( IllegalArgumentException.class, - () -> { - tableCatalog.createTable( - tableIdentifier4, - columns4, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t4_indexes); - }); + () -> + tableCatalog.createTable( + tableIdentifier4, + columns4, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t4_indexes)); Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - catalog.asTableCatalog().dropTable(tableIdentifier4); - }); + IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier4)); } @Test From c068b903f64f4a3df93d3733f91a9142ee255c92 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Wed, 19 Jun 2024 17:11:24 +0800 Subject: [PATCH 09/12] fix literal --- .../doris/operation/DorisTableOperations.java | 12 +++---- .../integration/test/CatalogDorisIT.java | 34 ++++++------------- .../operation/TestDorisTableOperations.java | 25 +++++--------- 3 files changed, 26 insertions(+), 45 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index f1863198887..ec7918fac9e 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -11,7 +11,6 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; -import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.exceptions.NoSuchColumnException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; @@ -20,6 +19,7 @@ import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Strategy; import com.datastrato.gravitino.rel.expressions.literals.Literal; +import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; @@ -233,18 +233,18 @@ private static void appendPartitionSql( .append(part.name()) .append(BACK_QUOTE) .append(" VALUES "); - LiteralDTO upper = (LiteralDTO) part.upper(); - LiteralDTO lower = (LiteralDTO) part.lower(); - if (LiteralDTO.NULL.equals(upper) && LiteralDTO.NULL.equals(lower)) { + Literal upper = part.upper(); + Literal lower = part.lower(); + if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { partitionAssignSqlBuilder.append("LESS THAN MAXVALUE"); - } else if (LiteralDTO.NULL.equals(lower)) { + } else if (Literals.NULL.equals(lower)) { partitionAssignSqlBuilder .append("LESS THAN (") .append(DOUBLE_QUOTE) .append(upper.value()) .append(DOUBLE_QUOTE) .append(")"); - } else if (LiteralDTO.NULL.equals(upper)) { + } else if (Literals.NULL.equals(upper)) { partitionAssignSqlBuilder .append("[(") .append(DOUBLE_QUOTE) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index ca328504c6e..144113047d5 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.SupportsSchemas; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetalake; -import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.integration.test.container.ContainerSuite; @@ -28,6 +27,7 @@ import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.literals.Literal; +import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; @@ -187,16 +187,11 @@ private Column[] createColumns() { private Transform[] createRangePartition() { LocalDate today = LocalDate.now(); LocalDate tomorrow = today.plusDays(1); - LiteralDTO todayLiteral = - LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); - LiteralDTO tomorrowLiteral = - LiteralDTO.builder() - .withDataType(Types.DateType.get()) - .withValue(tomorrow.toString()) - .build(); - RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); + Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); + RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, Literals.NULL, null); RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); - RangePartition rangePartition3 = Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + RangePartition rangePartition3 = Partitions.range("p3", Literals.NULL, tomorrowLiteral, null); return new Transform[] { Transforms.range( new String[] {DORIS_COL_NAME4}, @@ -207,26 +202,19 @@ private Transform[] createRangePartition() { private Transform[] createListPartition() { LocalDate today = LocalDate.now(); LocalDate tomorrow = today.plusDays(1); - LiteralDTO todayLiteral = - LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); - LiteralDTO tomorrowLiteral = - LiteralDTO.builder() - .withDataType(Types.DateType.get()) - .withValue(tomorrow.toString()) - .build(); - LiteralDTO intLiteral1 = - LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("1").build(); - LiteralDTO intLiteral2 = - LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); + Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); + Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); + Literals.LiteralImpl integerLiteral1 = Literals.integerLiteral(1); + Literals.LiteralImpl integerLiteral2 = Literals.integerLiteral(2); ListPartition listPartition1 = Partitions.list( "p1", - new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + new Literal[][] {{integerLiteral1, todayLiteral}, {integerLiteral1, tomorrowLiteral}}, null); ListPartition listPartition2 = Partitions.list( "p2", - new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + new Literal[][] {{integerLiteral2, todayLiteral}, {integerLiteral2, tomorrowLiteral}}, null); return new Transform[] { Transforms.list( diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java index 598c111f0f3..b8552ad4b8a 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -6,13 +6,13 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; -import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.literals.Literal; +import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; @@ -447,16 +447,11 @@ public void testCreateTableWithPartition() { String rangePartitionTableName = GravitinoITUtils.genRandomName("range_partition_table"); LocalDate today = LocalDate.now(); LocalDate tomorrow = today.plusDays(1); - LiteralDTO todayLiteral = - LiteralDTO.builder().withDataType(Types.DateType.get()).withValue(today.toString()).build(); - LiteralDTO tomorrowLiteral = - LiteralDTO.builder() - .withDataType(Types.DateType.get()) - .withValue(tomorrow.toString()) - .build(); - RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, LiteralDTO.NULL, null); + Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); + Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); + RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, Literals.NULL, null); RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); - RangePartition rangePartition3 = Partitions.range("p3", LiteralDTO.NULL, tomorrowLiteral, null); + RangePartition rangePartition3 = Partitions.range("p3", Literals.NULL, tomorrowLiteral, null); Transform[] rangePartition = new Transform[] { Transforms.range( @@ -483,19 +478,17 @@ public void testCreateTableWithPartition() { // create table with list partition String listPartitionTableName = GravitinoITUtils.genRandomName("list_partition_table"); - LiteralDTO intLiteral1 = - LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("1").build(); - LiteralDTO intLiteral2 = - LiteralDTO.builder().withDataType(Types.IntegerType.get()).withValue("2").build(); + Literals.LiteralImpl integerLiteral1 = Literals.integerLiteral(1); + Literals.LiteralImpl integerLiteral2 = Literals.integerLiteral(2); ListPartition listPartition1 = Partitions.list( "p1", - new Literal[][] {{intLiteral1, todayLiteral}, {intLiteral1, tomorrowLiteral}}, + new Literal[][] {{integerLiteral1, todayLiteral}, {integerLiteral1, tomorrowLiteral}}, null); ListPartition listPartition2 = Partitions.list( "p2", - new Literal[][] {{intLiteral2, todayLiteral}, {intLiteral2, tomorrowLiteral}}, + new Literal[][] {{integerLiteral2, todayLiteral}, {integerLiteral2, tomorrowLiteral}}, null); Transform[] listPartition = new Transform[] { From 4129994744033f2ad1571b3d3405096324263ebd Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Mon, 1 Jul 2024 16:51:42 +0800 Subject: [PATCH 10/12] makes code more readable --- .../doris/operation/DorisTableOperations.java | 223 ++++++++---------- 1 file changed, 102 insertions(+), 121 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index ec7918fac9e..cb8b83cecb2 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -204,141 +204,122 @@ private static void appendPartitionSql( Preconditions.checkArgument( partitioning.length == 1, "Composite partition type is not supported"); - StringBuilder partitionSqlBuilder = new StringBuilder(); + StringBuilder partitionSqlBuilder; Set columnNames = Arrays.stream(columns).map(JdbcColumn::name).collect(Collectors.toSet()); if (partitioning[0] instanceof Transforms.RangeTransform) { - partitionSqlBuilder.append(NEW_LINE).append(" PARTITION BY RANGE("); - // TODO support multi-column range partitioning in doris + // We do not support multi-column range partitioning in doris for now Transforms.RangeTransform rangePartition = (Transforms.RangeTransform) partitioning[0]; - - Preconditions.checkArgument( - rangePartition.fieldName().length == 1, "Doris partition does not support nested field"); - Preconditions.checkArgument( - columnNames.contains(rangePartition.fieldName()[0]), - "The partition field must be one of the columns"); - - String partitionColumn = BACK_QUOTE + rangePartition.fieldName()[0] + BACK_QUOTE; - partitionSqlBuilder.append(partitionColumn).append(") ").append(NEW_LINE).append("("); - - RangePartition[] assignments = rangePartition.assignments(); - if (!ArrayUtils.isEmpty(assignments)) { - ImmutableList.Builder partitions = ImmutableList.builder(); - for (RangePartition part : assignments) { - StringBuilder partitionAssignSqlBuilder = new StringBuilder(); - partitionAssignSqlBuilder - .append(" PARTITION ") - .append(BACK_QUOTE) - .append(part.name()) - .append(BACK_QUOTE) - .append(" VALUES "); - Literal upper = part.upper(); - Literal lower = part.lower(); - if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { - partitionAssignSqlBuilder.append("LESS THAN MAXVALUE"); - } else if (Literals.NULL.equals(lower)) { - partitionAssignSqlBuilder - .append("LESS THAN (") - .append(DOUBLE_QUOTE) - .append(upper.value()) - .append(DOUBLE_QUOTE) - .append(")"); - } else if (Literals.NULL.equals(upper)) { - partitionAssignSqlBuilder - .append("[(") - .append(DOUBLE_QUOTE) - .append(lower.value()) - .append(DOUBLE_QUOTE) - .append("), (MAXVALUE))"); - } else { - partitionAssignSqlBuilder - .append("[(") - .append(DOUBLE_QUOTE) - .append(lower.value()) - .append(DOUBLE_QUOTE) - .append("), (") - .append(DOUBLE_QUOTE) - .append(upper.value()) - .append(DOUBLE_QUOTE) - .append("))"); - } - partitions.add(partitionAssignSqlBuilder.toString()); - } - partitionSqlBuilder - .append(NEW_LINE) - .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); - } - - partitionSqlBuilder.append(NEW_LINE).append(")"); + partitionSqlBuilder = generateRangePartitionSql(rangePartition, columnNames); } else if (partitioning[0] instanceof Transforms.ListTransform) { Transforms.ListTransform listPartition = (Transforms.ListTransform) partitioning[0]; - partitionSqlBuilder.append(NEW_LINE).append(" PARTITION BY LIST("); + partitionSqlBuilder = generateListPartitionSql(listPartition, columnNames); + } else { + throw new IllegalArgumentException("Unsupported partition type of Doris"); + } - ImmutableList.Builder partitionColumnsBuilder = ImmutableList.builder(); - String[][] filedNames = listPartition.fieldNames(); - for (String[] filedName : filedNames) { - Preconditions.checkArgument( - filedName.length == 1, "Doris partition does not support nested field"); - Preconditions.checkArgument( - columnNames.contains(filedName[0]), "The partition field must be one of the columns"); + sqlBuilder.append(partitionSqlBuilder); + } - partitionColumnsBuilder.add(BACK_QUOTE + filedName[0] + BACK_QUOTE); + private static StringBuilder generateRangePartitionSql( + Transforms.RangeTransform rangePartition, Set columnNames) { + Preconditions.checkArgument( + rangePartition.fieldName().length == 1, "Doris partition does not support nested field"); + Preconditions.checkArgument( + columnNames.contains(rangePartition.fieldName()[0]), + "The partition field must be one of the columns"); + + StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE); + String partitionDefinition = + String.format(" PARTITION BY RANGE(`%s`) ", rangePartition.fieldName()[0]); + partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("("); + + // Assign range partitions + RangePartition[] assignments = rangePartition.assignments(); + if (!ArrayUtils.isEmpty(assignments)) { + ImmutableList.Builder partitions = ImmutableList.builder(); + for (RangePartition part : assignments) { + StringBuilder partitionAssignSqlBuilder = new StringBuilder(); + partitionAssignSqlBuilder.append(String.format(" PARTITION `%s` VALUES", part.name())); + Literal upper = part.upper(); + Literal lower = part.lower(); + if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { + partitionAssignSqlBuilder.append(" LESS THAN MAXVALUE"); + } else if (Literals.NULL.equals(lower)) { + partitionAssignSqlBuilder.append(String.format(" LESS THAN (\"%s\")", upper.value())); + } else if (Literals.NULL.equals(upper)) { + partitionAssignSqlBuilder.append(String.format(" [(\"%s\"), (MAXVALUE))", lower.value())); + } else { + partitionAssignSqlBuilder.append( + String.format(" [(\"%s\"), (\"%s\"))", lower.value(), upper.value())); + } + partitions.add(partitionAssignSqlBuilder.toString()); } - String partitionColumns = - partitionColumnsBuilder.build().stream().collect(Collectors.joining(",")); - partitionSqlBuilder.append(partitionColumns).append(") ").append(NEW_LINE).append("("); - - ListPartition[] assignments = listPartition.assignments(); - if (!ArrayUtils.isEmpty(assignments)) { - ImmutableList.Builder partitions = ImmutableList.builder(); - for (ListPartition parts : assignments) { - StringBuilder partitionAssignSqlBuilder = new StringBuilder(); - partitionAssignSqlBuilder - .append(" PARTITION ") - .append(BACK_QUOTE) - .append(parts.name()) - .append(BACK_QUOTE) - .append(" VALUES IN ") - .append("("); - ImmutableList.Builder partitionValues = ImmutableList.builder(); - for (Literal[] part : parts.lists()) { - Preconditions.checkArgument( - part.length == filedNames.length, - "The number of partitioning columns must be consistent."); - StringBuilder partitionValuesSqlBuilder = new StringBuilder(); - if (part.length > 1) { - partitionValuesSqlBuilder - .append("(") - .append( - Arrays.stream(part) - .map(p -> DOUBLE_QUOTE + p.value() + DOUBLE_QUOTE) - .collect(Collectors.joining(","))) - .append(")"); - } else { - partitionValuesSqlBuilder - .append(DOUBLE_QUOTE) - .append(part[0].value()) - .append(DOUBLE_QUOTE); - } - partitionValues.add(partitionValuesSqlBuilder.toString()); + partitionSqlBuilder + .append(NEW_LINE) + .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); + } + + partitionSqlBuilder.append(NEW_LINE).append(")"); + return partitionSqlBuilder; + } + + private static StringBuilder generateListPartitionSql( + Transforms.ListTransform listPartition, Set columnNames) { + ImmutableList.Builder partitionColumnsBuilder = ImmutableList.builder(); + String[][] filedNames = listPartition.fieldNames(); + for (String[] filedName : filedNames) { + Preconditions.checkArgument( + filedName.length == 1, "Doris partition does not support nested field"); + Preconditions.checkArgument( + columnNames.contains(filedName[0]), "The partition field must be one of the columns"); + + partitionColumnsBuilder.add(BACK_QUOTE + filedName[0] + BACK_QUOTE); + } + String partitionColumns = + partitionColumnsBuilder.build().stream().collect(Collectors.joining(",")); + + StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE); + String partitionDefinition = String.format(" PARTITION BY LIST(%s) ", partitionColumns); + partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("("); + + // Assign list partitions + ListPartition[] assignments = listPartition.assignments(); + if (!ArrayUtils.isEmpty(assignments)) { + ImmutableList.Builder partitions = ImmutableList.builder(); + for (ListPartition parts : assignments) { + ImmutableList.Builder partitionValues = ImmutableList.builder(); + for (Literal[] part : parts.lists()) { + Preconditions.checkArgument( + part.length == filedNames.length, + "The number of partitioning columns must be consistent."); + String partitionValuesSql; + if (part.length > 1) { + partitionValuesSql = + String.format( + "(%s)", + Arrays.stream(part) + .map(p -> DOUBLE_QUOTE + p.value() + DOUBLE_QUOTE) + .collect(Collectors.joining(","))); + } else { + partitionValuesSql = String.format("`%s`", part[0].value()); } - partitionAssignSqlBuilder - .append(partitionValues.build().stream().collect(Collectors.joining(","))) - .append(")"); - partitions.add(partitionAssignSqlBuilder.toString()); + partitionValues.add(partitionValuesSql); } - partitionSqlBuilder - .append(NEW_LINE) - .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); + String partitionAssignSql = + String.format( + " PARTITION `%s` VALUES IN (%s)", + parts.name(), partitionValues.build().stream().collect(Collectors.joining(","))); + partitions.add(partitionAssignSql); } - - partitionSqlBuilder.append(NEW_LINE).append(")"); - } else { - throw new IllegalArgumentException("Unsupported partition type of Doris"); + partitionSqlBuilder + .append(NEW_LINE) + .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); } - sqlBuilder.append(partitionSqlBuilder); + partitionSqlBuilder.append(NEW_LINE).append(")"); + return partitionSqlBuilder; } @Override @@ -453,7 +434,7 @@ protected void getTableStatus(Connection connection, String databaseName, String "Table {}.{} schema-change execution status: {}", databaseName, tableName, - jobStatus.toString()); + jobStatus); } } catch (SQLException e) { From 12eb158dfb8d20d231456f79300af51c29ffd343 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Mon, 1 Jul 2024 17:03:55 +0800 Subject: [PATCH 11/12] spotless --- .../catalog/doris/operation/DorisTableOperations.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index cb8b83cecb2..9d7aabf7612 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -431,10 +431,7 @@ protected void getTableStatus(Connection connection, String databaseName, String if (jobStatus.length() > 0) { LOG.info( - "Table {}.{} schema-change execution status: {}", - databaseName, - tableName, - jobStatus); + "Table {}.{} schema-change execution status: {}", databaseName, tableName, jobStatus); } } catch (SQLException e) { From 26f44f188b50d9d85285ed8ef7bbacfdd808de09 Mon Sep 17 00:00:00 2001 From: zhanghan18 Date: Thu, 25 Jul 2024 15:19:59 +0800 Subject: [PATCH 12/12] reformat some code --- .../doris/operation/DorisTableOperations.java | 69 +- .../DorisTablePartitionOperations.java | 70 +- .../catalog/doris/utils/DorisUtils.java | 71 ++ .../integration/test/CatalogDorisIT.java | 638 ------------------ .../integration/test/CatalogDorisIT.java | 200 ++++-- .../operation/TestDorisTableOperations.java | 62 +- .../catalog/doris/utils/TestDorisUtils.java | 93 ++- .../integration/test/util/ITUtils.java | 36 + 8 files changed, 413 insertions(+), 826 deletions(-) delete mode 100644 catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java index e88cd9314b6..64db27999e7 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog.doris.operation; +import static org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment; import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; import com.google.common.base.Preconditions; @@ -54,16 +55,18 @@ import org.apache.gravitino.rel.TableChange; import org.apache.gravitino.rel.expressions.distributions.Distribution; import org.apache.gravitino.rel.expressions.distributions.Strategy; +import org.apache.gravitino.rel.expressions.literals.Literal; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.expressions.transforms.Transforms; import org.apache.gravitino.rel.indexes.Index; import org.apache.gravitino.rel.indexes.Indexes; +import org.apache.gravitino.rel.partitions.ListPartition; +import org.apache.gravitino.rel.partitions.RangePartition; /** Table operations for Apache Doris. */ public class DorisTableOperations extends JdbcTableOperations { private static final String BACK_QUOTE = "`"; private static final String DORIS_AUTO_INCREMENT = "AUTO_INCREMENT"; - private static final String DOUBLE_QUOTE = "\""; private static final String NEW_LINE = "\n"; @Override @@ -194,7 +197,6 @@ private static void validateDistribution(Distribution distribution, JdbcColumn[] } private static void appendIndexesSql(Index[] indexes, StringBuilder sqlBuilder) { - if (indexes.length == 0) { return; } @@ -252,33 +254,17 @@ private static StringBuilder generateRangePartitionSql( StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE); String partitionDefinition = - String.format(" PARTITION BY RANGE(`%s`) ", rangePartition.fieldName()[0]); + String.format(" PARTITION BY RANGE(`%s`)", rangePartition.fieldName()[0]); partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("("); // Assign range partitions RangePartition[] assignments = rangePartition.assignments(); if (!ArrayUtils.isEmpty(assignments)) { - ImmutableList.Builder partitions = ImmutableList.builder(); - for (RangePartition part : assignments) { - StringBuilder partitionAssignSqlBuilder = new StringBuilder(); - partitionAssignSqlBuilder.append(String.format(" PARTITION `%s` VALUES", part.name())); - Literal upper = part.upper(); - Literal lower = part.lower(); - if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { - partitionAssignSqlBuilder.append(" LESS THAN MAXVALUE"); - } else if (Literals.NULL.equals(lower)) { - partitionAssignSqlBuilder.append(String.format(" LESS THAN (\"%s\")", upper.value())); - } else if (Literals.NULL.equals(upper)) { - partitionAssignSqlBuilder.append(String.format(" [(\"%s\"), (MAXVALUE))", lower.value())); - } else { - partitionAssignSqlBuilder.append( - String.format(" [(\"%s\"), (\"%s\"))", lower.value(), upper.value())); - } - partitions.add(partitionAssignSqlBuilder.toString()); - } - partitionSqlBuilder - .append(NEW_LINE) - .append(partitions.build().stream().collect(Collectors.joining("," + NEW_LINE))); + String partitionSqlFragments = + Arrays.stream(assignments) + .map(DorisUtils::generatePartitionSqlFragment) + .collect(Collectors.joining("," + NEW_LINE)); + partitionSqlBuilder.append(NEW_LINE).append(partitionSqlFragments); } partitionSqlBuilder.append(NEW_LINE).append(")"); @@ -301,37 +287,22 @@ private static StringBuilder generateListPartitionSql( partitionColumnsBuilder.build().stream().collect(Collectors.joining(",")); StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE); - String partitionDefinition = String.format(" PARTITION BY LIST(%s) ", partitionColumns); + String partitionDefinition = String.format(" PARTITION BY LIST(%s)", partitionColumns); partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("("); // Assign list partitions ListPartition[] assignments = listPartition.assignments(); if (!ArrayUtils.isEmpty(assignments)) { ImmutableList.Builder partitions = ImmutableList.builder(); - for (ListPartition parts : assignments) { - ImmutableList.Builder partitionValues = ImmutableList.builder(); - for (Literal[] part : parts.lists()) { - Preconditions.checkArgument( - part.length == filedNames.length, - "The number of partitioning columns must be consistent."); - String partitionValuesSql; - if (part.length > 1) { - partitionValuesSql = - String.format( - "(%s)", - Arrays.stream(part) - .map(p -> DOUBLE_QUOTE + p.value() + DOUBLE_QUOTE) - .collect(Collectors.joining(","))); - } else { - partitionValuesSql = String.format("`%s`", part[0].value()); - } - partitionValues.add(partitionValuesSql); - } - String partitionAssignSql = - String.format( - " PARTITION `%s` VALUES IN (%s)", - parts.name(), partitionValues.build().stream().collect(Collectors.joining(","))); - partitions.add(partitionAssignSql); + for (ListPartition part : assignments) { + Literal[][] lists = part.lists(); + Preconditions.checkArgument( + lists.length > 0, "The number of values in list partition must be greater than 0"); + Preconditions.checkArgument( + Arrays.stream(lists).allMatch(p -> p.length == filedNames.length), + "The number of partitioning columns must be consistent"); + + partitions.add(generatePartitionSqlFragment(part)); } partitionSqlBuilder .append(NEW_LINE) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java index c3e5a2bba4e..e075717f460 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java @@ -28,6 +28,7 @@ import static org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VALUES_RANGE; import static org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VISIBLE_VERSION; import static org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VISIBLE_VERSION_TIME; +import static org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -43,7 +44,6 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import javax.sql.DataSource; import org.apache.gravitino.catalog.jdbc.JdbcTable; import org.apache.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; @@ -145,8 +145,8 @@ public Partition addPartition(Partition partition) throws PartitionAlreadyExists try (Connection connection = getConnection(loadedTable.databaseName())) { Transform partitionInfo = loadedTable.partitioning()[0]; - String addPartitionSqlFormat = "ALTER TABLE `%s` ADD PARTITION `%s` VALUES %s"; - String partitionValues; + String addPartitionSqlFormat = "ALTER TABLE `%s` ADD %s"; + String partitionSqlFragment; Partition added; if (partition instanceof RangePartition) { @@ -156,7 +156,7 @@ public Partition addPartition(Partition partition) throws PartitionAlreadyExists loadedTable.name()); RangePartition rangePartition = (RangePartition) partition; - partitionValues = buildRangePartitionValues(rangePartition); + partitionSqlFragment = generatePartitionSqlFragment(rangePartition); // The partition properties actually cannot be passed into Doris, we just return an empty // map instead. @@ -173,9 +173,18 @@ public Partition addPartition(Partition partition) throws PartitionAlreadyExists loadedTable.name()); ListPartition listPartition = (ListPartition) partition; - partitionValues = - buildListPartitionValues( - listPartition, ((Transforms.ListTransform) partitionInfo).fieldNames().length); + Literal[][] lists = listPartition.lists(); + Preconditions.checkArgument( + lists.length > 0, "The number of values in list partition must be greater than 0"); + Preconditions.checkArgument( + Arrays.stream(lists) + .allMatch( + part -> + part.length + == ((Transforms.ListTransform) partitionInfo).fieldNames().length), + "The number of partitioning columns must be consistent"); + + partitionSqlFragment = generatePartitionSqlFragment(listPartition); added = Partitions.list(listPartition.name(), listPartition.lists(), Collections.emptyMap()); @@ -185,8 +194,7 @@ public Partition addPartition(Partition partition) throws PartitionAlreadyExists try (Statement statement = connection.createStatement()) { statement.executeUpdate( - String.format( - addPartitionSqlFormat, loadedTable.name(), partition.name(), partitionValues)); + String.format(addPartitionSqlFormat, loadedTable.name(), partitionSqlFragment)); return added; } } catch (SQLException e) { @@ -287,48 +295,4 @@ private Map getColumnType(Connection connection) throws SQLExcepti return columnTypes.build(); } } - - private String buildRangePartitionValues(RangePartition rangePartition) { - Literal upper = rangePartition.upper(); - Literal lower = rangePartition.lower(); - String partitionValues; - if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { - partitionValues = "LESS THAN MAXVALUE"; - } else if (Literals.NULL.equals(lower)) { - partitionValues = "LESS THAN (\"" + upper.value() + "\")"; - } else if (Literals.NULL.equals(upper)) { - partitionValues = "[(\"" + lower.value() + "\"), (MAXVALUE))"; - } else { - partitionValues = "[(\"" + lower.value() + "\"), (\"" + upper.value() + "\"))"; - } - return partitionValues; - } - - private String buildListPartitionValues(ListPartition listPartition, int partitionedFieldNums) { - Literal[][] lists = listPartition.lists(); - Preconditions.checkArgument( - lists.length > 0, "The number of values in list partition must be greater than 0"); - - ImmutableList.Builder listValues = ImmutableList.builder(); - for (Literal[] part : lists) { - Preconditions.checkArgument( - part.length == partitionedFieldNums, - "The number of partitioning columns must be consistent"); - - StringBuilder values = new StringBuilder(); - if (part.length > 1) { - values - .append("(") - .append( - Arrays.stream(part) - .map(p -> "\"" + p.value() + "\"") - .collect(Collectors.joining(","))) - .append(")"); - } else { - values.append("\"").append(part[0].value()).append("\""); - } - listValues.add(values.toString()); - } - return String.format("IN (%s)", listValues.build().stream().collect(Collectors.joining(","))); - } } diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java index 69eada5ee67..a3d2ccc915c 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog.doris.utils; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -25,8 +26,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.gravitino.rel.expressions.literals.Literal; +import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.expressions.transforms.Transforms; +import org.apache.gravitino.rel.partitions.ListPartition; +import org.apache.gravitino.rel.partitions.Partition; +import org.apache.gravitino.rel.partitions.RangePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,4 +111,69 @@ public static Optional extractPartitionInfoFromSql(String createTable return Optional.empty(); } } + + /** + * Generate sql fragment that create partition in Apache Doris. + * + *

The sql fragment looks like "PARTITION {partitionName} VALUES {values}", for example: + * + * 

PARTITION `p20240724` VALUES LESS THAN ("2024-07-24")
+ * + * 
PARTITION `p20240724_v1` VALUES IN ("2024-07-24", "v1")
+ * + * @param partition The partition to be created. + * @return The partition sql fragment. + */ + public static String generatePartitionSqlFragment(Partition partition) { + String partitionSqlFragment = "PARTITION `%s` VALUES %s"; + if (partition instanceof RangePartition) { + return String.format( + partitionSqlFragment, + partition.name(), + generateRangePartitionValues((RangePartition) partition)); + } else if (partition instanceof ListPartition) { + return String.format( + partitionSqlFragment, + partition.name(), + generateListPartitionSqlValues((ListPartition) partition)); + } else { + throw new IllegalArgumentException("Unsupported partition type of Doris"); + } + } + + private static String generateRangePartitionValues(RangePartition rangePartition) { + Literal upper = rangePartition.upper(); + Literal lower = rangePartition.lower(); + String partitionValues; + if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) { + partitionValues = "LESS THAN MAXVALUE"; + } else if (Literals.NULL.equals(lower)) { + partitionValues = String.format("LESS THAN (\"%s\")", upper.value()); + } else if (Literals.NULL.equals(upper)) { + partitionValues = String.format("[(\"%s\"), (MAXVALUE))", lower.value()); + } else { + partitionValues = String.format("[(\"%s\"), (\"%s\"))", lower.value(), upper.value()); + } + return partitionValues; + } + + private static String generateListPartitionSqlValues(ListPartition listPartition) { + Literal[][] lists = listPartition.lists(); + ImmutableList.Builder listValues = ImmutableList.builder(); + for (Literal[] part : lists) { + String values; + if (part.length > 1) { + values = + String.format( + "(%s)", + Arrays.stream(part) + .map(p -> "\"" + p.value() + "\"") + .collect(Collectors.joining(","))); + } else { + values = String.format("\"%s\"", part[0].value()); + } + listValues.add(values); + } + return String.format("IN (%s)", listValues.build().stream().collect(Collectors.joining(","))); + } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java deleted file mode 100644 index c0a848900e9..00000000000 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ /dev/null @@ -1,638 +0,0 @@ - -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.catalog.doris.integration.test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.datastrato.gravitino.Catalog; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Schema; -import com.datastrato.gravitino.SupportsSchemas; -import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; -import com.datastrato.gravitino.client.GravitinoMetalake; -import com.datastrato.gravitino.exceptions.NoSuchSchemaException; -import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; -import com.datastrato.gravitino.integration.test.container.ContainerSuite; -import com.datastrato.gravitino.integration.test.container.DorisContainer; -import com.datastrato.gravitino.integration.test.util.AbstractIT; -import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; -import com.datastrato.gravitino.integration.test.util.ITUtils; -import com.datastrato.gravitino.rel.Column; -import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.rel.TableCatalog; -import com.datastrato.gravitino.rel.TableChange; -import com.datastrato.gravitino.rel.expressions.NamedReference; -import com.datastrato.gravitino.rel.expressions.distributions.Distribution; -import com.datastrato.gravitino.rel.expressions.distributions.Distributions; -import com.datastrato.gravitino.rel.expressions.literals.Literal; -import com.datastrato.gravitino.rel.expressions.literals.Literals; -import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; -import com.datastrato.gravitino.rel.expressions.transforms.Transform; -import com.datastrato.gravitino.rel.expressions.transforms.Transforms; -import com.datastrato.gravitino.rel.indexes.Index; -import com.datastrato.gravitino.rel.indexes.Indexes; -import com.datastrato.gravitino.rel.partitions.ListPartition; -import com.datastrato.gravitino.rel.partitions.Partitions; -import com.datastrato.gravitino.rel.partitions.RangePartition; -import com.datastrato.gravitino.rel.types.Types; -import com.datastrato.gravitino.utils.RandomNameUtils; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import java.io.IOException; -import java.time.LocalDate; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -@Tag("gravitino-docker-test") -@TestInstance(Lifecycle.PER_CLASS) -public class CatalogDorisIT extends AbstractIT { - - private static final String provider = "jdbc-doris"; - - private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; - - public String metalakeName = GravitinoITUtils.genRandomName("doris_it_metalake"); - public String catalogName = GravitinoITUtils.genRandomName("doris_it_catalog"); - public String schemaName = GravitinoITUtils.genRandomName("doris_it_schema"); - public String tableName = GravitinoITUtils.genRandomName("doris_it_table"); - - public String table_comment = "table_comment"; - - // Doris doesn't support schema comment - public String schema_comment = null; - public String DORIS_COL_NAME1 = "doris_col_name1"; - public String DORIS_COL_NAME2 = "doris_col_name2"; - public String DORIS_COL_NAME3 = "doris_col_name3"; - public String DORIS_COL_NAME4 = "doris_col_name4"; - - // Because the creation of Schema Change is an asynchronous process, we need to wait for a while - // For more information, you can refer to the comment in - // DorisTableOperations.generateAlterTableSql(). - private static final long MAX_WAIT_IN_SECONDS = 30; - - private static final long WAIT_INTERVAL_IN_SECONDS = 1; - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private GravitinoMetalake metalake; - - protected Catalog catalog; - - @BeforeAll - public void startup() throws IOException { - containerSuite.startDorisContainer(); - - createMetalake(); - createCatalog(); - createSchema(); - } - - @AfterAll - public void stop() { - clearTableAndSchema(); - metalake.dropCatalog(catalogName); - AbstractIT.client.dropMetalake(metalakeName); - } - - @AfterEach - public void resetSchema() { - clearTableAndSchema(); - createSchema(); - } - - private void clearTableAndSchema() { - catalog.asSchemas().dropSchema(schemaName, true); - } - - private void createMetalake() { - GravitinoMetalake[] gravitinoMetaLakes = AbstractIT.client.listMetalakes(); - Assertions.assertEquals(0, gravitinoMetaLakes.length); - - GravitinoMetalake createdMetalake = - AbstractIT.client.createMetalake(metalakeName, "comment", Collections.emptyMap()); - GravitinoMetalake loadMetalake = AbstractIT.client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); - - metalake = loadMetalake; - } - - private void createCatalog() { - Map catalogProperties = Maps.newHashMap(); - - DorisContainer dorisContainer = containerSuite.getDorisContainer(); - - String jdbcUrl = - String.format( - "jdbc:mysql://%s:%d/", - dorisContainer.getContainerIpAddress(), DorisContainer.FE_MYSQL_PORT); - - catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); - catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), DRIVER_CLASS_NAME); - catalogProperties.put(JdbcConfig.USERNAME.getKey(), DorisContainer.USER_NAME); - catalogProperties.put(JdbcConfig.PASSWORD.getKey(), DorisContainer.PASSWORD); - - Catalog createdCatalog = - metalake.createCatalog( - catalogName, - Catalog.Type.RELATIONAL, - provider, - "doris catalog comment", - catalogProperties); - Catalog loadCatalog = metalake.loadCatalog(catalogName); - Assertions.assertEquals(createdCatalog, loadCatalog); - - catalog = loadCatalog; - } - - private void createSchema() { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); - String propKey = "key"; - String propValue = "value"; - Map prop = Maps.newHashMap(); - prop.put(propKey, propValue); - - Schema createdSchema = catalog.asSchemas().createSchema(ident.name(), schema_comment, prop); - Schema loadSchema = catalog.asSchemas().loadSchema(ident.name()); - Assertions.assertEquals(createdSchema.name(), loadSchema.name()); - - Assertions.assertEquals(createdSchema.properties().get(propKey), propValue); - } - - private Column[] createColumns() { - Column col1 = - Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment", false, false, null); - Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10), "col_2_comment"); - Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10), "col_3_comment"); - Column col4 = - Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment", false, false, null); - - return new Column[] {col1, col2, col3, col4}; - } - - private Transform[] createRangePartition() { - LocalDate today = LocalDate.now(); - LocalDate tomorrow = today.plusDays(1); - Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); - Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); - RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, Literals.NULL, null); - RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); - RangePartition rangePartition3 = Partitions.range("p3", Literals.NULL, tomorrowLiteral, null); - return new Transform[] { - Transforms.range( - new String[] {DORIS_COL_NAME4}, - new RangePartition[] {rangePartition1, rangePartition2, rangePartition3}) - }; - } - - private Transform[] createListPartition() { - LocalDate today = LocalDate.now(); - LocalDate tomorrow = today.plusDays(1); - Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); - Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); - Literals.LiteralImpl integerLiteral1 = Literals.integerLiteral(1); - Literals.LiteralImpl integerLiteral2 = Literals.integerLiteral(2); - ListPartition listPartition1 = - Partitions.list( - "p1", - new Literal[][] {{integerLiteral1, todayLiteral}, {integerLiteral1, tomorrowLiteral}}, - null); - ListPartition listPartition2 = - Partitions.list( - "p2", - new Literal[][] {{integerLiteral2, todayLiteral}, {integerLiteral2, tomorrowLiteral}}, - null); - return new Transform[] { - Transforms.list( - new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}}, - new ListPartition[] {listPartition1, listPartition2}) - }; - } - - private Map createTableProperties() { - Map properties = Maps.newHashMap(); - properties.put("replication_allocation", "tag.location.default: 1"); - return properties; - } - - private Distribution createDistribution() { - return Distributions.hash(2, NamedReference.field(DORIS_COL_NAME1)); - } - - @Test - void testDorisSchemaBasicOperation() { - SupportsSchemas schemas = catalog.asSchemas(); - - // test list schemas - String[] schemaNames = schemas.listSchemas(); - Assertions.assertTrue(Arrays.asList(schemaNames).contains(schemaName)); - - // test create schema already exists - String testSchemaName = GravitinoITUtils.genRandomName("create_schema_test"); - NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); - schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap()); - - List schemaNameList = Arrays.asList(schemas.listSchemas()); - Assertions.assertTrue(schemaNameList.contains(testSchemaName)); - - Assertions.assertThrows( - SchemaAlreadyExistsException.class, - () -> schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap())); - - // test drop schema - Assertions.assertTrue(schemas.dropSchema(schemaIdent.name(), false)); - - // check schema is deleted - // 1. check by load schema - Assertions.assertThrows( - NoSuchSchemaException.class, () -> schemas.loadSchema(schemaIdent.name())); - - // 2. check by list schema - schemaNameList = Arrays.asList(schemas.listSchemas()); - Assertions.assertFalse(schemaNameList.contains(testSchemaName)); - - // test drop schema not exists - NameIdentifier notExistsSchemaIdent = NameIdentifier.of(metalakeName, catalogName, "no-exits"); - Assertions.assertFalse(schemas.dropSchema(notExistsSchemaIdent.name(), false)); - } - - @Test - void testDropDorisSchema() { - String schemaName = GravitinoITUtils.genRandomName("doris_it_schema_dropped").toLowerCase(); - - catalog.asSchemas().createSchema(schemaName, "test_comment", ImmutableMap.of("key", "value")); - - catalog - .asTableCatalog() - .createTable( - NameIdentifier.of(schemaName, tableName), - createColumns(), - "Created by gravitino client", - createTableProperties(), - Transforms.EMPTY_TRANSFORM, - createDistribution(), - null); - - // Try to drop a database, and cascade equals to false, it should not be allowed. - Throwable excep = - Assertions.assertThrows( - RuntimeException.class, () -> catalog.asSchemas().dropSchema(schemaName, false)); - Assertions.assertTrue(excep.getMessage().contains("the value of cascade should be true.")); - - // Check the database still exists - catalog.asSchemas().loadSchema(schemaName); - - // Try to drop a database, and cascade equals to true, it should be allowed. - Assertions.assertTrue(catalog.asSchemas().dropSchema(schemaName, true)); - - // Check database has been dropped - SupportsSchemas schemas = catalog.asSchemas(); - Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(schemaName)); - } - - @Test - void testSchemaWithIllegalName() { - SupportsSchemas schemas = catalog.asSchemas(); - String databaseName = RandomNameUtils.genRandomName("it_db"); - Map properties = new HashMap<>(); - String comment = "comment"; - - // should throw an exception with string that might contain SQL injection - String sqlInjection = databaseName + "`; DROP TABLE important_table; -- "; - Assertions.assertThrows( - IllegalArgumentException.class, - () -> schemas.createSchema(sqlInjection, comment, properties)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection, false)); - - String sqlInjection1 = databaseName + "`; SLEEP(10); -- "; - Assertions.assertThrows( - IllegalArgumentException.class, - () -> schemas.createSchema(sqlInjection1, comment, properties)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection1, false)); - - String sqlInjection2 = - databaseName + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; - Assertions.assertThrows( - IllegalArgumentException.class, - () -> schemas.createSchema(sqlInjection2, comment, properties)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> schemas.dropSchema(sqlInjection2, false)); - - // should throw an exception with input that has more than 64 characters - String invalidInput = StringUtils.repeat("a", 65); - Assertions.assertThrows( - IllegalArgumentException.class, - () -> schemas.createSchema(invalidInput, comment, properties)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> schemas.dropSchema(invalidInput, false)); - } - - @Test - void testDorisTableBasicOperation() { - // create a table - NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); - Column[] columns = createColumns(); - - Distribution distribution = createDistribution(); - - Index[] indexes = - new Index[] { - Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}) - }; - - Map properties = createTableProperties(); - Transform[] partitions = createRangePartition(); - TableCatalog tableCatalog = catalog.asTableCatalog(); - Table createdTable = - tableCatalog.createTable( - tableIdentifier, - columns, - table_comment, - properties, - partitions, - distribution, - null, - indexes); - - ITUtils.assertionsTableInfo( - tableName, table_comment, Arrays.asList(columns), properties, indexes, createdTable); - - // load table - Table loadTable = tableCatalog.loadTable(tableIdentifier); - ITUtils.assertionsTableInfo( - tableName, table_comment, Arrays.asList(columns), properties, indexes, loadTable); - - // rename table - String newTableName = GravitinoITUtils.genRandomName("new_table_name"); - tableCatalog.alterTable(tableIdentifier, TableChange.rename(newTableName)); - NameIdentifier newTableIdentifier = NameIdentifier.of(schemaName, newTableName); - Table renamedTable = tableCatalog.loadTable(newTableIdentifier); - ITUtils.assertionsTableInfo( - newTableName, table_comment, Arrays.asList(columns), properties, indexes, renamedTable); - } - - @Test - void testDorisIllegalTableName() { - Map properties = createTableProperties(); - TableCatalog tableCatalog = catalog.asTableCatalog(); - String table_name = "t123"; - - String t1_name = table_name + "`; DROP TABLE important_table; -- "; - Column t1_col = Column.of(t1_name, Types.LongType.get(), "id", false, false, null); - Column[] columns = {t1_col}; - Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier, - columns, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t1_indexes)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier)); - - String t2_name = table_name + "`; SLEEP(10); -- "; - Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); - Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; - Column[] columns2 = new Column[] {t2_col}; - NameIdentifier tableIdentifier2 = - NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier2, - columns2, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t2_indexes)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier2)); - - String t3_name = - table_name + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; - Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); - Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; - Column[] columns3 = new Column[] {t3_col}; - NameIdentifier tableIdentifier3 = - NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier3, - columns3, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t3_indexes)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier3)); - - String invalidInput = StringUtils.repeat("a", 65); - Column t4_col = Column.of(invalidInput, Types.LongType.get(), "id", false, false, null); - Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{invalidInput}})}; - Column[] columns4 = new Column[] {t4_col}; - NameIdentifier tableIdentifier4 = - NameIdentifier.of(metalakeName, catalogName, schemaName, invalidInput); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier4, - columns4, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0], - t4_indexes)); - Assertions.assertThrows( - IllegalArgumentException.class, () -> catalog.asTableCatalog().dropTable(tableIdentifier4)); - } - - @Test - void testAlterDorisTable() { - // create a table - NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); - Column[] columns = createColumns(); - - Distribution distribution = createDistribution(); - - Index[] indexes = - new Index[] { - Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}) - }; - - Map properties = createTableProperties(); - Transform[] partitions = createListPartition(); - TableCatalog tableCatalog = catalog.asTableCatalog(); - Table createdTable = - tableCatalog.createTable( - tableIdentifier, - columns, - table_comment, - properties, - partitions, - distribution, - null, - indexes); - - ITUtils.assertionsTableInfo( - tableName, table_comment, Arrays.asList(columns), properties, indexes, createdTable); - - // Alter column type - tableCatalog.alterTable( - tableIdentifier, - TableChange.updateColumnType(new String[] {DORIS_COL_NAME3}, Types.VarCharType.of(255))); - - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - ITUtils.assertColumn( - Column.of(DORIS_COL_NAME3, Types.VarCharType.of(255), "col_3_comment"), - tableCatalog.loadTable(tableIdentifier).columns()[2])); - - // update column comment - // Alter column type - tableCatalog.alterTable( - tableIdentifier, - TableChange.updateColumnComment(new String[] {DORIS_COL_NAME3}, "new_comment")); - - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - ITUtils.assertColumn( - Column.of(DORIS_COL_NAME3, Types.VarCharType.of(255), "new_comment"), - tableCatalog.loadTable(tableIdentifier).columns()[2])); - - // add new column - tableCatalog.alterTable( - tableIdentifier, - TableChange.addColumn( - new String[] {"col_5"}, Types.VarCharType.of(255), "col_5_comment", true)); - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - Assertions.assertEquals( - 5, tableCatalog.loadTable(tableIdentifier).columns().length)); - - ITUtils.assertColumn( - Column.of("col_5", Types.VarCharType.of(255), "col_5_comment"), - tableCatalog.loadTable(tableIdentifier).columns()[4]); - - // change column position - // TODO: change column position is unstable, add it later - - // drop column - tableCatalog.alterTable( - tableIdentifier, TableChange.deleteColumn(new String[] {"col_5"}, true)); - - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - Assertions.assertEquals( - 4, tableCatalog.loadTable(tableIdentifier).columns().length)); - } - - @Test - void testDorisIndex() { - String tableName = GravitinoITUtils.genRandomName("test_add_index"); - - NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); - Column[] columns = createColumns(); - - Distribution distribution = createDistribution(); - - Map properties = createTableProperties(); - TableCatalog tableCatalog = catalog.asTableCatalog(); - Table createdTable = - tableCatalog.createTable( - tableIdentifier, - columns, - table_comment, - properties, - Transforms.EMPTY_TRANSFORM, - distribution, - null); - Assertions.assertEquals(createdTable.name(), tableName); - - // add index test. - tableCatalog.alterTable( - NameIdentifier.of(schemaName, tableName), - TableChange.addIndex( - Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}})); - - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - assertEquals( - 1, - tableCatalog - .loadTable(NameIdentifier.of(schemaName, tableName)) - .index() - .length)); - - // delete index and add new column and index. - tableCatalog.alterTable( - NameIdentifier.of(schemaName, tableName), - TableChange.deleteIndex("k1_index", true), - TableChange.addIndex( - Index.IndexType.PRIMARY_KEY, "k2_index", new String[][] {{DORIS_COL_NAME2}})); - - Awaitility.await() - .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) - .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) - .untilAsserted( - () -> - assertEquals( - 1, - tableCatalog - .loadTable(NameIdentifier.of(schemaName, tableName)) - .index() - .length)); - } -} \ No newline at end of file diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 2f32dc15515..7bc073a0b60 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog.doris.integration.test; +import static org.apache.gravitino.integration.test.util.ITUtils.assertPartition; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -67,6 +68,7 @@ import org.apache.gravitino.rel.partitions.ListPartition; import org.apache.gravitino.rel.partitions.Partition; import org.apache.gravitino.rel.partitions.Partitions; +import org.apache.gravitino.rel.partitions.RangePartition; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.utils.RandomNameUtils; import org.junit.jupiter.api.AfterAll; @@ -98,6 +100,7 @@ public class CatalogDorisIT extends AbstractIT { public String DORIS_COL_NAME1 = "doris_col_name1"; public String DORIS_COL_NAME2 = "doris_col_name2"; public String DORIS_COL_NAME3 = "doris_col_name3"; + public String DORIS_COL_NAME4 = "doris_col_name4"; // Because the creation of Schema Change is an asynchronous process, we need to wait for a while // For more information, you can refer to the comment in @@ -196,8 +199,9 @@ private Column[] createColumns() { Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment", false, false, null); Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10), "col_2_comment"); Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10), "col_3_comment"); - - return new Column[] {col1, col2, col3}; + Column col4 = + Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment", false, false, null); + return new Column[] {col1, col2, col3, col4}; } private Map createTableProperties() { @@ -540,29 +544,29 @@ void testAlterDorisTable() { tableCatalog.alterTable( tableIdentifier, TableChange.addColumn( - new String[] {"col_4"}, Types.VarCharType.of(255), "col_4_comment", true)); + new String[] {"col_5"}, Types.VarCharType.of(255), "col_5_comment", true)); Awaitility.await() .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) .untilAsserted( - () -> assertEquals(4, tableCatalog.loadTable(tableIdentifier).columns().length)); + () -> assertEquals(5, tableCatalog.loadTable(tableIdentifier).columns().length)); ITUtils.assertColumn( - Column.of("col_4", Types.VarCharType.of(255), "col_4_comment"), - tableCatalog.loadTable(tableIdentifier).columns()[3]); + Column.of("col_5", Types.VarCharType.of(255), "col_5_comment"), + tableCatalog.loadTable(tableIdentifier).columns()[4]); // change column position // TODO: change column position is unstable, add it later // drop column tableCatalog.alterTable( - tableIdentifier, TableChange.deleteColumn(new String[] {"col_4"}, true)); + tableIdentifier, TableChange.deleteColumn(new String[] {"col_5"}, true)); Awaitility.await() .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) .untilAsserted( - () -> assertEquals(3, tableCatalog.loadTable(tableIdentifier).columns().length)); + () -> assertEquals(4, tableCatalog.loadTable(tableIdentifier).columns().length)); } @Test @@ -703,23 +707,11 @@ void testDorisTablePartitionOperation() { ListPartition p2 = Partitions.list("p2", p2Values, Collections.emptyMap()); ListPartition p3 = Partitions.list("p3", p3Values, Collections.emptyMap()); ListPartition p1Added = (ListPartition) tablePartitionOperations.addPartition(p1); - assertEquals("p1", p1Added.name()); - assertEquals(1, p1Added.lists().length); - assertEquals(1, p1Added.lists()[0].length); - assertEquals("1", p1Added.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), p1Added.lists()[0][0].dataType()); + assertPartition(p1, p1Added); ListPartition p2Added = (ListPartition) tablePartitionOperations.addPartition(p2); - assertEquals("p2", p2Added.name()); - assertEquals(1, p2Added.lists().length); - assertEquals(1, p2Added.lists()[0].length); - assertEquals("2", p2Added.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), p2Added.lists()[0][0].dataType()); + assertPartition(p2, p2Added); ListPartition p3Added = (ListPartition) tablePartitionOperations.addPartition(p3); - assertEquals("p3", p3Added.name()); - assertEquals(1, p3Added.lists().length); - assertEquals(1, p3Added.lists()[0].length); - assertEquals("3", p3Added.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), p3Added.lists()[0][0].dataType()); + assertPartition(p3, p3Added); // check partitions Set partitionNames = @@ -733,43 +725,13 @@ void testDorisTablePartitionOperation() { Arrays.stream(tablePartitionOperations.listPartitions()) .collect(Collectors.toMap(p -> p.name(), p -> (ListPartition) p)); assertEquals(3, partitions.size()); - ListPartition actualP1 = partitions.get("p1"); - assertEquals("p1", actualP1.name()); - assertEquals(1, actualP1.lists().length); - assertEquals(1, actualP1.lists()[0].length); - assertEquals("1", actualP1.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP1.lists()[0][0].dataType()); - ListPartition actualP2 = partitions.get("p2"); - assertEquals("p2", actualP2.name()); - assertEquals(1, actualP2.lists().length); - assertEquals(1, actualP2.lists()[0].length); - assertEquals("2", actualP2.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP2.lists()[0][0].dataType()); - ListPartition actualP3 = partitions.get("p3"); - assertEquals("p3", actualP3.name()); - assertEquals(1, actualP3.lists().length); - assertEquals(1, actualP3.lists()[0].length); - assertEquals("3", actualP3.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP3.lists()[0][0].dataType()); - - actualP1 = (ListPartition) tablePartitionOperations.getPartition("p1"); - assertEquals("p1", actualP1.name()); - assertEquals(1, actualP1.lists().length); - assertEquals(1, actualP1.lists()[0].length); - assertEquals("1", actualP1.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP1.lists()[0][0].dataType()); - actualP2 = (ListPartition) tablePartitionOperations.getPartition("p2"); - assertEquals("p2", actualP2.name()); - assertEquals(1, actualP2.lists().length); - assertEquals(1, actualP2.lists()[0].length); - assertEquals("2", actualP2.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP2.lists()[0][0].dataType()); - actualP3 = (ListPartition) tablePartitionOperations.getPartition("p3"); - assertEquals("p3", actualP3.name()); - assertEquals(1, actualP3.lists().length); - assertEquals(1, actualP3.lists()[0].length); - assertEquals("3", actualP3.lists()[0][0].value()); - assertEquals(Types.IntegerType.get(), actualP3.lists()[0][0].dataType()); + assertPartition(p1, partitions.get("p1")); + assertPartition(p2, partitions.get("p2")); + assertPartition(p3, partitions.get("p3")); + + assertPartition(p1, tablePartitionOperations.getPartition("p1")); + assertPartition(p2, tablePartitionOperations.getPartition("p2")); + assertPartition(p3, tablePartitionOperations.getPartition("p3")); // drop partition assertTrue(tablePartitionOperations.dropPartition("p3")); @@ -783,6 +745,124 @@ void testDorisTablePartitionOperation() { assertFalse(tablePartitionOperations.dropPartition("p3")); } + @Test + void testCreatePartitionedTable() { + // create a range-partitioned table with assignments + String tableName = GravitinoITUtils.genRandomName("test_create_range_partitioned_table"); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); + Column[] columns = createColumns(); + Distribution distribution = createDistribution(); + Index[] indexes = + new Index[] { + Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}) + }; + Map properties = createTableProperties(); + Literal todayLiteral = Literals.of("2024-07-24", Types.DateType.get()); + Literal tomorrowLiteral = Literals.of("2024-07-25", Types.DateType.get()); + RangePartition p1 = Partitions.range("p1", todayLiteral, Literals.NULL, Collections.emptyMap()); + RangePartition p2 = + Partitions.range("p2", tomorrowLiteral, todayLiteral, Collections.emptyMap()); + RangePartition p3 = + Partitions.range("p3", Literals.NULL, tomorrowLiteral, Collections.emptyMap()); + Transform[] partitioning = { + Transforms.range(new String[] {DORIS_COL_NAME4}, new RangePartition[] {p1, p2, p3}) + }; + TableCatalog tableCatalog = catalog.asTableCatalog(); + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + null, + indexes); + Table loadedTable = tableCatalog.loadTable(tableIdentifier); + ITUtils.assertionsTableInfo( + tableName, + table_comment, + Arrays.asList(columns), + properties, + indexes, + new Transform[] {Transforms.range(new String[] {DORIS_COL_NAME4})}, + loadedTable); + + // assert partition info + SupportsPartitions tablePartitionOperations = loadedTable.supportPartitions(); + Map loadedRangePartitions = + Arrays.stream(tablePartitionOperations.listPartitions()) + .collect(Collectors.toMap(Partition::name, p -> (RangePartition) p)); + assertTrue(loadedRangePartitions.size() == 3); + assertTrue(loadedRangePartitions.containsKey("p1")); + assertPartition( + Partitions.range( + "p1", + todayLiteral, + Literals.of("0000-01-01", Types.DateType.get()), + Collections.emptyMap()), + loadedRangePartitions.get("p1")); + assertTrue(loadedRangePartitions.containsKey("p2")); + assertPartition(p2, loadedRangePartitions.get("p2")); + assertTrue(loadedRangePartitions.containsKey("p3")); + assertPartition( + Partitions.range( + "p3", + Literals.of("MAXVALUE", Types.DateType.get()), + tomorrowLiteral, + Collections.emptyMap()), + loadedRangePartitions.get("p3")); + + // create a list-partitioned table with assignments + tableName = GravitinoITUtils.genRandomName("test_create_list_partitioned_table"); + tableIdentifier = NameIdentifier.of(schemaName, tableName); + Literal integerLiteral1 = Literals.integerLiteral(1); + Literal integerLiteral2 = Literals.integerLiteral(2); + ListPartition p4 = + Partitions.list( + "p4", + new Literal[][] {{integerLiteral1, todayLiteral}, {integerLiteral1, tomorrowLiteral}}, + Collections.emptyMap()); + ListPartition p5 = + Partitions.list( + "p5", + new Literal[][] {{integerLiteral2, todayLiteral}, {integerLiteral2, tomorrowLiteral}}, + Collections.emptyMap()); + partitioning = + new Transform[] { + Transforms.list( + new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}}, new ListPartition[] {p4, p5}) + }; + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + null, + indexes); + loadedTable = tableCatalog.loadTable(tableIdentifier); + ITUtils.assertionsTableInfo( + tableName, + table_comment, + Arrays.asList(columns), + properties, + indexes, + new Transform[] {Transforms.list(new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}})}, + loadedTable); + + // assert partition info + tablePartitionOperations = loadedTable.supportPartitions(); + Map loadedListPartitions = + Arrays.stream(tablePartitionOperations.listPartitions()) + .collect(Collectors.toMap(Partition::name, p -> (ListPartition) p)); + assertTrue(loadedListPartitions.size() == 2); + assertTrue(loadedListPartitions.containsKey("p4")); + assertPartition(p4, loadedListPartitions.get("p4")); + assertTrue(loadedListPartitions.containsKey("p5")); + assertPartition(p5, loadedListPartitions.get("p5")); + } + @Test void testNonPartitionedTable() { // create a non-partitioned table diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java index 169f9759248..ee5de70f38a 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -18,6 +18,9 @@ */ package org.apache.gravitino.catalog.doris.operation; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import com.google.common.collect.Maps; import java.time.LocalDate; import java.util.ArrayList; @@ -27,17 +30,27 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.gravitino.catalog.doris.converter.DorisTypeConverter; import org.apache.gravitino.catalog.jdbc.JdbcColumn; import org.apache.gravitino.catalog.jdbc.JdbcTable; +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.catalog.jdbc.operation.JdbcTablePartitionOperations; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.TableChange; import org.apache.gravitino.rel.expressions.NamedReference; import org.apache.gravitino.rel.expressions.distributions.Distribution; import org.apache.gravitino.rel.expressions.distributions.Distributions; +import org.apache.gravitino.rel.expressions.literals.Literal; +import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.expressions.transforms.Transforms; import org.apache.gravitino.rel.indexes.Index; import org.apache.gravitino.rel.indexes.Indexes; +import org.apache.gravitino.rel.partitions.ListPartition; +import org.apache.gravitino.rel.partitions.Partition; +import org.apache.gravitino.rel.partitions.Partitions; +import org.apache.gravitino.rel.partitions.RangePartition; import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.utils.RandomNameUtils; @@ -49,6 +62,7 @@ @Tag("gravitino-docker-test") public class TestDorisTableOperations extends TestDoris { + private static final JdbcTypeConverter TYPE_CONVERTER = new DorisTypeConverter(); private static final Type VARCHAR_255 = Types.VarCharType.of(255); private static final Type VARCHAR_1024 = Types.VarCharType.of(1024); @@ -112,7 +126,7 @@ public void testBasicTableOperation() { distribution, indexes); List listTables = TABLE_OPERATIONS.listTables(databaseName); - Assertions.assertTrue(listTables.contains(tableName)); + assertTrue(listTables.contains(tableName)); JdbcTable load = TABLE_OPERATIONS.load(databaseName, tableName); assertionsTableInfo( tableName, tableComment, columns, properties, indexes, Transforms.EMPTY_TRANSFORM, load); @@ -445,7 +459,7 @@ public void testCreateNotSupportTypeTable() { } @Test - public void testCreateTableWithPartition() { + public void testCreatePartitionedTable() { String tableComment = "partition_table_comment"; JdbcColumn col1 = JdbcColumn.builder() @@ -472,8 +486,8 @@ public void testCreateTableWithPartition() { String rangePartitionTableName = GravitinoITUtils.genRandomName("range_partition_table"); LocalDate today = LocalDate.now(); LocalDate tomorrow = today.plusDays(1); - Literals.LiteralImpl todayLiteral = Literals.dateLiteral(today); - Literals.LiteralImpl tomorrowLiteral = Literals.dateLiteral(tomorrow); + Literal todayLiteral = Literals.dateLiteral(today); + Literal tomorrowLiteral = Literals.dateLiteral(tomorrow); RangePartition rangePartition1 = Partitions.range("p1", todayLiteral, Literals.NULL, null); RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral, todayLiteral, null); RangePartition rangePartition3 = Partitions.range("p3", Literals.NULL, tomorrowLiteral, null); @@ -499,13 +513,33 @@ public void testCreateTableWithPartition() { columns, Collections.emptyMap(), null, - rangePartition, + new Transform[] {Transforms.range(new String[] {col4.name()})}, rangePartitionTable); + // assert partition info + JdbcTablePartitionOperations tablePartitionOperations = + new DorisTablePartitionOperations( + DATA_SOURCE, rangePartitionTable, JDBC_EXCEPTION_CONVERTER, TYPE_CONVERTER); + Map loadedRangePartitions = + Arrays.stream(tablePartitionOperations.listPartitions()) + .collect(Collectors.toMap(Partition::name, p -> (RangePartition) p)); + assertTrue(loadedRangePartitions.containsKey("p1")); + RangePartition actualP1 = loadedRangePartitions.get("p1"); + assertEquals(todayLiteral, actualP1.upper()); + assertEquals(Literals.of("0000-01-01", Types.DateType.get()), actualP1.lower()); + assertTrue(loadedRangePartitions.containsKey("p2")); + RangePartition actualP2 = loadedRangePartitions.get("p2"); + assertEquals(tomorrowLiteral, actualP2.upper()); + assertEquals(todayLiteral, actualP2.lower()); + assertTrue(loadedRangePartitions.containsKey("p3")); + RangePartition actualP3 = loadedRangePartitions.get("p3"); + assertEquals(Literals.of("MAXVALUE", Types.DateType.get()), actualP3.upper()); + assertEquals(tomorrowLiteral, actualP3.lower()); + // create table with list partition String listPartitionTableName = GravitinoITUtils.genRandomName("list_partition_table"); - Literals.LiteralImpl integerLiteral1 = Literals.integerLiteral(1); - Literals.LiteralImpl integerLiteral2 = Literals.integerLiteral(2); + Literal integerLiteral1 = Literals.integerLiteral(1); + Literal integerLiteral2 = Literals.integerLiteral(2); ListPartition listPartition1 = Partitions.list( "p1", @@ -538,7 +572,19 @@ public void testCreateTableWithPartition() { columns, Collections.emptyMap(), null, - listPartition, + new Transform[] {Transforms.list(new String[][] {{col1.name()}, {col4.name()}})}, listPartitionTable); + + // assert partition info + tablePartitionOperations = + new DorisTablePartitionOperations( + DATA_SOURCE, listPartitionTable, JDBC_EXCEPTION_CONVERTER, TYPE_CONVERTER); + Map loadedListPartitions = + Arrays.stream(tablePartitionOperations.listPartitions()) + .collect(Collectors.toMap(Partition::name, p -> (ListPartition) p, (p1, p2) -> p2)); + assertTrue(loadedListPartitions.containsKey("p1")); + assertTrue(Arrays.deepEquals(listPartition1.lists(), loadedListPartitions.get("p1").lists())); + assertTrue(loadedListPartitions.containsKey("p2")); + assertTrue(Arrays.deepEquals(listPartition2.lists(), loadedListPartitions.get("p2").lists())); } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java index f1428f2b011..26d011af8e7 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java @@ -18,13 +18,21 @@ */ package org.apache.gravitino.catalog.doris.utils; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import org.apache.gravitino.rel.expressions.literals.Literal; +import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.expressions.transforms.Transforms; -import org.junit.jupiter.api.Assertions; +import org.apache.gravitino.rel.partitions.Partition; +import org.apache.gravitino.rel.partitions.Partitions; +import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Test; public class TestDorisUtils { @@ -33,17 +41,17 @@ public void testGeneratePropertiesSql() { // Test when properties is null Map properties = null; String result = DorisUtils.generatePropertiesSql(properties); - Assertions.assertEquals("", result); + assertEquals("", result); // Test when properties is empty properties = Collections.emptyMap(); result = DorisUtils.generatePropertiesSql(properties); - Assertions.assertEquals("", result); + assertEquals("", result); // Test when properties has single entry properties = Collections.singletonMap("key", "value"); result = DorisUtils.generatePropertiesSql(properties); - Assertions.assertEquals(" PROPERTIES (\n\"key\"=\"value\"\n)", result); + assertEquals(" PROPERTIES (\n\"key\"=\"value\"\n)", result); // Test when properties has multiple entries properties = new HashMap<>(); @@ -53,7 +61,7 @@ public void testGeneratePropertiesSql() { String expectedStr = " PROPERTIES (\n\"key1\"=\"value1\",\n\"key2\"=\"value2\"\n)"; result = DorisUtils.generatePropertiesSql(properties); - Assertions.assertEquals(expectedStr, result); + assertEquals(expectedStr, result); } @Test @@ -62,27 +70,27 @@ public void testExtractTablePropertiesFromSql() { String createTableSql = "CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT 'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\""; Map result = DorisUtils.extractPropertiesFromSql(createTableSql); - Assertions.assertTrue(result.isEmpty()); + assertTrue(result.isEmpty()); // Test when properties exist createTableSql = "CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT 'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"\nPROPERTIES (\n\"test_property\"=\"test_value\"\n)"; result = DorisUtils.extractPropertiesFromSql(createTableSql); - Assertions.assertEquals("test_value", result.get("test_property")); + assertEquals("test_value", result.get("test_property")); // Test when multiple properties exist createTableSql = "CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT 'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"\nPROPERTIES (\n\"test_property1\"=\"test_value1\",\n\"test_property2\"=\"test_value2\"\n)"; result = DorisUtils.extractPropertiesFromSql(createTableSql); - Assertions.assertEquals("test_value1", result.get("test_property1")); - Assertions.assertEquals("test_value2", result.get("test_property2")); + assertEquals("test_value1", result.get("test_property1")); + assertEquals("test_value2", result.get("test_property2")); // test when properties has blank createTableSql = "CREATE DATABASE `test`\nPROPERTIES (\n\"property1\" = \"value1\",\n\"comment\"= \"comment\"\n)"; result = DorisUtils.extractPropertiesFromSql(createTableSql); - Assertions.assertEquals("value1", result.get("property1")); - Assertions.assertEquals("comment", result.get("comment")); + assertEquals("value1", result.get("property1")); + assertEquals("comment", result.get("comment")); } @Test @@ -91,27 +99,76 @@ public void testExtractPartitionInfoFromSql() { String createTableSql = "CREATE TABLE `testTable` (\n`col1` date NOT NULL\n) ENGINE=OLAP\n PARTITION BY RANGE(`col1`)\n()\n DISTRIBUTED BY HASH(`col1`) BUCKETS 2"; Optional transform = DorisUtils.extractPartitionInfoFromSql(createTableSql); - Assertions.assertTrue(transform.isPresent()); - Assertions.assertEquals(Transforms.range(new String[] {"col1"}), transform.get()); + assertTrue(transform.isPresent()); + assertEquals(Transforms.range(new String[] {"col1"}), transform.get()); // test list partition createTableSql = "CREATE TABLE `testTable` (\n`col1` int(11) NOT NULL\n) ENGINE=OLAP\n PARTITION BY LIST(`col1`)\n()\n DISTRIBUTED BY HASH(`col1`) BUCKETS 2"; transform = DorisUtils.extractPartitionInfoFromSql(createTableSql); - Assertions.assertTrue(transform.isPresent()); - Assertions.assertEquals(Transforms.list(new String[][] {{"col1"}}), transform.get()); + assertTrue(transform.isPresent()); + assertEquals(Transforms.list(new String[][] {{"col1"}}), transform.get()); // test multi-column list partition createTableSql = "CREATE TABLE `testTable` (\n`col1` date NOT NULL,\n`col2` int(11) NOT NULL\n) ENGINE=OLAP\n PARTITION BY LIST(`col1`, `col2`)\n()\n DISTRIBUTED BY HASH(`col1`) BUCKETS 2"; transform = DorisUtils.extractPartitionInfoFromSql(createTableSql); - Assertions.assertTrue(transform.isPresent()); - Assertions.assertEquals(Transforms.list(new String[][] {{"col1"}, {"col2"}}), transform.get()); + assertTrue(transform.isPresent()); + assertEquals(Transforms.list(new String[][] {{"col1"}, {"col2"}}), transform.get()); // test non-partitioned table createTableSql = "CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT 'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\""; transform = DorisUtils.extractPartitionInfoFromSql(createTableSql); - Assertions.assertFalse(transform.isPresent()); + assertFalse(transform.isPresent()); + } + + @Test + public void testGeneratePartitionSqlFragment() { + // test range partition + Partition partition = Partitions.range("p1", Literals.NULL, Literals.NULL, null); + String partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p1` VALUES LESS THAN MAXVALUE", partitionSqlFragment); + + partition = + Partitions.range( + "p2", Literals.of("2024-07-23", Types.DateType.get()), Literals.NULL, null); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p2` VALUES LESS THAN (\"2024-07-23\")", partitionSqlFragment); + + partition = + Partitions.range( + "p3", + Literals.of("2024-07-24", Types.DateType.get()), + Literals.of("2024-07-23", Types.DateType.get()), + null); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals( + "PARTITION `p3` VALUES [(\"2024-07-23\"), (\"2024-07-24\"))", partitionSqlFragment); + + partition = + Partitions.range( + "p4", Literals.NULL, Literals.of("2024-07-24", Types.DateType.get()), null); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p4` VALUES [(\"2024-07-24\"), (MAXVALUE))", partitionSqlFragment); + + // test list partition + Literal[][] p5values = {{Literals.of("2024-07-24", Types.DateType.get())}}; + partition = Partitions.list("p5", p5values, Collections.emptyMap()); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p5` VALUES IN (\"2024-07-24\")", partitionSqlFragment); + + Literal[][] p6values = {{Literals.integerLiteral(1)}, {Literals.integerLiteral(2)}}; + partition = Partitions.list("p6", p6values, Collections.emptyMap()); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p6` VALUES IN (\"1\",\"2\")", partitionSqlFragment); + + Literal[][] p7values = { + {Literals.integerLiteral(1), Literals.integerLiteral(2)}, + {Literals.integerLiteral(3), Literals.integerLiteral(4)} + }; + partition = Partitions.list("p7", p7values, Collections.emptyMap()); + partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition); + assertEquals("PARTITION `p7` VALUES IN ((\"1\",\"2\"),(\"3\",\"4\"))", partitionSqlFragment); } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java index bb97d04a376..596b0c23c4f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java @@ -35,10 +35,15 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.expressions.LiteralDTO; +import org.apache.gravitino.dto.rel.partitions.IdentityPartitionDTO; +import org.apache.gravitino.dto.rel.partitions.ListPartitionDTO; +import org.apache.gravitino.dto.rel.partitions.PartitionDTO; +import org.apache.gravitino.dto.rel.partitions.RangePartitionDTO; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.indexes.Index; +import org.apache.gravitino.rel.partitions.Partition; import org.junit.jupiter.api.Assertions; public class ITUtils { @@ -139,5 +144,36 @@ public static void assertColumn(Column expected, Column actual) { } } + public static void assertPartition(Partition expected, Partition actual) { + if (!(expected instanceof PartitionDTO)) { + expected = toDTO(expected); + } + if (!(actual instanceof PartitionDTO)) { + actual = toDTO(actual); + } + + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(((PartitionDTO) expected).type(), ((PartitionDTO) actual).type()); + if (expected instanceof RangePartitionDTO) { + Assertions.assertEquals( + ((RangePartitionDTO) expected).upper(), ((RangePartitionDTO) actual).upper()); + Assertions.assertEquals( + ((RangePartitionDTO) expected).lower(), ((RangePartitionDTO) actual).lower()); + } else if (expected instanceof ListPartitionDTO) { + Assertions.assertTrue( + Arrays.deepEquals( + ((ListPartitionDTO) expected).lists(), ((ListPartitionDTO) actual).lists())); + } else if (expected instanceof IdentityPartitionDTO) { + Assertions.assertTrue( + Arrays.deepEquals( + ((IdentityPartitionDTO) expected).fieldNames(), + ((IdentityPartitionDTO) actual).fieldNames())); + Assertions.assertTrue( + Arrays.equals( + ((IdentityPartitionDTO) expected).values(), + ((IdentityPartitionDTO) actual).values())); + } + } + private ITUtils() {} }