From ebd801a061b219f1ae047c3dd5cc1297ea0fb1be Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 19 Nov 2024 11:07:18 +0800 Subject: [PATCH] Support Iceberg partition transform with multi partition columns Signed-off-by: shuming.li --- .../starrocks/catalog/MaterializedView.java | 18 + .../iceberg/IcebergPartitionUtils.java | 108 ++- .../mv/MVPCTRefreshListPartitioner.java | 87 ++- .../com/starrocks/server/LocalMetastore.java | 34 +- .../sql/analyzer/AnalyzeStmtAnalyzer.java | 2 +- .../analyzer/MaterializedViewAnalyzer.java | 270 +++++++- .../ast/CreateMaterializedViewStatement.java | 7 + .../com/starrocks/sql/common/PListCell.java | 25 + .../rewrite/OptExternalPartitionPruner.java | 2 +- .../materialization/MvUtils.java | 17 +- .../compensation/MVCompensationBuilder.java | 54 +- .../sql/plan/PlanFragmentBuilder.java | 2 +- .../analysis/CreateMaterializedViewTest.java | 2 +- .../analysis/RefreshMaterializedViewTest.java | 17 +- .../PCTRefreshListPartitionOlapTest.java | 30 + ...ionBasedMvRefreshProcessorIcebergTest.java | 63 +- .../MaterializedViewAnalyzerTest.java | 5 +- ...MvTransparentRewriteWithOlapTableTest.java | 108 +++ ...t_mv_with_multi_partition_columns_iceberg1 | 459 +++++++++++++ ...t_mv_with_multi_partition_columns_iceberg2 | 625 ++++++++++++++++++ ...t_mv_with_multi_partition_columns_iceberg1 | 292 ++++++++ ...t_mv_with_multi_partition_columns_iceberg2 | 223 +++++++ 22 files changed, 2281 insertions(+), 169 deletions(-) create mode 100644 test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg1 create mode 100644 test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg2 create mode 100644 test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg1 create mode 100644 test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg2 diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java index 1cd9bc171cdd2..7c364a7a85371 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java @@ -2098,4 +2098,22 @@ public synchronized ParseNode getDefineQueryParseNode() { } return this.defineQueryParseNode; } + + @Override + public List getBaseSchema() { + if (!hasGeneratedColumn()) { + return getSchemaByIndexId(baseIndexId); + } + + List schema = Lists.newArrayList(getSchemaByIndexId(baseIndexId)); + while (schema.size() > 0) { + // check last column is whether is a generated column or not + if (schema.get(schema.size() - 1).isGeneratedColumn()) { + schema.remove(schema.size() - 1); + } else { + break; + } + } + return schema; + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergPartitionUtils.java index d70d52b3fe142..75449c5da3fd3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergPartitionUtils.java @@ -14,20 +14,14 @@ package com.starrocks.connector.iceberg; -import autovalue.shaded.com.google.common.common.collect.ImmutableList; import com.google.common.base.Preconditions; -import com.starrocks.analysis.Expr; -import com.starrocks.analysis.FunctionCallExpr; -import com.starrocks.analysis.SlotRef; +import com.google.common.collect.Range; import com.starrocks.catalog.Column; -import com.starrocks.catalog.Function; -import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.IcebergTable; import com.starrocks.catalog.Type; import com.starrocks.common.util.TimeUtils; import com.starrocks.connector.PartitionUtil; import com.starrocks.connector.exception.StarRocksConnectorException; -import com.starrocks.sql.analyzer.FunctionAnalyzer; import com.starrocks.statistic.StatisticUtils; import org.apache.iceberg.PartitionField; import org.apache.iceberg.Schema; @@ -53,14 +47,18 @@ public class IcebergPartitionUtils { // month(ts) partitionName : 2023-01 return 2023-01-01 (Date) or 2023-01-01 00:00:00 (Datetime) // day(ts) partitionName : 2023-01-01 return 2023-01-01 (Date) or 2023-01-01 00:00:00 (Datetime) // hour(ts) partitionName : 2023-01-01-12 return 2023-01-01 12:00:00 (Datetime) - public static String normalizeTimePartitionName(String partitionName, PartitionField partitionField, Schema schema, + public static String normalizeTimePartitionName(String partitionName, + PartitionField partitionField, + Schema schema, Type type) { DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); boolean parseFromDate = true; IcebergPartitionTransform transform = IcebergPartitionTransform.fromString(partitionField.transform().toString()); if (transform == YEAR) { + Preconditions.checkArgument(partitionName.length() == 4, "Invalid partition name: %s", partitionName); partitionName += "-01-01"; } else if (transform == IcebergPartitionTransform.MONTH) { + Preconditions.checkArgument(partitionName.length() == 7, "Invalid partition name: %s", partitionName); partitionName += "-01"; } else if (transform == IcebergPartitionTransform.DAY) { dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); @@ -89,21 +87,18 @@ public static String normalizeTimePartitionName(String partitionName, PartitionF String result; try { + LocalDateTime datetime; if (parseFromDate) { - LocalDate date = LocalDate.parse(partitionName, dateTimeFormatter); - if (type.isDate()) { - result = date.format(formatter); - } else { - LocalDateTime dateTime = date.atStartOfDay().atZone(ZoneOffset.UTC). - withZoneSameInstant(zoneId).toLocalDateTime(); - result = dateTime.format(formatter); - } + // since it's from date, it can be converted to LocalDateTime by atStartOfDay + datetime = LocalDate.parse(partitionName, dateTimeFormatter).atStartOfDay(); } else { // parse from datetime which contains hour - LocalDateTime dateTime = LocalDateTime.parse(partitionName, dateTimeFormatter).atZone(ZoneOffset.UTC). - withZoneSameInstant(zoneId).toLocalDateTime(); - result = dateTime.format(formatter); + datetime = LocalDateTime.parse(partitionName, dateTimeFormatter); } + // convert from UTC to local time + LocalDateTime localDateTime = convertTimezone(datetime, ZoneOffset.UTC, zoneId); + // format to string + result = localDateTime.format(formatter); } catch (Exception e) { LOG.warn("parse partition name failed, partitionName: {}, partitionField: {}, type: {}", partitionName, partitionField, type); @@ -112,6 +107,10 @@ public static String normalizeTimePartitionName(String partitionName, PartitionF return result; } + public static LocalDateTime convertTimezone(LocalDateTime time, ZoneId from, ZoneId to) { + return time.atZone(from).withZoneSameInstant(to).toLocalDateTime(); + } + // Get the date interval from iceberg partition transform public static PartitionUtil.DateTimeInterval getDateTimeIntervalFromIceberg(IcebergTable table, Column partitionColumn) { @@ -180,8 +179,9 @@ public static LocalDateTime addDateTimeInterval(LocalDateTime dateTime, IcebergP partitionValue: 2023-01-01-12 transform: hour return ts >= '2023-01-01 12:00:00' and ts < '2023-01-01 13:00:00' */ - public static String convertPartitionFieldToPredicate(IcebergTable table, String partitionColumn, - String partitionValue) { + public static Range toPartitionRange(IcebergTable table, String partitionColumn, + String partitionValue, + boolean isFromIcebergTime) { PartitionField partitionField = table.getPartitionFiled(partitionColumn); if (partitionField == null) { throw new StarRocksConnectorException("Partition column %s not found in table %s.%s.%s", @@ -189,73 +189,41 @@ public static String convertPartitionFieldToPredicate(IcebergTable table, String } IcebergPartitionTransform transform = IcebergPartitionTransform.fromString(partitionField.transform().toString()); if (transform == IcebergPartitionTransform.IDENTITY) { - return StatisticUtils.quoting(partitionColumn) + " = '" + partitionValue + "'"; + return Range.singleton(partitionValue); } else { // transform is year, month, day, hour Type partitiopnColumnType = table.getColumn(partitionColumn).getType(); Preconditions.checkState(partitiopnColumnType.isDateType(), "Partition column %s type must be date or datetime", partitionColumn); - String normalizedPartitionValue = normalizeTimePartitionName(partitionValue, partitionField, - table.getNativeTable().schema(), partitiopnColumnType); - + if (isFromIcebergTime) { + partitionValue = normalizeTimePartitionName(partitionValue, partitionField, + table.getNativeTable().schema(), partitiopnColumnType); + } LocalDateTime startDateTime = null; DateTimeFormatter dateTimeFormatter = null; if (partitiopnColumnType.isDate()) { dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - startDateTime = LocalDate.parse(normalizedPartitionValue, dateTimeFormatter).atStartOfDay(); + startDateTime = LocalDate.parse(partitionValue, dateTimeFormatter).atStartOfDay(); } else { dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - startDateTime = LocalDateTime.parse(normalizedPartitionValue, dateTimeFormatter); + startDateTime = LocalDateTime.parse(partitionValue, dateTimeFormatter); } LocalDateTime endDateTime = addDateTimeInterval(startDateTime, transform); String endDateTimeStr = endDateTime.format(dateTimeFormatter); - - return StatisticUtils.quoting(partitionColumn) + " >= '" + normalizedPartitionValue + "' and " + - StatisticUtils.quoting(partitionColumn) + " < '" + endDateTimeStr + "'"; + return Range.closedOpen(partitionValue, endDateTimeStr); } } - public static Expr toTransformExpr(IcebergTable table, - String partitionColumn, - SlotRef slotRef) { - PartitionField partitionField = table.getPartitionFiled(partitionColumn); - if (partitionField == null) { - throw new StarRocksConnectorException("Partition column %s not found in table %s.%s.%s", - partitionColumn, table.getCatalogName(), table.getRemoteDbName(), table.getRemoteTableName()); - } - IcebergPartitionTransform transform = IcebergPartitionTransform.fromString(partitionField.transform().toString()); - if (transform == IcebergPartitionTransform.IDENTITY) { - return slotRef; + public static String convertPartitionFieldToPredicate(IcebergTable table, String partitionColumn, + String partitionValue) { + Range range = toPartitionRange(table, partitionColumn, partitionValue, true); + String partitionCol = StatisticUtils.quoting(partitionColumn); + if (range.lowerEndpoint().equals(range.upperEndpoint())) { + return String.format("%s = '%s'", partitionCol, range.lowerEndpoint()); } else { - // transform is year, month, day, hour - Type partitiopnColumnType = table.getColumn(partitionColumn).getType(); - Preconditions.checkState(partitiopnColumnType.isDateType(), - "Partition column %s type must be date or datetime", partitionColumn); - String transformFuncName; - switch (transform) { - case HOUR: - transformFuncName = FunctionSet.HOUR; - break; - case DAY: - transformFuncName = FunctionSet.DAY; - break; - case MONTH: - transformFuncName = FunctionSet.MONTH; - break; - case YEAR: - transformFuncName = FunctionSet.YEAR; - break; - default: - throw new StarRocksConnectorException("Unsupported partition transform to add: %s", transform); - } - Type[] argTypes = {slotRef.getType()}; - Function transformFn = Expr.getBuiltinFunction(transformFuncName, - argTypes, Function.CompareMode.IS_IDENTICAL); - FunctionCallExpr result = new FunctionCallExpr(transformFuncName, ImmutableList.of(slotRef)); - result.setFn(transformFn); - result.setType(slotRef.getType()); - FunctionAnalyzer.analyze(result); - return result; + String lowerEndpoint = range.lowerEndpoint(); + String upperEndpoint = range.upperEndpoint(); + return String.format("%s >= '%s' and %s < '%s'", partitionCol, lowerEndpoint, partitionCol, upperEndpoint); } } } \ No newline at end of file diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java index 3d0b5419b0bea..0be8dafa57fe3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java @@ -16,8 +16,10 @@ package com.starrocks.scheduler.mv; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.starrocks.analysis.BinaryPredicate; import com.starrocks.analysis.BinaryType; @@ -39,7 +41,8 @@ import com.starrocks.common.util.concurrent.lock.LockTimeoutException; import com.starrocks.common.util.concurrent.lock.LockType; import com.starrocks.common.util.concurrent.lock.Locker; -import com.starrocks.connector.iceberg.IcebergPartitionUtils; +import com.starrocks.connector.exception.StarRocksConnectorException; +import com.starrocks.connector.iceberg.IcebergPartitionTransform; import com.starrocks.scheduler.MvTaskRunContext; import com.starrocks.scheduler.TableSnapshotInfo; import com.starrocks.scheduler.TaskRunContext; @@ -57,6 +60,7 @@ import com.starrocks.sql.common.SyncPartitionUtils; import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import org.apache.commons.collections4.CollectionUtils; +import org.apache.iceberg.PartitionField; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -69,6 +73,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static com.starrocks.connector.iceberg.IcebergPartitionUtils.toPartitionRange; + public final class MVPCTRefreshListPartitioner extends MVPCTRefreshPartitioner { private static final Logger LOG = LogManager.getLogger(MVPCTRefreshListPartitioner.class); @@ -176,14 +182,52 @@ public Expr generatePartitionPredicate(Table refBaseTable, Set refBaseTa refPartitionColumns, baseListPartitionMap); } - private static Expr getRefBaseTablePartitionTransformExpr(Table table, + private static Expr getRefBaseTablePartitionPredicateExpr(Table table, String partitionColumn, - SlotRef slotRef) { + SlotRef slotRef, + List selectedPartitionValues) { if (!(table instanceof IcebergTable)) { - return slotRef; + return MvUtils.convertToInPredicate(slotRef, selectedPartitionValues); + } else { + IcebergTable icebergTable = (IcebergTable) table; + return getIcebergTablePartitionPredicateExpr(icebergTable, partitionColumn, slotRef, selectedPartitionValues); + } + } + + public static Expr getIcebergTablePartitionPredicateExpr(IcebergTable table, + String partitionColumn, + SlotRef slotRef, + List selectedPartitionValues) { + PartitionField partitionField = table.getPartitionFiled(partitionColumn); + if (partitionField == null) { + throw new StarRocksConnectorException("Partition column %s not found in table %s.%s.%s", + partitionColumn, table.getCatalogName(), table.getRemoteDbName(), table.getRemoteTableName()); + } + IcebergPartitionTransform transform = IcebergPartitionTransform.fromString(partitionField.transform().toString()); + if (transform == IcebergPartitionTransform.IDENTITY) { + return MvUtils.convertToInPredicate(slotRef, selectedPartitionValues); + } else { + List exprs = com.google.api.client.util.Lists.newArrayList(); + for (Expr expr : selectedPartitionValues) { + if (!(expr instanceof LiteralExpr)) { + throw new StarRocksConnectorException("Partition value must be literal"); + } + String partitionVal = ((LiteralExpr) expr).getStringValue(); + Range range = toPartitionRange(table, partitionColumn, partitionVal, false); + Preconditions.checkArgument(!range.lowerEndpoint().equals(range.upperEndpoint()), + "Partition value must be range"); + try { + LiteralExpr lowerExpr = LiteralExpr.create(range.lowerEndpoint(), slotRef.getType()); + LiteralExpr upperExpr = LiteralExpr.create(range.upperEndpoint(), slotRef.getType()); + Expr lower = new BinaryPredicate(BinaryType.GE, slotRef, lowerExpr); + Expr upper = new BinaryPredicate(BinaryType.LT, slotRef, upperExpr); + exprs.add(Expr.compoundAnd(ImmutableList.of(lower, upper))); + } catch (AnalysisException e) { + throw new StarRocksConnectorException("Create literal expr failed", e); + } + } + return Expr.compoundOr(exprs); } - IcebergTable icebergTable = (IcebergTable) table; - return IcebergPartitionUtils.toTransformExpr(icebergTable, partitionColumn, slotRef); } private static @Nullable Expr genPartitionPredicate(Table refBaseTable, @@ -194,8 +238,9 @@ private static Expr getRefBaseTablePartitionTransformExpr(Table table, Preconditions.checkArgument(refBaseTablePartitionSlotRefs.size() == refPartitionColumns.size()); if (refPartitionColumns.size() == 1) { boolean isContainsNullPartition = false; - List sourceTablePartitionList = Lists.newArrayList(); - Type partitionType = refPartitionColumns.get(0).getType(); + Column refPartitionColumn = refPartitionColumns.get(0); + List selectedPartitionValues = Lists.newArrayList(); + Type partitionType = refPartitionColumn.getType(); for (String tablePartitionName : refBaseTablePartitionNames) { PListCell cell = baseListPartitionMap.get(tablePartitionName); for (List values : cell.getPartitionItems()) { @@ -207,16 +252,16 @@ private static Expr getRefBaseTablePartitionTransformExpr(Table table, isContainsNullPartition = true; continue; } - sourceTablePartitionList.add(partitionValue); + selectedPartitionValues.add(partitionValue); } } - Expr refBaseTablePartitionExpr = getRefBaseTablePartitionTransformExpr(refBaseTable, - refPartitionColumns.get(0).getName(), (SlotRef) refBaseTablePartitionSlotRefs.get(0)); - Expr inPredicate = MvUtils.convertToInPredicate(refBaseTablePartitionExpr, sourceTablePartitionList); + SlotRef refBaseTablePartitionSlotRef = (SlotRef) refBaseTablePartitionSlotRefs.get(0); + Expr inPredicate = getRefBaseTablePartitionPredicateExpr(refBaseTable, refPartitionColumn.getName(), + refBaseTablePartitionSlotRef, selectedPartitionValues); // NOTE: If target partition values contain `null partition`, the generated predicate should // contain `is null` predicate rather than `in (null) or = null` because the later one is not correct. if (isContainsNullPartition) { - IsNullPredicate isNullPredicate = new IsNullPredicate(refBaseTablePartitionExpr, false); + IsNullPredicate isNullPredicate = new IsNullPredicate(refBaseTablePartitionSlotRef, false); return Expr.compoundOr(Lists.newArrayList(inPredicate, isNullPredicate)); } else { return inPredicate; @@ -231,20 +276,20 @@ private static Expr getRefBaseTablePartitionTransformExpr(Table table, } List predicates = Lists.newArrayList(); for (int i = 0; i < refPartitionColumns.size(); i++) { - Type partitionType = refPartitionColumns.get(i).getType(); + Column refPartitionColumn = refPartitionColumns.get(i); + Type partitionType = refPartitionColumn.getType(); LiteralExpr partitionValue = new PartitionValue(values.get(i)).getValue(partitionType); - Expr refBaseTablePartitionExpr = getRefBaseTablePartitionTransformExpr(refBaseTable, - refPartitionColumns.get(i).getName(), (SlotRef) refBaseTablePartitionSlotRefs.get(i)); + Expr refBaseTablePartitionExpr = refBaseTablePartitionSlotRefs.get(i); + Expr predicate; if (partitionValue.isConstantNull()) { // NOTE: If target partition values contain `null partition`, the generated predicate should // contain `is null` predicate rather than `in (null) or = null` because the later one is not correct. - IsNullPredicate isNullPredicate = new IsNullPredicate(refBaseTablePartitionExpr, false); - predicates.add(isNullPredicate); + predicate = new IsNullPredicate(refBaseTablePartitionExpr, false); } else { - BinaryPredicate binaryPredicate = new BinaryPredicate(BinaryType.EQ, refBaseTablePartitionExpr, - partitionValue); - predicates.add(binaryPredicate); + predicate = getRefBaseTablePartitionPredicateExpr(refBaseTable, refPartitionColumn.getName(), + (SlotRef) refBaseTablePartitionExpr, Lists.newArrayList(partitionValue)); } + predicates.add(predicate); } partitionPredicates.add(Expr.compoundAnd(predicates)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 3e14046b8fe2f..fb98582b14f7b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -66,6 +66,7 @@ import com.starrocks.catalog.DataProperty; import com.starrocks.catalog.Database; import com.starrocks.catalog.DistributionInfo; +import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.HashDistributionInfo; import com.starrocks.catalog.HiveTable; import com.starrocks.catalog.Index; @@ -226,6 +227,7 @@ import com.starrocks.sql.common.PListCell; import com.starrocks.sql.common.SyncPartitionUtils; import com.starrocks.sql.optimizer.Utils; +import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import com.starrocks.sql.optimizer.statistics.IDictManager; import com.starrocks.sql.util.EitherOr; import com.starrocks.system.Backend; @@ -2801,7 +2803,9 @@ public void createMaterializedView(CreateMaterializedViewStatement stmt) } // create partition info - PartitionInfo partitionInfo = buildPartitionInfo(stmt); + // add generate columns into base schema + PartitionInfo partitionInfo = buildPartitionInfo(stmt, baseSchema); + // create distribution info DistributionDesc distributionDesc = stmt.getDistributionDesc(); Preconditions.checkNotNull(distributionDesc); @@ -3023,7 +3027,8 @@ private long getRandomStart(IntervalLiteral interval, long randomizeStart) throw return randomInterval > 0 ? ThreadLocalRandom.current().nextLong(randomInterval) : randomInterval; } - public static PartitionInfo buildPartitionInfo(CreateMaterializedViewStatement stmt) throws DdlException { + public static PartitionInfo buildPartitionInfo(CreateMaterializedViewStatement stmt, + List baseSchema) throws DdlException { List partitionByExprs = stmt.getPartitionByExprs(); PartitionType partitionType = stmt.getPartitionType(); List mvPartitionColumns = stmt.getPartitionColumns(); @@ -3034,13 +3039,30 @@ public static PartitionInfo buildPartitionInfo(CreateMaterializedViewStatement s } if (partitionType == PartitionType.LIST) { - for (Expr partitionByExpr : partitionByExprs) { - if (!(partitionByExpr instanceof SlotRef)) { - throw new DdlException("List partition only support partition by slot ref column:" + Map generatedPartitionCols = stmt.getGeneratedPartitionCols(); + List newPartitionColumns = new ArrayList<>(); + Preconditions.checkNotNull(baseSchema); + for (int i = 0; i < partitionByExprs.size(); i++) { + Expr partitionByExpr = partitionByExprs.get(i); + Column mvPartitionColumn = mvPartitionColumns.get(i); + if (!(partitionByExpr instanceof SlotRef || MvUtils.isFuncCallExpr(partitionByExpr, + FunctionSet.DATE_TRUNC))) { + throw new DdlException("List partition only support partition by slot ref column or date_trunc:" + partitionByExpr.toSql()); } + if (!(partitionByExpr instanceof SlotRef)) { + Column generatedCol = generatedPartitionCols.get(i); + if (generatedCol == null) { + throw new DdlException("Partition expression for list must be a generated column: " + + partitionByExpr.toSql()); + } + baseSchema.add(generatedCol); + newPartitionColumns.add(generatedCol); + } else { + newPartitionColumns.add(mvPartitionColumn); + } } - return new ListPartitionInfo(PartitionType.LIST, mvPartitionColumns); + return new ListPartitionInfo(PartitionType.LIST, newPartitionColumns); } else { if (partitionByExprs.size() > 1) { throw new DdlException("Only support one partition column for range partition"); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java index 8eb593869a28a..b30b38a939ab8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java @@ -317,7 +317,7 @@ private void analyzeAnalyzeTypeDesc(ConnectContext session, StatementBase statem for (String partName : partitionNames) { List values = toPartitionValues(partName); PartitionKey partitionKey = createPartitionKey(values, analyzeTable.getPartitionColumns(), - analyzeTable.getType()); + analyzeTable); keys.add(partitionKey); } } catch (AnalysisException e) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java index 6281313a50b4f..bba2718643463 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java @@ -26,15 +26,20 @@ import com.starrocks.alter.AlterJobMgr; import com.starrocks.analysis.Expr; import com.starrocks.analysis.FunctionCallExpr; +import com.starrocks.analysis.IntLiteral; import com.starrocks.analysis.SlotDescriptor; import com.starrocks.analysis.SlotId; import com.starrocks.analysis.SlotRef; +import com.starrocks.analysis.StringLiteral; import com.starrocks.analysis.TableName; +import com.starrocks.analysis.TimestampArithmeticExpr; +import com.starrocks.analysis.TypeDef; import com.starrocks.catalog.AggregateType; import com.starrocks.catalog.BaseTableInfo; import com.starrocks.catalog.Column; import com.starrocks.catalog.ColumnId; import com.starrocks.catalog.Database; +import com.starrocks.catalog.Function; import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.HiveMetaStoreTable; import com.starrocks.catalog.IcebergTable; @@ -50,6 +55,7 @@ import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.RangePartitionInfo; +import com.starrocks.catalog.ScalarType; import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.common.DdlException; @@ -59,6 +65,8 @@ import com.starrocks.common.Pair; import com.starrocks.common.util.DateUtils; import com.starrocks.common.util.PropertyAnalyzer; +import com.starrocks.common.util.TimeUtils; +import com.starrocks.connector.iceberg.IcebergPartitionTransform; import com.starrocks.mv.analyzer.MVPartitionSlotRefResolver; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; @@ -67,6 +75,7 @@ import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.CancelRefreshMaterializedViewStmt; import com.starrocks.sql.ast.ColWithComment; +import com.starrocks.sql.ast.ColumnDef; import com.starrocks.sql.ast.CreateMaterializedViewStatement; import com.starrocks.sql.ast.DistributionDesc; import com.starrocks.sql.ast.DropMaterializedViewStmt; @@ -93,18 +102,23 @@ import com.starrocks.sql.optimizer.transformer.LogicalPlan; import com.starrocks.sql.optimizer.transformer.OptExprBuilder; import com.starrocks.sql.optimizer.transformer.RelationTransformer; +import com.starrocks.sql.parser.ParsingException; import com.starrocks.sql.plan.ExecPlan; import com.starrocks.sql.plan.PlanFragmentBuilder; import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.collections4.CollectionUtils; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.types.Types; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.Strings; import org.jetbrains.annotations.NotNull; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; @@ -353,7 +367,9 @@ public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement // check window function can be used in partitioned mv checkWindowFunctions(statement, columnExprMap); // determine mv partition 's type: list or range - determinePartitionType(statement, aliasTableMap); + checkMVPartitionInfoType(statement, aliasTableMap); + // deduce generate partition infos for list partition expressions + checkMVGeneratedPartitionColumns(statement, aliasTableMap); } // check and analyze distribution checkDistribution(statement, aliasTableMap); @@ -816,7 +832,7 @@ private void checkPartitionColumnWithBaseTable(CreateMaterializedViewStatement s } else if (table.isHiveTable() || table.isHudiTable() || table.isOdpsTable()) { checkPartitionColumnWithBaseHMSTable(slotRef, (HiveMetaStoreTable) table); } else if (table.isIcebergTable()) { - checkPartitionColumnWithBaseIcebergTable(slotRef, (IcebergTable) table); + checkPartitionColumnWithBaseIcebergTable(expr, slotRef, (IcebergTable) table); } else if (table.isJDBCTable()) { checkPartitionColumnWithBaseJDBCTable(slotRef, (JDBCTable) table); } else if (table.isPaimonTable()) { @@ -829,8 +845,8 @@ private void checkPartitionColumnWithBaseTable(CreateMaterializedViewStatement s } } - private void determinePartitionType(CreateMaterializedViewStatement statement, - Map tableNameTableMap) { + private void checkMVPartitionInfoType(CreateMaterializedViewStatement statement, + Map tableNameTableMap) { List mvPartitionByExprs = statement.getPartitionByExprs(); if (CollectionUtils.isEmpty(mvPartitionByExprs)) { statement.setPartitionType(PartitionType.UNPARTITIONED); @@ -915,6 +931,35 @@ private void determinePartitionType(CreateMaterializedViewStatement statement, } } + private void checkMVGeneratedPartitionColumns(CreateMaterializedViewStatement statement, + Map tableNameTableMap) { + PartitionType partitionType = statement.getPartitionType(); + if (partitionType != PartitionType.LIST) { + return; + } + // For list partition, deduce generated partition columns if its partition expression is a function call. + int placeholder = 0; + Map generatedPartitionColumns = statement.getGeneratedPartitionCols(); + TableName mvTableName = statement.getTableName(); + List mvColumns = statement.getMvColumnItems(); + List partitionRefTableExprs = statement.getPartitionRefTableExpr(); + for (int i = 0; i < partitionRefTableExprs.size(); i++) { + Expr expr = partitionRefTableExprs.get(i); + if (expr instanceof SlotRef) { + continue; + } + FunctionCallExpr partitionByExpr = (FunctionCallExpr) expr; + FunctionCallExpr cloned = (FunctionCallExpr) partitionByExpr.clone(); + Column generatedCol = getGeneratedPartitionColumn(mvColumns, mvTableName, + cloned, tableNameTableMap, placeholder); + if (generatedCol == null) { + throw new SemanticException("Create generated partition expression column failed:", + partitionByExpr.toSql()); + } + generatedPartitionColumns.put(i, generatedCol); + } + } + private void checkRangePartitionColumnLimit(List partitionByExprs) { if (partitionByExprs.size() > 1) { throw new SemanticException("Materialized view with range partition type " + @@ -1004,7 +1049,9 @@ private void checkWindowFunctions(CreateMaterializedViewStatement statement, Map MVPartitionSlotRefResolver.checkWindowFunction(statement, refTablePartitionExprs); } - private void checkPartitionColumnWithBaseIcebergTable(SlotRef slotRef, IcebergTable table) { + private void checkPartitionColumnWithBaseIcebergTable(Expr partitionByExpr, + SlotRef slotRef, + IcebergTable table) { org.apache.iceberg.Table icebergTable = table.getNativeTable(); PartitionSpec partitionSpec = icebergTable.spec(); if (partitionSpec.isUnpartitioned()) { @@ -1017,15 +1064,36 @@ private void checkPartitionColumnWithBaseIcebergTable(SlotRef slotRef, IcebergTa } boolean found = false; for (PartitionField partitionField : partitionSpec.fields()) { - String transformName = partitionField.transform().toString(); + IcebergPartitionTransform transform = + IcebergPartitionTransform.fromString(partitionField.transform().toString()); String partitionColumnName = icebergTable.schema().findColumnName(partitionField.sourceId()); if (partitionColumnName.equalsIgnoreCase(slotRef.getColumnName())) { checkPartitionColumnType(table.getColumn(partitionColumnName)); - if (transformName.startsWith("bucket") || transformName.startsWith("truncate")) { - throw new SemanticException("Do not support create materialized view when " + - "base iceberg table partition transform has bucket or truncate"); - } found = true; + switch (transform) { + case YEAR: + case MONTH: + case DAY: + case HOUR: + if (!isDateTruncWithUnit(partitionByExpr, transform.name())) { + throw new SemanticException("Materialized view partition expr %s " + + "must be the same with base table partition transform %s, please use date_trunc" + + "(, ) instead.", partitionByExpr.toSql(), + transform.name()); + } + break; + case IDENTITY: + if (!(partitionByExpr instanceof SlotRef) && !MvUtils.isStr2Date(partitionByExpr) && + !MvUtils.isFuncCallExpr(partitionByExpr, FunctionSet.DATE_TRUNC)) { + throw new SemanticException("Materialized view partition expr %s: " + + "only support ref partition column for transform %s, please use " + + " instead.", partitionByExpr.toSql(), transform.name()); + } + break; + default: + throw new SemanticException("Do not support create materialized view when " + + "base iceberg table partition transform is: " + transform.name()); + } break; } } @@ -1036,6 +1104,18 @@ private void checkPartitionColumnWithBaseIcebergTable(SlotRef slotRef, IcebergTa } } + private boolean isDateTruncWithUnit(Expr partitionExpr, String timeUnit) { + if (MvUtils.isFuncCallExpr(partitionExpr, FunctionSet.DATE_TRUNC)) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) partitionExpr; + if (!(functionCallExpr.getChild(0) instanceof StringLiteral)) { + return false; + } + StringLiteral stringLiteral = (StringLiteral) functionCallExpr.getChild(0); + return stringLiteral.getStringValue().equalsIgnoreCase(timeUnit); + } + return false; + } + @VisibleForTesting public void checkPartitionColumnWithBasePaimonTable(SlotRef slotRef, PaimonTable table) { if (table.isUnPartitioned()) { @@ -1256,6 +1336,19 @@ public Void visitRefreshMaterializedViewStatement(RefreshMaterializedViewStateme if (CollectionUtils.isEmpty(listCells)) { throw new SemanticException("Refresh list partition values is empty"); } + // refresh partition value's size should be equal to partition column's size + List partitionCols = mv.getPartitionColumns(); + for (PListCell cell : listCells) { + for (List items : cell.getPartitionItems()) { + if (items.size() != partitionCols.size()) { + throw new SemanticException(String.format("Partition column size %s is not match with " + + "input partition value's size %s: please use `REFRESH MATERIALIZED VIEW " + + "(('col1', 'col2'))` if the mv contains multi partition columns; otherwise use " + + "`REFRESH MATERIALIZED VIEW ('v1', " + + "'v2')`", partitionCols.size(), items.size()), mvName.getPos()); + } + } + } } return null; } @@ -1332,4 +1425,161 @@ public Void visitCancelRefreshMaterializedViewStatement(CancelRefreshMaterialize slotRef.setType(mvPartitionColumn.getType()); return mvPartitionColumn; } + + public static Column getGeneratedPartitionColumn(List mvColumns, + TableName mvTableName, + FunctionCallExpr partitionByExpr, + Map tableNameTableMap, + int placeHolderSlotId) { + List slotRefs = partitionByExpr.collectAllSlotRefs(); + if (slotRefs.size() != 1) { + throw new SemanticException("Partition expr only can ref one slot refs:", + partitionByExpr.toSql()); + } + SlotRef slotRef = slotRefs.get(0); + TableName refTableName = slotRef.getTblNameWithoutAnalyzed(); + Table refBaseTable = tableNameTableMap.get(refTableName); + if (refBaseTable == null) { + throw new SemanticException("Materialized view partition expression %s could only ref to base table", + slotRef.toSql()); + } + if (!(refBaseTable instanceof IcebergTable)) { + throw new SemanticException("Partition expression with function is not supported yet:" + + partitionByExpr.toSqlImpl()); + } + + // why resolve slot ref to mv columns? + // generated column should refer mv's defined column, but partitionByExpr is ref to base table's column, + // so resolve slot ref to mv columns. + resolveRefToMVColumns(mvColumns, slotRef, mvTableName); + + // change timezone, date_trunc('day', dt) -> date_trunc('day', date_sub(dt, interval 8 hour)) + IcebergTable icebergTable = (IcebergTable) refBaseTable; + int zoneHourOffset = getIcebergPartitionColumnTimeZoneOffset(icebergTable, slotRef); + Expr adjustedPartitionByExpr = partitionByExpr; + if (MvUtils.isFuncCallExpr(partitionByExpr, FunctionSet.DATE_TRUNC)) { + adjustedPartitionByExpr = getAdjustPartitionExpr(partitionByExpr, slotRef, zoneHourOffset); + } + ExpressionAnalyzer.analyzeExpression(adjustedPartitionByExpr, new AnalyzeState(), new Scope(RelationId.anonymous(), + new RelationFields(mvColumns.stream().map(col -> new Field(col.getName(), + col.getType(), mvTableName, null)).collect(Collectors.toList()))), + new ConnectContext()); + String columnName = FeConstants.GENERATED_PARTITION_COLUMN_PREFIX + placeHolderSlotId; + Type type = adjustedPartitionByExpr.getType(); + if (type.isScalarType()) { + ScalarType scalarType = (ScalarType) type; + if (scalarType.isWildcardChar()) { + type = ScalarType.createCharType(ScalarType.getOlapMaxVarcharLength()); + } else if (scalarType.isWildcardVarchar()) { + type = ScalarType.createVarcharType(ScalarType.getOlapMaxVarcharLength()); + } + } + TypeDef typeDef = new TypeDef(type); + try { + typeDef.analyze(); + } catch (Exception e) { + throw new ParsingException("Generate partition column " + columnName + + " for multi expression partition error: " + e.getMessage()); + } + ColumnDef generatedPartitionColumn = new ColumnDef( + columnName, typeDef, null, false, null, null, true, + ColumnDef.DefaultValueDef.NOT_SET, null, adjustedPartitionByExpr, ""); + return generatedPartitionColumn.toColumn(null); + } + + private static int getIcebergPartitionColumnTimeZoneOffset(IcebergTable icebergTable, + SlotRef slotRef) { + List refPartitionCols = icebergTable.getPartitionColumns(); + Optional refPartitionColOpt = refPartitionCols.stream() + .filter(col -> col.getName().equals(slotRef.getColumnName())) + .findFirst(); + if (refPartitionColOpt.isEmpty()) { + throw new SemanticException("Materialized view partition column in partition exp " + + "must be base table partition column"); + } + PartitionField partitionField = getIcebergPartitionField(icebergTable, refPartitionColOpt.get()); + if (partitionField.transform().dedupName().equalsIgnoreCase("time")) { + org.apache.iceberg.Schema icebegSchema = icebergTable.getNativeTable().schema(); + if (icebegSchema.findType(partitionField.sourceId()).equals(Types.TimestampType.withZone())) { + ZoneId zoneId = TimeUtils.getTimeZone().toZoneId(); + return getZoneHourOffset(ZoneOffset.UTC, zoneId); + } else { + return 0; + } + } + return 0; + } + + private static int getZoneHourOffset(ZoneId zoneId1, ZoneId zoneId2) { + ZonedDateTime nowInZone1 = ZonedDateTime.now(zoneId1); + ZonedDateTime nowInZone2 = ZonedDateTime.now(zoneId2); + ZoneOffset offset1 = nowInZone1.getOffset(); + ZoneOffset offset2 = nowInZone2.getOffset(); + return (offset2.getTotalSeconds() - offset1.getTotalSeconds()) / 3600; + } + + private static PartitionField getIcebergPartitionField(IcebergTable icebergTable, + Column refPartitionCol) { + return icebergTable.getPartitionField(refPartitionCol.getName()); + } + + private static Expr getAdjustPartitionExpr(Expr partitionByExpr, + SlotRef slotRef, + int zoneHourOffset) { + if (zoneHourOffset == 0) { + return partitionByExpr; + } + // date_trunc('day', dt) -> date_trunc('day', date_sub(dt, interval 8 hour)) + { + IntLiteral interval = new IntLiteral(zoneHourOffset, Type.INT); + Type[] argTypes = {slotRef.getType(), interval.getType()}; + Function dateSubFn = Expr.getBuiltinFunction(FunctionSet.DATE_SUB, + argTypes, Function.CompareMode.IS_IDENTICAL); + Preconditions.checkNotNull(dateSubFn, "date_sub function is not found"); + TimestampArithmeticExpr newChild = new TimestampArithmeticExpr(FunctionSet.DATE_SUB, slotRef, + interval, "HOUR"); + newChild.setFn(dateSubFn); + newChild.setType(slotRef.getType()); + partitionByExpr.setChild(1, newChild); + } + + // date_trunc('day', date_sub(dt, interval 8 hour)) -> date_add(date_trunc('day', date_sub(dt, interval 8 hour)), 8) + { + IntLiteral interval = new IntLiteral(zoneHourOffset, Type.INT); + Type[] argTypes = {slotRef.getType(), interval.getType()}; + Function dateAddFn = Expr.getBuiltinFunction(FunctionSet.DATE_ADD, + argTypes, Function.CompareMode.IS_IDENTICAL); + Preconditions.checkNotNull(dateAddFn, "date_add function is not found"); + TimestampArithmeticExpr dateAddFunc = new TimestampArithmeticExpr(FunctionSet.DATE_ADD, partitionByExpr, + interval, "HOUR"); + dateAddFunc.setFn(dateAddFn); + dateAddFunc.setType(slotRef.getType()); + + return dateAddFunc; + } + } + + private static void resolveRefToMVColumns(List columns, SlotRef slotRef, TableName mvTableName) { + Column mvPartitionColumn = null; + int columnId = 0; + for (Column column : columns) { + if (slotRef.getColumnName().equalsIgnoreCase(column.getName())) { + mvPartitionColumn = column; + break; + } + columnId++; + } + if (mvPartitionColumn == null) { + throw new SemanticException("Materialized view partition exp column:" + + slotRef.getColumnName() + " is not found in query statement"); + } + SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(columnId), slotRef.getColumnName(), + mvPartitionColumn.getType(), mvPartitionColumn.isAllowNull()); + slotRef.setDesc(slotDescriptor); + slotRef.setType(mvPartitionColumn.getType()); + slotRef.setNullable(mvPartitionColumn.isAllowNull()); + slotRef.setType(mvPartitionColumn.getType()); + slotRef.setColumnName(mvPartitionColumn.getName()); + slotRef.setTblName(mvTableName); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java index 9d158edb5c4cf..c8167bc2d559b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java @@ -16,6 +16,7 @@ package com.starrocks.sql.ast; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.starrocks.analysis.Expr; import com.starrocks.analysis.TableName; import com.starrocks.catalog.BaseTableInfo; @@ -89,6 +90,8 @@ public class CreateMaterializedViewStatement extends DdlStmt { // queryOutputIndexes : 1, 0, 2 // which means 0th of query output column is in 1th mv's output columns, and 1th -> 0th, 2th -> 2th. private List queryOutputIndices = Lists.newArrayList(); + // Generated partition columns for mv's partition by expressions, partition expression index to generated column. + private Map generatedPartitionCols = Maps.newHashMap(); public CreateMaterializedViewStatement(TableName tableName, boolean ifNotExists, List colWithComments, @@ -300,6 +303,10 @@ public void setMaintenancePlan(ExecPlan maintenancePlan, ColumnRefFactory column this.columnRefFactory = columnRefFactory; } + public Map getGeneratedPartitionCols() { + return generatedPartitionCols; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitCreateMaterializedViewStatement(this, context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/PListCell.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/PListCell.java index 7d908f01ea1b4..9b83381774d67 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/PListCell.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/PListCell.java @@ -14,10 +14,18 @@ package com.starrocks.sql.common; +import com.google.api.client.util.Lists; import com.google.api.client.util.Sets; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; import com.google.gson.annotations.SerializedName; +import com.starrocks.analysis.LiteralExpr; +import com.starrocks.catalog.Column; +import com.starrocks.catalog.PartitionKey; +import com.starrocks.catalog.PrimitiveType; +import com.starrocks.catalog.Type; +import com.starrocks.common.AnalysisException; import com.starrocks.connector.PartitionUtil; import com.starrocks.connector.hive.HiveMetaClient; import com.starrocks.persist.gson.GsonUtils; @@ -30,6 +38,7 @@ import java.util.Set; import java.util.stream.Collectors; + /** * {@code PListCell} means a list partition's multiple values. * eg: partition p1 values in ((1, 'a'), (2, 'b')) is a partition items which contains multi values @@ -98,6 +107,22 @@ public Set toAtoms() { .collect(Collectors.toSet()); } + public List toPartitionKeys(List columns) throws AnalysisException { + List partitionKeys = Lists.newArrayList(); + List types = columns.stream() + .map(Column::getType).map(Type::getPrimitiveType).collect(Collectors.toList()); + for (List item : partitionItems) { + Preconditions.checkArgument(item.size() == columns.size(), + String.format("item size %s is not equal to columns size %s", item.size(), columns.size())); + List literalExprs = Lists.newArrayList(); + for (int i = 0; i < item.size(); i++) { + literalExprs.add(LiteralExpr.create(item.get(i), columns.get(i).getType())); + } + partitionKeys.add(new PartitionKey(literalExprs, types)); + } + return partitionKeys; + } + /** * Add a list of partition items as the partition values * @param items new partition items diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java index 11d2bc289054c..eb38b5b2914be 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java @@ -336,7 +336,7 @@ private static void initPartitionInfo(LogicalScanOperator operator, OptimizerCon List ids = new ArrayList<>(); for (String partName : partitionNames) { List values = toPartitionValues(partName); - PartitionKey partitionKey = createPartitionKey(values, partitionColumns, table.getType()); + PartitionKey partitionKey = createPartitionKey(values, partitionColumns, table); keys.add(partitionKey); ids.add(context.getNextUniquePartitionId()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index 84329f310d22a..eda6096485c5d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -991,7 +991,11 @@ public static Expr convertToInPredicate(Expr slotRef, List values) { return new BoolLiteral(true); } // to avoid duplicate values - return new InPredicate(slotRef, Lists.newArrayList(Sets.newHashSet(values)), false); + if (values.size() == 1) { + return new BinaryPredicate(BinaryType.EQ, slotRef, values.get(0)); + } else { + return new InPredicate(slotRef, Lists.newArrayList(Sets.newHashSet(values)), false); + } } /** @@ -1433,9 +1437,16 @@ public static Optional getStr2DateExpr(Expr partitionExpr) { return Optional.of(matches.get(0).cast()); } - public static boolean isStr2Date(Expr expr) { + public static boolean isFuncCallExpr(Expr expr, String expectFuncName) { + if (expr == null) { + return false; + } return expr instanceof FunctionCallExpr - && ((FunctionCallExpr) expr).getFnName().getFunction().equalsIgnoreCase(FunctionSet.STR2DATE); + && ((FunctionCallExpr) expr).getFnName().getFunction().equalsIgnoreCase(expectFuncName); + } + + public static boolean isStr2Date(Expr expr) { + return isFuncCallExpr(expr, FunctionSet.STR2DATE); } public static Map getPartitionProperties(MaterializedView materializedView) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java index d009cfe71261f..1cd994b2a2b18 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java @@ -25,6 +25,7 @@ import com.starrocks.catalog.MvUpdateInfo; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.Table; import com.starrocks.common.AnalysisException; @@ -212,21 +213,44 @@ private MVCompensation getMVCompensationOfTable(Table refBaseTable, if (mvBaseTableUpdateInfo == null) { return null; } - Map> refTablePartitionNameWithRanges = - mvBaseTableUpdateInfo.getPartitionNameWithRanges(); - List partitionKeys = Lists.newArrayList(); - try { - for (String partitionName : refTablePartitionNamesToRefresh) { - Preconditions.checkState(refTablePartitionNameWithRanges.containsKey(partitionName)); - Range partitionKeyRange = refTablePartitionNameWithRanges.get(partitionName); - partitionKeys.add(partitionKeyRange.lowerEndpoint()); + PartitionInfo partitionInfo = mvContext.getMv().getPartitionInfo(); + if (partitionInfo.isRangePartition()) { + Map> refTablePartitionNameWithRanges = + mvBaseTableUpdateInfo.getPartitionNameWithRanges(); + List partitionKeys = Lists.newArrayList(); + try { + for (String partitionName : refTablePartitionNamesToRefresh) { + Preconditions.checkState(refTablePartitionNameWithRanges.containsKey(partitionName)); + Range partitionKeyRange = refTablePartitionNameWithRanges.get(partitionName); + partitionKeys.add(partitionKeyRange.lowerEndpoint()); + } + } catch (Exception e) { + logMVRewrite("Failed to get partition keys for ref base table: {}", refBaseTable.getName(), + DebugUtil.getStackTrace(e)); + return MVCompensation.createUnkownState(sessionVariable); } - } catch (Exception e) { - logMVRewrite("Failed to get partition keys for ref base table: {}", refBaseTable.getName(), - DebugUtil.getStackTrace(e)); - return MVCompensation.createUnkownState(sessionVariable); + return ofExternalTableCompensation(refBaseTable, partitionKeys); + } else { + Preconditions.checkArgument(partitionInfo.isListPartition()); + Map refTablePartitionNameWithRanges = mvBaseTableUpdateInfo.getPartitionNameWithLists(); + List partitionKeys = Lists.newArrayList(); + try { + List partitionCols = refBaseTable.getPartitionColumns(); + for (String partitionName : refTablePartitionNamesToRefresh) { + Preconditions.checkState(refTablePartitionNameWithRanges.containsKey(partitionName)); + PListCell pCell = refTablePartitionNameWithRanges.get(partitionName); + // TODO: we are assuming PListCell's cells' order is by partition's columns order, we may introduce + // partition columns in PListCell. + List keys = pCell.toPartitionKeys(partitionCols); + partitionKeys.addAll(keys); + } + } catch (Exception e) { + logMVRewrite("Failed to get partition keys for ref base table: {}", refBaseTable.getName(), + DebugUtil.getStackTrace(e)); + return MVCompensation.createUnkownState(sessionVariable); + } + return ofExternalTableCompensation(refBaseTable, partitionKeys); } - return ofExternalTableCompensation(refBaseTable, partitionKeys); } else { return MVCompensation.createUnkownState(sessionVariable); } @@ -434,7 +458,9 @@ private List getMVCompensatePartitionsOfExternalWithoutPartitionPr partitionKeys.add(((PRangeCell) pCell).getRange().lowerEndpoint()); } else if (pCell instanceof PListCell) { List partitionColumns = refBaseTable.getPartitionColumns(); - partitionKeys.add(createPartitionKey(((PListCell) pCell).getPartitionItems().get(0), partitionColumns)); + PartitionKey partitionKey = createPartitionKey(((PListCell) pCell).getPartitionItems().get(0), + partitionColumns, refBaseTable); + partitionKeys.add(partitionKey); } } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java index fb02623307931..fab2b286af4de 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java @@ -270,7 +270,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext, Collections.reverse(execPlan.getFragments()); // Create a fake table sink here, replaced it after created the MV - PartitionInfo partitionInfo = LocalMetastore.buildPartitionInfo(createStmt); + PartitionInfo partitionInfo = LocalMetastore.buildPartitionInfo(createStmt, null); long mvId = GlobalStateMgr.getCurrentState().getNextId(); long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createStmt.getTableName().getDb()).getId(); MaterializedView view = GlobalStateMgr.getCurrentState().getMaterializedViewMgr() diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java index 64ad19a57c3a6..3c47417522363 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java @@ -4881,7 +4881,7 @@ public void testCreateMVWithMultiPartitionColumns1() throws Exception { "as select dt, province, sum(age) from t3 group by dt, province;"); Assert.fail(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("List partition only support partition by slot ref column")); + Assert.assertTrue(e.getMessage().contains("Partition expression with function is not supported yet")); } } starRocksAssert.dropTable("t3"); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java index 0e741af7d3189..175030b914fd7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java @@ -1199,15 +1199,14 @@ public void testRefreshListPartitionMV1() { { String sql = "REFRESH MATERIALIZED VIEW test_mv1 PARTITION (('20240101', 'beijing'), ('20240101', " + "'nanjing')) FORCE;"; - RefreshMaterializedViewStatement statement = - (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - Assert.assertTrue(statement.isForceRefresh()); - Assert.assertNull(statement.getPartitionRangeDesc()); - Set expect = ImmutableSet.of( - new PListCell(ImmutableList.of(ImmutableList.of("20240101", "beijing"))), - new PListCell(ImmutableList.of(ImmutableList.of("20240101", "nanjing"))) - ); - Assert.assertEquals(expect, statement.getPartitionListDesc()); + try { + RefreshMaterializedViewStatement statement = + (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Partition column size 1 is not match with input partition" + + " value's size 2")); + } } }); } diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java index ad923b7e9bddc..104f4fea460fa 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java @@ -25,12 +25,14 @@ import com.starrocks.qe.StmtExecutor; import com.starrocks.scheduler.persist.MVTaskRunExtraMessage; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.RefreshMaterializedViewStatement; import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.common.PListCell; import com.starrocks.sql.parser.SqlParser; import com.starrocks.sql.plan.ExecPlan; import com.starrocks.sql.plan.PlanTestBase; import com.starrocks.thrift.TExplainLevel; +import com.starrocks.utframe.UtFrameUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -44,6 +46,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static com.starrocks.sql.plan.PlanTestBase.cleanupEphemeralMVs; @@ -1310,4 +1313,31 @@ public void testCreateMVWithMultiPartitionColumns() { starRocksAssert.dropMaterializedView("mv1"); }); } + + @Test + public void testRefreshListPartitionMVWithMultiPartitionColumns() { + starRocksAssert.withTable(T3, () -> { + starRocksAssert.withMaterializedView("create materialized view test_mv1\n" + + "partition by (dt, province) \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '1')" + + "as select dt, province, sum(age) from t3 group by dt, province;", + (obj) -> { + { + String sql = "REFRESH MATERIALIZED VIEW test_mv1 PARTITION (('20240101', 'beijing'), ('20240101', " + + "'nanjing')) FORCE;"; + RefreshMaterializedViewStatement statement = + (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + Assert.assertTrue(statement.isForceRefresh()); + Assert.assertNull(statement.getPartitionRangeDesc()); + Set expect = ImmutableSet.of( + new PListCell(ImmutableList.of(ImmutableList.of("20240101", "beijing"))), + new PListCell(ImmutableList.of(ImmutableList.of("20240101", "nanjing"))) + ); + Assert.assertEquals(expect, statement.getPartitionListDesc()); + } + }); + }); + } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java index 21892ac17c976..f2de928fa9fcd 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java @@ -16,13 +16,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.starrocks.catalog.Database; import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.Partition; import com.starrocks.common.util.RuntimeProfile; import com.starrocks.connector.iceberg.MockIcebergMetadata; import com.starrocks.server.GlobalStateMgr; -import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.common.QueryDebugOptions; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.plan.ConnectorPlanTestBase; @@ -208,7 +208,7 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform1() throws Ex String mvName = "iceberg_year_mv1"; starRocksAssert.useDatabase("test") .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_year_mv1`\n" + - "PARTITION BY ts\n" + + "PARTITION BY date_trunc('year', ts)\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "REFRESH DEFERRED MANUAL\n" + "PROPERTIES (\n" + @@ -225,8 +225,7 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform1() throws Ex Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); - List partitionNames = ImmutableList.of("p20190101000000", "p20200101000000", "p20210101000000", - "p20220101000000", "p20230101000000"); + Set partitionNames = ImmutableSet.of("p2020_2021", "p2022_2023", "p2019_2020", "p2023_2024", "p2021_2022"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); MockIcebergMetadata mockIcebergMetadata = @@ -243,7 +242,6 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform1() throws Ex MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); - System.out.println(execPlan.getExplainString(StatementBase.ExplainLevel.NORMAL)); assertPlanContains(execPlan, "3: ts >= '2020-01-01 00:00:00', 3: ts < '2021-01-01 00:00:00'"); // test rewrite @@ -258,7 +256,7 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform2() throws Ex String mvName = "iceberg_month_mv1"; starRocksAssert.useDatabase("test") .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_month_mv1`\n" + - "PARTITION BY ts\n" + + "PARTITION BY date_trunc('month', ts)\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "REFRESH DEFERRED MANUAL\n" + "PROPERTIES (\n" + @@ -275,8 +273,8 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform2() throws Ex Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); - List partitionNames = ImmutableList.of("p20220101000000", "p20220201000000", "p20220301000000", - "p20220401000000", "p20220501000000"); + Set partitionNames = ImmutableSet.of("p202202_202203", "p202205_202206", "p202203_202204", + "p202201_202202", "p202204_202205"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_month`") @@ -290,7 +288,7 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform3() throws Ex String mvName = "iceberg_day_mv1"; starRocksAssert.useDatabase("test") .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_day_mv1`\n" + - "PARTITION BY ts\n" + + "PARTITION BY date_trunc('day', ts)\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "REFRESH DEFERRED MANUAL\n" + "PROPERTIES (\n" + @@ -307,8 +305,8 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform3() throws Ex Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); - List partitionNames = ImmutableList.of("p20220101000000", "p20220102000000", "p20220103000000", - "p20220104000000", "p20220105000000"); + Set partitionNames = ImmutableSet.of("p20220103_20220104", "p20220104_20220105", "p20220105_20220106", + "p20220101_20220102", "p20220102_20220103"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_day`") @@ -322,7 +320,7 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform4() throws Ex String mvName = "iceberg_hour_mv1"; starRocksAssert.useDatabase("test") .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_hour_mv1`\n" + - "PARTITION BY ts\n" + + "PARTITION BY date_trunc('hour', ts)\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "REFRESH DEFERRED MANUAL\n" + "PROPERTIES (\n" + @@ -339,8 +337,8 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform4() throws Ex Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); - List partitionNames = ImmutableList.of("p20220101000000", "p20220101010000", "p20220101020000", - "p20220101030000", "p20220101040000"); + Set partitionNames = ImmutableSet.of("p2022010104_2022010105", "p2022010102_2022010103", + "p2022010100_2022010101", "p2022010103_2022010104", "p2022010101_2022010102"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_hour`") @@ -376,6 +374,7 @@ public void testRefreshWithCachePartitionTraits() { } private void testCreateMVWithMultiPartitionColumns(String icebergTable, + String transform, String updatePartitionName, List expectedPartitionNames, String expectedExecPlan) throws Exception { @@ -384,10 +383,10 @@ private void testCreateMVWithMultiPartitionColumns(String icebergTable, String query = String.format("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.%s as a", icebergTable); String ddl = String.format("CREATE MATERIALIZED VIEW `%s`\n" + - "PARTITION BY (id, data, ts)\n" + + "PARTITION BY (id, data, date_trunc('%s', ts))\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "REFRESH DEFERRED MANUAL\n" + - "AS %s;", mvName, query); + "AS %s;", mvName, transform, query); starRocksAssert.useDatabase("test").withMaterializedView(ddl); Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); @@ -399,7 +398,6 @@ private void testCreateMVWithMultiPartitionColumns(String icebergTable, Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(expectedPartitionNames.size(), partitions.size()); List partitionNames = partitions.stream().map(Partition::getName).collect(Collectors.toList()); - System.out.println(partitionNames); Assert.assertTrue(partitionNames.stream().allMatch(expectedPartitionNames::contains)); // update partition @@ -439,42 +437,49 @@ private void testCreateMVWithMultiPartitionColumns(String icebergTable, @Test public void testCreatePartitionedMVWithMultiPartitionColumnsHour() throws Exception { - testCreateMVWithMultiPartitionColumns("t0_multi_hour", "id=1/data=a/ts_hour=2022-01-01-00", + testCreateMVWithMultiPartitionColumns("t0_multi_hour", "hour", + "id=1/data=a/ts_hour=2022-01-01-00", ImmutableList.of("p1_a_20220101000000", "p2_a_20220101010000"), - "PREDICATES: 1: id = 1, 2: data = 'a', hour(3: ts) = '2022-01-01 00:00:00'"); + "PREDICATES: 1: id = 1, 2: data = 'a', 3: ts >= '2022-01-01 00:00:00', " + + "3: ts < '2022-01-01 01:00:00'"); } @Test public void testCreatePartitionedMVWithMultiPartitionColumnsDay() throws Exception { - testCreateMVWithMultiPartitionColumns("t0_multi_day", "id=1/data=a/ts_day=2022-01-01", + testCreateMVWithMultiPartitionColumns("t0_multi_day", "day", + "id=1/data=a/ts_day=2022-01-01", ImmutableList.of("p1_a_20220101000000", "p2_a_20220102000000"), - "PREDICATES: 1: id = 1, 2: data = 'a', day(3: ts) = '2022-01-01 00:00:00'"); + "PREDICATES: 1: id = 1, 2: data = 'a', 3: ts >= '2022-01-01 00:00:00', " + + "3: ts < '2022-01-02 00:00:00'"); } @Test public void testCreatePartitionedMVWithMultiPartitionColumnsMonth() throws Exception { - testCreateMVWithMultiPartitionColumns("t0_multi_month", "id=1/data=a/ts_month=2022-01", + testCreateMVWithMultiPartitionColumns("t0_multi_month", "month", + "id=1/data=a/ts_month=2022-01", ImmutableList.of("p1_a_20220101000000", "p2_a_20220201000000"), - "PREDICATES: 1: id = 1, 2: data = 'a', month(3: ts) = '2022-01-01 00:00:00'"); + "PREDICATES: 1: id = 1, 2: data = 'a', 3: ts >= '2022-01-01 00:00:00', " + + "3: ts < '2022-02-01 00:00:00'"); } @Test public void testCreatePartitionedMVWithMultiPartitionColumnsYear() throws Exception { - testCreateMVWithMultiPartitionColumns("t0_multi_year", "id=2/data=a/ts_year=2024", - ImmutableList.of("p1_a_20240101000000", "p2_a_20240101000000"), - "PREDICATES: 1: id = 2, 2: data = 'a', year(3: ts) = '2024-01-01 00:00:00'"); + testCreateMVWithMultiPartitionColumns("t0_multi_year", "year", + "id=2/data=a/ts_year=2024", ImmutableList.of("p1_a_20240101000000", "p2_a_20240101000000"), + "PREDICATES: 1: id = 2, 2: data = 'a', 3: ts >= '2024-01-01 00:00:00', " + + "3: ts < '2025-01-01 00:00:00'"); } @Test public void testCreatePartitionedMVWithMultiPartitionColumnsBucket() { try { - testCreateMVWithMultiPartitionColumns("t0_multi_bucket", "id=1/data=a/ts_bucket=0", + testCreateMVWithMultiPartitionColumns("t0_multi_bucket", "bucket", + "id=1/data=a/ts_bucket=0", ImmutableList.of("p1_a_20240101000000", "p2_a_20240101000000"), "3: ts >= '2024-01-01 00:00:00', 3: ts < '2025-01-01 00:00:00'"); Assert.fail(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("Do not support create materialized view when base " + - "iceberg table partition transform has bucket or truncate.")); + Assert.assertTrue(e.getMessage().contains("Unsupported expr 'date_trunc('bucket', ts)' in PARTITION BY clause")); } } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java index 0a80b20108d4c..882abc4024698 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java @@ -238,8 +238,7 @@ public void testCreateIcebergTable1() throws Exception { Assert.fail(); } catch (Exception e) { Assert.assertTrue(e.getMessage(). - contains("Do not support create materialized view when base iceberg table partition transform " + - "has bucket or truncate.")); + contains("Do not support create materialized view when base iceberg table partition transform ")); } } @@ -265,7 +264,7 @@ public void testCreateIcebergTable3() { try { starRocksAssert.useDatabase("test") .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_bucket_mv1`\n" + - "PARTITION BY (id, data, ts)\n" + + "PARTITION BY (id, data, date_trunc('year', ts))\n" + "REFRESH DEFERRED MANUAL\n" + "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_multi_year` as a;"); Table mv = starRocksAssert.getTable("test", "iceberg_bucket_mv1"); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentRewriteWithOlapTableTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentRewriteWithOlapTableTest.java index cf9b100f48c0c..618312910e0ce 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentRewriteWithOlapTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentRewriteWithOlapTableTest.java @@ -33,6 +33,9 @@ public class MvTransparentRewriteWithOlapTableTest extends MvRewriteTestBase { private static MTable m1; private static MTable m2; private static MTable m3; + private static String t1; + private static String t2; + private static String t3; @BeforeClass public static void beforeClass() throws Exception { @@ -84,6 +87,49 @@ public static void beforeClass() throws Exception { "PARTITION `p3` VALUES LESS THAN ('9')" ) ); + + // table whose partitions have multiple values + t1 = "CREATE TABLE t1 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; + + // table whose partitions have only single values + t2 = "CREATE TABLE t2 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; + // table whose partitions have multi columns + t3 = "CREATE TABLE t3 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province, dt) (\n" + + " PARTITION p1 VALUES IN ((\"beijing\", \"2024-01-01\")),\n" + + " PARTITION p2 VALUES IN ((\"guangdong\", \"2024-01-01\")), \n" + + " PARTITION p3 VALUES IN ((\"beijing\", \"2024-01-02\")),\n" + + " PARTITION p4 VALUES IN ((\"guangdong\", \"2024-01-02\")) \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; } @Before @@ -869,4 +915,66 @@ public void testTransparentRewriteWithPartitionPrune1() { }); }); } + + @Test + public void testTransparentMVWithListPartitions1() { + starRocksAssert.withTable(t1, () -> { + String insertSql = "insert into t1 values(1, 1, '2021-12-01', 'beijing'), (1, 1, '2021-12-01', 'guangdong');"; + cluster.runSql("test", insertSql); + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW mv0 " + + " PARTITION BY (province) " + + " DISTRIBUTED BY HASH(province) " + + " REFRESH DEFERRED MANUAL " + + " PROPERTIES (\n" + + " 'transparent_mv_rewrite_mode' = 'true'" + + " ) " + + " AS select * from t1;", + () -> { + cluster.runSql("test", String.format("REFRESH MATERIALIZED VIEW mv0 PARTITION ('%s') with sync mode", + "beijing")); + MaterializedView mv1 = getMv("test", "mv0"); + Set mvNames = mv1.getPartitionNames(); + Assert.assertEquals("[p1, p2]", mvNames.toString()); + // transparent mv + { + String plan = getFragmentPlan("select * from mv0", "MV"); + System.out.println(plan); + } + }); + }); + } + + @Test + public void testTransparentMVWithListPartitions2() { + starRocksAssert.withTable(t3, () -> { + String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing')," + + "(1, 1, '2022-01-01', 'hangzhou');"; + cluster.runSql("test", insertSql); + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW mv0 " + + " PARTITION BY (province) " + + " DISTRIBUTED BY HASH(province) " + + " REFRESH DEFERRED MANUAL " + + " PROPERTIES (\n" + + " 'transparent_mv_rewrite_mode' = 'true'" + + " ) " + + " AS select * from t3;", + () -> { + { + String plan = getFragmentPlan("select * from mv0", "MV"); + System.out.println(plan); + } + + cluster.runSql("test", String.format("REFRESH MATERIALIZED VIEW mv0 PARTITION ('%s') with sync mode", + "beijing")); + MaterializedView mv1 = getMv("test", "mv0"); + Set mvNames = mv1.getPartitionNames(); + Assert.assertEquals("[p1, p2]", mvNames.toString()); + // transparent mv + { + } + }); + }); + + + } } diff --git a/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg1 b/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg1 new file mode 100644 index 0000000000000..130cc98b056c6 --- /dev/null +++ b/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg1 @@ -0,0 +1,459 @@ +-- name: test_mv_with_multi_partition_columns_iceberg1 +create database db_${uuid0}; +-- result: +-- !result +use db_${uuid0}; +-- result: +-- !result +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); +-- result: +-- !result +CREATE MATERIALIZED VIEW test_date +PARTITION BY (l_returnflag, l_linestatus, l_shipdate) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_date PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_date WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +-- result: +test_date +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_date; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_date +PARTITION BY (l_returnflag, l_linestatus, l_shipdate) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date; +-- result: +-- !result +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_date +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +-- result: +test_date +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_date PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_date WITH SYNC MODE; +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 2024-11-20 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 2024-11-21 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 2024-11-22 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_date; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_hours +PARTITION BY (l_returnflag, l_linestatus, date_trunc('hour', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_hours PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +-- result: +test_hours +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +REFRESH MATERIALIZED VIEW test_hours WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +-- result: +test_hours +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_hours; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_hours +PARTITION BY (l_returnflag, l_linestatus, date_trunc('hour', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours; +-- result: +-- !result +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +-- result: +test_hours +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_hours PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_hours +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +-- result: +test_hours +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_hours WITH SYNC MODE; +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_hours; +-- result: +-- !result +drop database db_${uuid0} force; +-- result: +-- !result +drop catalog mv_iceberg_${uuid0}; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg2 b/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg2 new file mode 100644 index 0000000000000..bc26c9900699c --- /dev/null +++ b/test/sql/test_transparent_mv/R/test_mv_with_multi_partition_columns_iceberg2 @@ -0,0 +1,625 @@ +-- name: test_mv_with_multi_partition_columns_iceberg2 +create database db_${uuid0}; +-- result: +-- !result +use db_${uuid0}; +-- result: +-- !result +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); +-- result: +-- !result +CREATE MATERIALIZED VIEW test_days +PARTITION BY (l_returnflag, l_linestatus, date_trunc('day', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_days PARTITION (('R', 'F', '2024-11-13')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_days WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +-- result: +test_days +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_days; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_days +PARTITION BY (l_returnflag, l_linestatus, date_trunc('day', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days; +-- result: +-- !result +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_days +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +-- result: +test_days +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_days PARTITION (('R', 'F', '2024-11-13')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_days WITH SYNC MODE; +SELECT * FROM test_days order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_days order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 00:00:00 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 00:00:00 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 00:00:00 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_days; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_months +PARTITION BY (l_returnflag, l_linestatus, date_trunc('month', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_months PARTITION (('R', 'F', '2024-11')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_months WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +-- result: +test_months +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_months; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_months +PARTITION BY (l_returnflag, l_linestatus, date_trunc('month', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months; +-- result: +-- !result +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_months +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +-- result: +test_months +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_months PARTITION (('R', 'F', '2024-11')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_months WITH SYNC MODE; +SELECT * FROM test_months order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_months order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_months; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_years +PARTITION BY (l_returnflag, l_linestatus, date_trunc('year', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_years PARTITION (('R', 'F', '2024')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +REFRESH MATERIALIZED VIEW test_years WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +-- result: +test_years +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_years; +-- result: +-- !result +CREATE MATERIALIZED VIEW test_years +PARTITION BY (l_returnflag, l_linestatus, date_trunc('year', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years; +-- result: +-- !result +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: +test_years +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +-- result: +test_years +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_years PARTITION (('R', 'F', '2024')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +-- result: + +-- !result +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +-- result: + +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +-- !result +REFRESH MATERIALIZED VIEW test_years WITH SYNC MODE; +SELECT * FROM test_years order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_years order by l_orderkey; +-- result: +1 1001 5001 1 10.00 1000.00 0.05 0.08 N O 2024-11-12 2024-11-15 00:00:00 2024-11-20 00:00:00 DELIVER IN PERSON AIR Quick delivery required +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +-- !result +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +-- result: +2 1002 5002 2 20.00 2000.00 0.10 0.15 R F 2024-11-13 2024-11-16 00:00:00 2024-11-21 00:00:00 TAKE BACK RETURN RAIL Handle with care +3 1003 5003 3 30.00 3000.00 0.15 0.20 A P 2024-11-14 2024-11-17 00:00:00 2024-11-22 00:00:00 NONE SHIP Fragile item +-- !result +DROP MATERIALIZED VIEW test_years; +-- result: +-- !result +drop database db_${uuid0} force; +-- result: +-- !result +drop catalog mv_iceberg_${uuid0}; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg1 b/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg1 new file mode 100644 index 0000000000000..90348464c662e --- /dev/null +++ b/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg1 @@ -0,0 +1,292 @@ +-- name: test_mv_with_multi_partition_columns_iceberg1 + +-- !!!! SCIPRTS: prepare iceberg table before tests !!!! +-- CREATE DATABASE sql_test_db; +-- use sql_test_db; +-- CREATE TABLES IN ICEBERG + SPARK +-- CREATE TABLE t1 ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate STRING, +-- l_commitdate STRING, +-- l_receiptdate STRING, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus); + +-- INSERT INTO t1 VALUES( 1, 1001, 5001, 1, 10.00, 1000.00, 0.05, 0.08, 'N', 'O', '2024-11-12', '2024-11-15', '2024-11-20', 'DELIVER IN PERSON', 'AIR', 'Quick delivery required'); +-- INSERT INTO t1 VALUES (2, 1002, 5002, 2, 20.00, 2000.00, 0.10, 0.15, 'R', 'F', '2024-11-13', '2024-11-16', '2024-11-21', 'TAKE BACK RETURN', 'RAIL', 'Handle with care'); +-- INSERT INTO t1 VALUES (3, 1003, 5003, 3, 30.00, 3000.00, 0.15, 0.20, 'A', 'P', '2024-11-14', '2024-11-17', '2024-11-22', 'NONE', 'SHIP', 'Fragile item'); + +-- CREATE TABLE lineitem_date ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate DATE, +-- l_commitdate DATE, +-- l_receiptdate DATE, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus, l_shipdate); +-- INSERT INTO lineitem_date SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, to_date(l_shipdate), to_date(l_commitdate), to_date(l_receiptdate), l_shipinstruct, l_shipmode, l_comment FROM t1; + +-- CREATE TABLE lineitem_hours ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate TIMESTAMP, +-- l_commitdate TIMESTAMP, +-- l_receiptdate TIMESTAMP, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus, hours(l_shipdate)); +-- INSERT INTO lineitem_hours SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, to_date(l_shipdate), to_date(l_commitdate), to_date(l_receiptdate), l_shipinstruct, l_shipmode, l_comment FROM t1; + + +-- CREATE TABLE lineitem_days ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate TIMESTAMP, +-- l_commitdate TIMESTAMP, +-- l_receiptdate TIMESTAMP, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus, days(l_shipdate)); +-- INSERT INTO lineitem_days SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, to_date(l_shipdate), to_date(l_commitdate), to_date(l_receiptdate), l_shipinstruct, l_shipmode, l_comment FROM t1; + +-- CREATE TABLE lineitem_months ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate DATE, +-- l_commitdate TIMESTAMP, +-- l_receiptdate TIMESTAMP, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus, months(l_shipdate)); +-- INSERT INTO lineitem_months SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, to_date(l_shipdate), to_date(l_commitdate), to_date(l_receiptdate), l_shipinstruct, l_shipmode, l_comment FROM t1; + + +-- CREATE TABLE lineitem_years ( +-- l_orderkey BIGINT, +-- l_partkey INT, +-- l_suppkey INT, +-- l_linenumber INT, +-- l_quantity DECIMAL(15, 2), +-- l_extendedprice DECIMAL(15, 2), +-- l_discount DECIMAL(15, 2), +-- l_tax DECIMAL(15, 2), +-- l_returnflag VARCHAR(1), +-- l_linestatus VARCHAR(1), +-- l_shipdate DATE, +-- l_commitdate TIMESTAMP, +-- l_receiptdate TIMESTAMP, +-- l_shipinstruct VARCHAR(25), +-- l_shipmode VARCHAR(10), +-- l_comment VARCHAR(44) +-- ) USING ICEBERG +-- PARTITIONED BY (l_returnflag, l_linestatus, years(l_shipdate)); +-- INSERT INTO lineitem_years SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, to_date(l_shipdate), to_date(l_commitdate), to_date(l_receiptdate), l_shipinstruct, l_shipmode, l_comment FROM t1; + +-- create mv +create database db_${uuid0}; +use db_${uuid0}; + +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); + + +-------------------------------- IDENTITY -------------------------------- +CREATE MATERIALIZED VIEW test_date +PARTITION BY (l_returnflag, l_linestatus, l_shipdate) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date; +REFRESH MATERIALIZED VIEW test_date PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey; + +REFRESH MATERIALIZED VIEW test_date WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey; +DROP MATERIALIZED VIEW test_date; + +CREATE MATERIALIZED VIEW test_date +PARTITION BY (l_returnflag, l_linestatus, l_shipdate) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date; + +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_date PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_date order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_date order by l_orderkey;") +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_date WITH SYNC MODE; +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date order by l_orderkey; +SELECT * FROM test_date where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_date where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; + +DROP MATERIALIZED VIEW test_date; + + +-------------------------------- HOURS -------------------------------- +CREATE MATERIALIZED VIEW test_hours +PARTITION BY (l_returnflag, l_linestatus, date_trunc('hour', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours; +REFRESH MATERIALIZED VIEW test_hours PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey; + +REFRESH MATERIALIZED VIEW test_hours WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey; +DROP MATERIALIZED VIEW test_hours; + +CREATE MATERIALIZED VIEW test_hours +PARTITION BY (l_returnflag, l_linestatus, date_trunc('hour', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours; + +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_hours PARTITION (('R', 'F', '2024-11-13 00:00:00')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_hours order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_hours order by l_orderkey;") +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_hours WITH SYNC MODE; +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours order by l_orderkey; +SELECT * FROM test_hours where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_hours where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; + +DROP MATERIALIZED VIEW test_hours; + + +drop database db_${uuid0} force; +drop catalog mv_iceberg_${uuid0}; \ No newline at end of file diff --git a/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg2 b/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg2 new file mode 100644 index 0000000000000..02d69a3af813c --- /dev/null +++ b/test/sql/test_transparent_mv/T/test_mv_with_multi_partition_columns_iceberg2 @@ -0,0 +1,223 @@ +-- name: test_mv_with_multi_partition_columns_iceberg2 + +-- create mv +create database db_${uuid0}; +use db_${uuid0}; + +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); + +-------------------------------- DAYS -------------------------------- +CREATE MATERIALIZED VIEW test_days +PARTITION BY (l_returnflag, l_linestatus, date_trunc('day', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days; +REFRESH MATERIALIZED VIEW test_days PARTITION (('R', 'F', '2024-11-13')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey; + +REFRESH MATERIALIZED VIEW test_days WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey; +DROP MATERIALIZED VIEW test_days; + +CREATE MATERIALIZED VIEW test_days +PARTITION BY (l_returnflag, l_linestatus, date_trunc('day', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days; + +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_days PARTITION (('R', 'F', '2024-11-13')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_days order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_days order by l_orderkey;") +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_days WITH SYNC MODE; +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days order by l_orderkey; +SELECT * FROM test_days where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_days where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; + +DROP MATERIALIZED VIEW test_days; + +-------------------------------- MONTHS -------------------------------- +CREATE MATERIALIZED VIEW test_months +PARTITION BY (l_returnflag, l_linestatus, date_trunc('month', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months; +REFRESH MATERIALIZED VIEW test_months PARTITION (('R', 'F', '2024-11')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey; + +REFRESH MATERIALIZED VIEW test_months WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey; +DROP MATERIALIZED VIEW test_months; + +CREATE MATERIALIZED VIEW test_months +PARTITION BY (l_returnflag, l_linestatus, date_trunc('month', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months; + +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_months PARTITION (('R', 'F', '2024-11')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_months order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_months order by l_orderkey;") +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_months WITH SYNC MODE; +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months order by l_orderkey; +SELECT * FROM test_months where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_months where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; + +DROP MATERIALIZED VIEW test_months; + +-------------------------------- YEARS -------------------------------- +CREATE MATERIALIZED VIEW test_years +PARTITION BY (l_returnflag, l_linestatus, date_trunc('year', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ("replication_num" = "1") +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years; +REFRESH MATERIALIZED VIEW test_years PARTITION (('R', 'F', '2024')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey; + +REFRESH MATERIALIZED VIEW test_years WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey; +DROP MATERIALIZED VIEW test_years; + +CREATE MATERIALIZED VIEW test_years +PARTITION BY (l_returnflag, l_linestatus, date_trunc('year', l_shipdate)) +REFRESH DEFERRED MANUAL +PROPERTIES ( + "replication_num" = "1", + "transparent_mv_rewrite_mode" = "true" +) +AS + SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years; + +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_years PARTITION (('R', 'F', '2024')) WITH SYNC MODE; +function: print_hit_materialized_views("SELECT * FROM test_years order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey;") +function: print_hit_materialized_views("SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.lineitem_years order by l_orderkey;") +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; +REFRESH MATERIALIZED VIEW test_years WITH SYNC MODE; +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years order by l_orderkey; +SELECT * FROM test_years where l_returnflag = 'R' and l_linestatus = 'F' and l_shipdate = '2024-11-13 00:00:00' order by l_orderkey; +SELECT * FROM test_years where l_shipdate >= '2024-11-13 00:00:00' order by l_orderkey; + +DROP MATERIALIZED VIEW test_years; + +drop database db_${uuid0} force; +drop catalog mv_iceberg_${uuid0}; \ No newline at end of file