diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MvBaseTableUpdateInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MvBaseTableUpdateInfo.java index a45a6a8f2a250..db89f3bbb5f6f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MvBaseTableUpdateInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MvBaseTableUpdateInfo.java @@ -52,6 +52,10 @@ public Set getToRefreshPartitionNames() { return toRefreshPartitionNames; } + public Map getNameToPartKeys() { + return nameToPartKeys; + } + /** * Add partition names that base table needs to be refreshed * @param toRefreshPartitionNames the partition names that need to be refreshed diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/CatalogConnectorMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/CatalogConnectorMetadata.java index 5c4e34a037587..a3d8329fd5fcd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/CatalogConnectorMetadata.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/CatalogConnectorMetadata.java @@ -210,11 +210,6 @@ public Statistics getTableStatistics(OptimizerContext session, Table table, Map< return normal.getTableStatistics(session, table, columns, partitionKeys, predicate, limit, version); } - @Override - public List getPrunedPartitions(Table table, ScalarOperator predicate, long limit, TableVersionRange version) { - return normal.getPrunedPartitions(table, predicate, limit, version); - } - @Override public Set getDeleteFiles(IcebergTable table, Long snapshotId, ScalarOperator predicate, FileContent content) { return normal.getDeleteFiles(table, snapshotId, predicate, content); diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java index 0f203a47601f4..377463c3372f9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java @@ -218,11 +218,6 @@ default boolean prepareMetadata(MetaPreparationItem item, Tracers tracers, Conne return true; } - default List getPrunedPartitions(Table table, ScalarOperator predicate, - long limit, TableVersionRange version) { - throw new StarRocksConnectorException("This connector doesn't support pruning partitions"); - } - // return true if the connector has self info schema default boolean hasSelfInfoSchema() { return false; diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java index 4e9213ef0398f..1799b16173292 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java @@ -707,7 +707,6 @@ private void triggerIcebergPlanFilesIfNeeded(PredicateSearchKey key, IcebergTabl } } - @Override public List getPrunedPartitions(Table table, ScalarOperator predicate, long limit, TableVersionRange version) { IcebergTable icebergTable = (IcebergTable) table; String dbName = icebergTable.getRemoteDbName(); @@ -942,9 +941,9 @@ private Iterator buildFileScanTaskIterator(IcebergTable icebergTab scanContext.setLocalPlanningMaxSlotSize(catalogProperties.getLocalPlanningMaxSlotBytes()); TableScan scan = icebergCatalog.getTableScan(nativeTbl, scanContext) - .useSnapshot(snapshotId) - .metricsReporter(metricsReporter) - .planWith(jobPlanningExecutor); + .useSnapshot(snapshotId) + .metricsReporter(metricsReporter) + .planWith(jobPlanningExecutor); if (enableCollectColumnStats) { scan = scan.includeColumnStats(); diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/unified/UnifiedMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/unified/UnifiedMetadata.java index d462b00255260..ab881e25cd863 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/unified/UnifiedMetadata.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/unified/UnifiedMetadata.java @@ -198,12 +198,6 @@ public List getPartitions(Table table, List partitionName return metadata.getPartitions(table, partitionNames); } - @Override - public List getPrunedPartitions(Table table, ScalarOperator predicate, long limit, TableVersionRange version) { - ConnectorMetadata metadata = metadataOfTable(table); - return metadata.getPrunedPartitions(table, predicate, limit, version); - } - @Override public Statistics getTableStatistics(OptimizerContext session, Table table, Map columns, List partitionKeys, ScalarOperator predicate, long limit, diff --git a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java index 1ed8f876fe64e..af7ac2e83120c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java @@ -659,20 +659,6 @@ public List listPartitionNamesByValue(String catalogName, String dbName, return ImmutableList.copyOf(partitionNames.build()); } - public List getPrunedPartitions(String catalogName, Table table, ScalarOperator predicate, - long limit, TableVersionRange version) { - Optional connectorMetadata = getOptionalMetadata(catalogName); - if (connectorMetadata.isPresent()) { - try { - return connectorMetadata.get().getPrunedPartitions(table, predicate, limit, version); - } catch (Exception e) { - LOG.error("Failed to getPrunedPartitions on [{}.{}]", catalogName, table, e); - throw e; - } - } - return new ArrayList<>(); - } - public Statistics getTableStatisticsFromInternalStatistics(Table table, Map columns) { List requiredColumnRefs = new ArrayList<>(columns.keySet()); List columnNames = requiredColumnRefs.stream().map(col -> columns.get(col).getName()).collect( diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java index c181563761089..11d2bc289054c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptExternalPartitionPruner.java @@ -25,7 +25,6 @@ import com.starrocks.catalog.Column; import com.starrocks.catalog.DeltaLakeTable; import com.starrocks.catalog.HiveMetaStoreTable; -import com.starrocks.catalog.IcebergTable; import com.starrocks.catalog.PaimonTable; import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionKey; @@ -424,30 +423,6 @@ private static void computePartitionInfo(LogicalScanOperator operator, Optimizer scanOperatorPredicates.setSelectedPartitionIds(selectedPartitionIds); scanOperatorPredicates.getNoEvalPartitionConjuncts().addAll(partitionPruner.getNoEvalConjuncts()); - } else if (table instanceof IcebergTable) { - IcebergTable icebergTable = (IcebergTable) table; - if (operator.getTableVersionRange().end().isEmpty()) { - // TODO: for iceberg table, it cannot decide whether it's pruned or not when `selectedPartitionIds` - // is empty. It's expensive to set all partitions here. - return; - } - - // Use mutable map instead of immutable map so can be re-partition-prune, see ScanOperatorPredicates#clear(). - Map partitionKeyMap = Maps.newHashMap(); - if (table.isUnPartitioned()) { - partitionKeyMap.put(0L, new PartitionKey()); - } else { - String catalogName = icebergTable.getCatalogName(); - List partitionKeys = GlobalStateMgr.getCurrentState().getMetadataMgr() - .getPrunedPartitions(catalogName, icebergTable, operator.getPredicate(), - operator.getLimit(), operator.getTableVersionRange()); - for (PartitionKey partitionKey : partitionKeys) { - partitionKeyMap.put(context.getNextUniquePartitionId(), partitionKey); - } - } - - scanOperatorPredicates.getIdToPartitionKey().putAll(partitionKeyMap); - scanOperatorPredicates.setSelectedPartitionIds(partitionKeyMap.keySet()); } else if (table instanceof PaimonTable) { PaimonTable paimonTable = (PaimonTable) table; List fieldNames = operator.getColRefToColumnMetaMap().keySet().stream() diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java index a5c4056686157..72ea63a877a9a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java @@ -99,10 +99,7 @@ public class MvPartitionCompensator { private static final Logger LOG = LogManager.getLogger(MvPartitionCompensator.class); - /** - * External scan operators should be supported if it has been supported in - * {@link com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner} - */ + // supported external scan types for partition compensate public static final ImmutableSet SUPPORTED_PARTITION_COMPENSATE_EXTERNAL_SCAN_TYPES = ImmutableSet.builder() .add(OperatorType.LOGICAL_HIVE_SCAN) @@ -115,6 +112,15 @@ public class MvPartitionCompensator { .addAll(SUPPORTED_PARTITION_COMPENSATE_EXTERNAL_SCAN_TYPES) .build(); + /** + * External scan operators could use {@link com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner} + * to prune partitions, so we need to compensate partition predicates for them. + */ + public static final ImmutableSet SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES = + ImmutableSet.builder() + .add(OperatorType.LOGICAL_HIVE_SCAN) + .build(); + /** * Whether the table is supported to compensate extra partition predicates. * @param t: input table @@ -436,6 +442,10 @@ private static boolean supportCompensatePartitionPredicateForHiveScan(List compensatePartitionPredicateForExternalTables(MaterializationContext mvContext, LogicalScanOperator scanOperator) { + if (!SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(scanOperator.getOpType())) { + return Lists.newArrayList(); + } + ScanOperatorPredicates scanOperatorPredicates = null; try { scanOperatorPredicates = scanOperator.getScanOperatorPredicates(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java index 52fb08c5bda73..d009cfe71261f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/MVCompensationBuilder.java @@ -31,6 +31,9 @@ import com.starrocks.common.util.DebugUtil; import com.starrocks.connector.PartitionUtil; import com.starrocks.qe.SessionVariable; +import com.starrocks.sql.common.PCell; +import com.starrocks.sql.common.PListCell; +import com.starrocks.sql.common.PRangeCell; import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.operator.OperatorType; @@ -53,8 +56,10 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.starrocks.connector.PartitionUtil.createPartitionKey; import static com.starrocks.connector.PartitionUtil.generateMVPartitionName; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; +import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvPartitionCompensator.SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES; /** * MVCompensationBuilder is used to build a mv compensation for materialized view in mv rewrite. @@ -234,7 +239,7 @@ private MVCompensation getMVCompensationOfTable(Table refBaseTable, return getMVCompensationOfOlapTable(refBaseTable, partitionNamesToRefresh, (LogicalOlapScanOperator) scanOperator); } else if (MvPartitionCompensator.isTableSupportedPartitionCompensate(refBaseTable)) { - return getMVCompensationForExternal(partitionNamesToRefresh, scanOperator); + return getMVCompensationForExternal(refBaseTable, partitionNamesToRefresh, scanOperator); } else { SessionVariable sessionVariable = mvContext.getOptimizerContext().getSessionVariable(); return MVCompensation.createUnkownState(sessionVariable); @@ -295,38 +300,47 @@ private MVCompensation ofBaseTableCompensations(Map> } } - private MVCompensation getMVCompensationForExternal(Set refTablePartitionNamesToRefresh, + private MVCompensation getMVCompensationForExternal(Table refBaseTable, + Set refTablePartitionNamesToRefresh, LogicalScanOperator refScanOperator) { SessionVariable sessionVariable = mvContext.getOptimizerContext().getSessionVariable(); try { ScanOperatorPredicates scanOperatorPredicates = refScanOperator.getScanOperatorPredicates(); Collection selectPartitionIds = scanOperatorPredicates.getSelectedPartitionIds(); - if (Objects.isNull(selectPartitionIds) || selectPartitionIds.size() == 0) { - // see OptExternalPartitionPruner#computePartitionInfo: - // it's not the same meaning when selectPartitionIds is null and empty for hive and other tables - if (refScanOperator.getOpType() == OperatorType.LOGICAL_HIVE_SCAN) { + List selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys(); + // For scan operator which support prune partitions with OptExternalPartitionPruner, + // we could only compensate partitions which selected partitions need to refresh. + if (SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(refScanOperator.getOpType())) { + if (Objects.isNull(selectPartitionIds) || selectPartitionIds.isEmpty()) { + // see OptExternalPartitionPruner#computePartitionInfo: + // it's not the same meaning when selectPartitionIds is null and empty for hive and other tables + if (refScanOperator.getOpType() == OperatorType.LOGICAL_HIVE_SCAN) { + return MVCompensation.createNoCompensateState(sessionVariable); + } else { + return MVCompensation.createUnkownState(sessionVariable); + } + } + + if (selectPartitionKeys.stream() + .map(PartitionUtil::generateMVPartitionName) + .noneMatch(refTablePartitionNamesToRefresh::contains)) { return MVCompensation.createNoCompensateState(sessionVariable); - } else { - return MVCompensation.createUnkownState(sessionVariable); } } - List selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys(); - if (selectPartitionKeys.stream() - .map(PartitionUtil::generateMVPartitionName) - .noneMatch(x -> refTablePartitionNamesToRefresh.contains(x))) { - return MVCompensation.createNoCompensateState(sessionVariable); - } // if mv's to refresh partitions contains any of query's select partition ids, then rewrite with compensate. - List toRefreshRefTablePartitions = getMVCompensatePartitionsOfExternal( + List toRefreshRefTablePartitions = getMVCompensatePartitionsOfExternal(refBaseTable, refTablePartitionNamesToRefresh, refScanOperator); if (toRefreshRefTablePartitions == null) { return MVCompensation.createUnkownState(sessionVariable); } + Table table = refScanOperator.getTable(); - if (Sets.newHashSet(toRefreshRefTablePartitions).containsAll(selectPartitionKeys)) { - logMVRewrite(mvContext, "All external table {}'s selected partitions {} need to refresh, no rewrite", - table.getName(), selectPartitionIds); - return MVCompensation.createNoRewriteState(sessionVariable); + if (SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(refScanOperator.getOpType())) { + if (Sets.newHashSet(toRefreshRefTablePartitions).containsAll(selectPartitionKeys)) { + logMVRewrite(mvContext, "All external table {}'s selected partitions {} need to refresh, no rewrite", + table.getName(), selectPartitionIds); + return MVCompensation.createNoRewriteState(sessionVariable); + } } return ofExternalTableCompensation(table, toRefreshRefTablePartitions); } catch (AnalysisException e) { @@ -359,9 +373,24 @@ private List getMVCompensatePartitionsOfOlap(Set partitionNamesToR return refTableCompensatePartitionIds; } - private List getMVCompensatePartitionsOfExternal(Set refTablePartitionNamesToRefresh, + private List getMVCompensatePartitionsOfExternal(Table refBaseTable, + Set refTablePartitionNamesToRefresh, LogicalScanOperator refScanOperator) throws AnalysisException { + if (SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(refScanOperator.getOpType())) { + // For external table which support partition prune with OptExternalPartitionPruner, + // could use selectPartitionKeys to get the compensate partitions. + return getMVCompensatePartitionsOfExternalWithPartitionPruner(refTablePartitionNamesToRefresh, + refScanOperator); + } else { + return getMVCompensatePartitionsOfExternalWithoutPartitionPruner(refBaseTable, refTablePartitionNamesToRefresh); + } + } + + private List getMVCompensatePartitionsOfExternalWithPartitionPruner( + Set refTablePartitionNamesToRefresh, + LogicalScanOperator refScanOperator) { + List refTableCompensatePartitionKeys = Lists.newArrayList(); ScanOperatorPredicates scanOperatorPredicates = null; try { scanOperatorPredicates = refScanOperator.getScanOperatorPredicates(); @@ -371,7 +400,6 @@ private List getMVCompensatePartitionsOfExternal(Set refTa if (scanOperatorPredicates == null) { return null; } - List refTableCompensatePartitionKeys = Lists.newArrayList(); List selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys(); // different behavior for different external table types if (selectPartitionKeys.isEmpty() && refScanOperator.getOpType() != OperatorType.LOGICAL_HIVE_SCAN) { @@ -385,4 +413,35 @@ private List getMVCompensatePartitionsOfExternal(Set refTa } return refTableCompensatePartitionKeys; } + + private List getMVCompensatePartitionsOfExternalWithoutPartitionPruner( + Table refBaseTable, + Set refTablePartitionNamesToRefresh) { + MvBaseTableUpdateInfo baseTableUpdateInfo = mvUpdateInfo.getBaseTableUpdateInfos().get(refBaseTable); + if (baseTableUpdateInfo == null) { + return null; + } + + Map nameToPartitionKeys = baseTableUpdateInfo.getNameToPartKeys(); + List partitionKeys = Lists.newArrayList(); + try { + for (String partitionName : refTablePartitionNamesToRefresh) { + if (!nameToPartitionKeys.containsKey(partitionName)) { + return null; + } + PCell pCell = nameToPartitionKeys.get(partitionName); + if (pCell instanceof PRangeCell) { + partitionKeys.add(((PRangeCell) pCell).getRange().lowerEndpoint()); + } else if (pCell instanceof PListCell) { + List partitionColumns = refBaseTable.getPartitionColumns(); + partitionKeys.add(createPartitionKey(((PListCell) pCell).getPartitionItems().get(0), partitionColumns)); + } + } + } catch (Exception e) { + logMVRewrite("Failed to get partition keys for ref base table: {}", refBaseTable.getName(), + DebugUtil.getStackTrace(e)); + return null; + } + return partitionKeys; + } } diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/MockIcebergMetadata.java b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/MockIcebergMetadata.java index 10c88326bb6e9..4749de50a5175 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/MockIcebergMetadata.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/MockIcebergMetadata.java @@ -45,7 +45,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -494,12 +493,6 @@ public Statistics getTableStatistics(OptimizerContext session, com.starrocks.cat } } - @Override - public List getPrunedPartitions(com.starrocks.catalog.Table table, ScalarOperator predicate, - long limit, TableVersionRange version) { - return new ArrayList<>(); - } - public void addRowsToPartition(String dbName, String tableName, int rowCount, String partitionName) { IcebergTable icebergTable = MOCK_TABLE_MAP.get(dbName).get(tableName).icebergTable; Table nativeTable = icebergTable.getNativeTable(); diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/unified/UnifiedMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/unified/UnifiedMetadataTest.java index 57c1ce3abaf65..219f3d06aac49 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/unified/UnifiedMetadataTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/unified/UnifiedMetadataTest.java @@ -290,12 +290,6 @@ public void testRouteToIcebergConnector(@Mocked HiveTable hiveTable) throws DdlE times = 1; } - { - icebergMetadata.getPrunedPartitions(icebergTable, null, -1, TableVersionRange.empty()); - result = ImmutableList.of(); - times = 1; - } - { icebergMetadata.prepareMetadata((MetaPreparationItem) any, null, null); result = true; @@ -317,7 +311,6 @@ public void testRouteToIcebergConnector(@Mocked HiveTable hiveTable) throws DdlE unifiedMetadata.finishSink("test_db", "test_tbl", ImmutableList.of(), null); createTableStmt.setEngineName("iceberg"); assertTrue(unifiedMetadata.createTable(createTableStmt)); - Assert.assertTrue(unifiedMetadata.getPrunedPartitions(table, null, -1, TableVersionRange.empty()).isEmpty()); Assert.assertTrue(unifiedMetadata.prepareMetadata(new MetaPreparationItem(icebergTable, null, -1, TableVersionRange.empty()), null, null)); Assert.assertNotNull(unifiedMetadata.getSerializedMetaSpec("test_db", "test_tbl", -1, null, null)); diff --git a/fe/fe-core/src/test/java/com/starrocks/server/MetadataMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/server/MetadataMgrTest.java index 6d6383209e11f..ac70d2edcacb1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/MetadataMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/MetadataMgrTest.java @@ -24,7 +24,6 @@ import com.starrocks.connector.ConnectorMetadata; import com.starrocks.connector.ConnectorMgr; import com.starrocks.connector.MockedMetadataMgr; -import com.starrocks.connector.TableVersionRange; import com.starrocks.connector.exception.StarRocksConnectorException; import com.starrocks.connector.hive.HiveMetastoreApiConverter; import com.starrocks.connector.hive.MockedHiveMetadata; @@ -424,12 +423,6 @@ public void testDropDbIfExists() throws DdlException, MetaNotFoundException { metadataMgr.dropDb("hive_catalog", "hive_db", false); } - @Test(expected = StarRocksConnectorException.class) - public void testGetPrunedPartition() { - MetadataMgr metadataMgr = AnalyzeTestUtil.getConnectContext().getGlobalStateMgr().getMetadataMgr(); - metadataMgr.getPrunedPartitions("hive_catalog", null, null, -1, TableVersionRange.empty()); - } - @Test public void testGetMetadataTable() throws Exception { new MockUp() { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownMinMaxConjunctsRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownMinMaxConjunctsRuleTest.java deleted file mode 100644 index 24cb2bb838692..0000000000000 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownMinMaxConjunctsRuleTest.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -package com.starrocks.sql.optimizer.rule.transformation; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.starrocks.analysis.BinaryType; -import com.starrocks.catalog.Column; -import com.starrocks.catalog.IcebergTable; -import com.starrocks.catalog.Type; -import com.starrocks.connector.TableVersionRange; -import com.starrocks.connector.iceberg.TableTestBase; -import com.starrocks.sql.analyzer.AnalyzeTestUtil; -import com.starrocks.sql.optimizer.Memo; -import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; -import com.starrocks.sql.optimizer.base.ColumnRefFactory; -import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator; -import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator; -import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; -import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; -import com.starrocks.sql.optimizer.operator.scalar.PredicateOperator; -import com.starrocks.utframe.StarRocksAssert; -import com.starrocks.utframe.UtFrameUtils; -import mockit.Mocked; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.starrocks.catalog.Type.INT; -import static com.starrocks.catalog.Type.STRING; -import static com.starrocks.sql.analyzer.AnalyzeTestUtil.getStarRocksAssert; -import static org.junit.Assert.assertEquals; - -public class PushDownMinMaxConjunctsRuleTest extends TableTestBase { - - @BeforeClass - public static void beforeClass() throws Exception { - UtFrameUtils.createMinStarRocksCluster(); - AnalyzeTestUtil.init(); - StarRocksAssert starRocksAssert = getStarRocksAssert(); - String createIcebergCatalogStmt = "create external catalog iceberg_catalog properties (\"type\"=\"iceberg\", " + - "\"hive.metastore.uris\"=\"thrift://hms:9083\", \"iceberg.catalog.type\"=\"hive\")"; - starRocksAssert.withCatalog(createIcebergCatalogStmt); - } - - @Test - public void transformIceberg(@Mocked IcebergTable table) { - ExternalScanPartitionPruneRule rule0 = ExternalScanPartitionPruneRule.ICEBERG_SCAN; - - ColumnRefOperator colRef = new ColumnRefOperator(1, Type.INT, "id", true); - Column col = new Column("id", Type.INT, true); - PredicateOperator binaryPredicateOperator = new BinaryPredicateOperator(BinaryType.EQ, colRef, - ConstantOperator.createInt(1)); - - Map colRefToColumnMetaMap = new HashMap<>(); - Map columnMetaToColRefMap = new HashMap<>(); - colRefToColumnMetaMap.put(colRef, col); - columnMetaToColRefMap.put(col, colRef); - OptExpression scan = - new OptExpression(new LogicalIcebergScanOperator(table, colRefToColumnMetaMap, columnMetaToColRefMap, - -1, binaryPredicateOperator, TableVersionRange.empty())); - - assertEquals(0, ((LogicalIcebergScanOperator) scan.getOp()).getScanOperatorPredicates().getMinMaxConjuncts().size()); - - rule0.transform(scan, new OptimizerContext(new Memo(), new ColumnRefFactory())); - - assertEquals(2, ((LogicalIcebergScanOperator) scan.getOp()).getScanOperatorPredicates().getMinMaxConjuncts().size()); - - PredicateOperator binaryPredicateOperatorNoPushDown = new BinaryPredicateOperator(BinaryType.EQ, - new ColumnRefOperator(2, Type.INT, "id_noexist", true), ConstantOperator.createInt(1)); - - OptExpression scanNoPushDown = - new OptExpression(new LogicalIcebergScanOperator(table, colRefToColumnMetaMap, columnMetaToColRefMap, - -1, binaryPredicateOperatorNoPushDown, TableVersionRange.empty())); - - assertEquals(0, - ((LogicalIcebergScanOperator) scanNoPushDown.getOp()).getScanOperatorPredicates().getMinMaxConjuncts().size()); - - rule0.transform(scanNoPushDown, new OptimizerContext(new Memo(), new ColumnRefFactory())); - - assertEquals(0, - ((LogicalIcebergScanOperator) scanNoPushDown.getOp()).getScanOperatorPredicates().getMinMaxConjuncts().size()); - } - - @Test - public void testPartitionPrune() { - ExternalScanPartitionPruneRule rule0 = ExternalScanPartitionPruneRule.ICEBERG_SCAN; - - mockedNativeTableA.newAppend().appendFile(FILE_A).commit(); - mockedNativeTableA.refresh(); - List columns = Lists.newArrayList(new Column("id", INT), new Column("data", STRING)); - IcebergTable icebergTable = new IcebergTable(1, "srTableName", "iceberg_catalog", "resource_name", "iceberg_db", - "iceberg_table", "", columns, mockedNativeTableA, Maps.newHashMap()); - - ColumnRefOperator colRef1 = new ColumnRefOperator(1, Type.INT, "id", true); - Column col1 = new Column("id", Type.INT, true); - ColumnRefOperator colRef2 = new ColumnRefOperator(2, Type.STRING, "data", true); - Column col2 = new Column("data", Type.STRING, true); - - Map colRefToColumnMetaMap = new HashMap<>(); - Map columnMetaToColRefMap = new HashMap<>(); - colRefToColumnMetaMap.put(colRef1, col1); - columnMetaToColRefMap.put(col2, colRef2); - TableVersionRange version = TableVersionRange.withEnd(Optional.of( - mockedNativeTableA.currentSnapshot().snapshotId())); - OptExpression scan = - new OptExpression(new LogicalIcebergScanOperator(icebergTable, colRefToColumnMetaMap, columnMetaToColRefMap, - -1, null, version)); - - rule0.transform(scan, new OptimizerContext(new Memo(), new ColumnRefFactory())); - assertEquals(1, ((LogicalIcebergScanOperator) scan.getOp()).getScanOperatorPredicates() - .getSelectedPartitionIds().size()); - - } -} diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java index 49c556a862517..1a197d0f2cc59 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java @@ -39,6 +39,7 @@ public static void beforeClass() throws Exception { MvRewriteTestBase.beforeClass(); ConnectorPlanTestBase.mockCatalog(connectContext, MockIcebergMetadata.MOCKED_ICEBERG_CATALOG_NAME); connectContext.getSessionVariable().setMaterializedViewUnionRewriteMode(1); + connectContext.getSessionVariable().setEnableMaterializedViewTransparentUnionRewrite(false); } @Test diff --git a/test/sql/test_materialized_view/R/test_mv_iceberg_rewrite b/test/sql/test_materialized_view/R/test_mv_iceberg_rewrite index e1050eca9211e..1df2f20cec28e 100644 --- a/test/sql/test_materialized_view/R/test_mv_iceberg_rewrite +++ b/test/sql/test_materialized_view/R/test_mv_iceberg_rewrite @@ -58,10 +58,16 @@ function: check_hit_materialized_view("SELECT dt,sum(col_int) FROM mv_iceberg_${ -- result: None -- !result +set enable_materialized_view_transparent_union_rewrite=false; +-- result: +-- !result function: check_no_hit_materialized_view("SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-03' GROUP BY dt", "test_mv1") -- result: None -- !result +set enable_materialized_view_transparent_union_rewrite=true; +-- result: +-- !result SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-02' GROUP BY dt; -- result: 2023-12-02 4000 diff --git a/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part1 b/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part1 index fd212cd68f22e..abb7e6a98c014 100644 --- a/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part1 +++ b/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part1 @@ -270,6 +270,9 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result +set enable_materialized_view_transparent_union_rewrite=true; +-- result: +-- !result function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") -- result: None @@ -282,15 +285,18 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result @@ -653,6 +659,9 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result +set enable_materialized_view_transparent_union_rewrite=true; +-- result: +-- !result function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") -- result: None @@ -665,15 +674,18 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result diff --git a/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part2 b/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part2 index f2f5801a806fa..de12ab2d2b872 100644 --- a/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part2 +++ b/test/sql/test_materialized_view/R/test_mv_partition_compensate_iceberg_part2 @@ -262,11 +262,14 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=true; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") -- result: None -- !result @@ -282,15 +285,18 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result @@ -645,11 +651,14 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=true; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") -- result: None -- !result @@ -665,15 +674,18 @@ function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; +-- result: +-- !result +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- result: None -- !result -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -- result: None -- !result diff --git a/test/sql/test_materialized_view/T/test_mv_iceberg_rewrite b/test/sql/test_materialized_view/T/test_mv_iceberg_rewrite index 771558c020ad2..f9f000ce52cb8 100644 --- a/test/sql/test_materialized_view/T/test_mv_iceberg_rewrite +++ b/test/sql/test_materialized_view/T/test_mv_iceberg_rewrite @@ -36,8 +36,9 @@ FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} GROUP BY dt; REFRESH MATERIALIZED VIEW test_mv1 PARTITION START ("2023-12-01") END ("2023-12-03") WITH SYNC MODE; function: check_hit_materialized_view("SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-01' GROUP BY dt", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-02' GROUP BY dt", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; function: check_no_hit_materialized_view("SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-03' GROUP BY dt", "test_mv1") - +set enable_materialized_view_transparent_union_rewrite=true; SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-02' GROUP BY dt; SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt='2023-12-03' GROUP BY dt; SELECT dt,sum(col_int) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.mv_ice_tbl_${uuid0} WHERE dt>='2023-12-03' GROUP BY dt order by dt; diff --git a/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part1 b/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part1 index 143abf0674872..b1e62b6064087 100644 --- a/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part1 +++ b/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part1 @@ -88,13 +88,17 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('day', dt) ='2020-06-21' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") + +set enable_materialized_view_transparent_union_rewrite=true; function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt <= '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where (dt>'2020-06-15' and dt <= '2020-06-22') or dt>'2020-07-01' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; + function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-15' and '2020-07-01' GROUP BY dt;", "test_mv1", "UNION") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>='2020-06-15' GROUP BY dt;", "test_mv1", "UNION") @@ -163,13 +167,17 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('day', dt) ='2020-06-21' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") + +set enable_materialized_view_transparent_union_rewrite=true; function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>'2020-06-15' and dt <= '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where (dt>'2020-06-15' and dt <= '2020-06-22') or dt>'2020-07-01' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; + function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt between '2020-06-15' and '2020-07-01' GROUP BY dt;", "test_mv1", "UNION") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t1 where dt>='2020-06-15' GROUP BY dt;", "test_mv1", "UNION") diff --git a/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part2 b/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part2 index e77af677cd9d4..9102c3ddf6432 100644 --- a/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part2 +++ b/test/sql/test_materialized_view/T/test_mv_partition_compensate_iceberg_part2 @@ -87,15 +87,18 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('day', dt) ='2020-06-21' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=true; +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' and dt <= '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where (dt>'2020-06-15' and dt <= '2020-06-22') or dt>'2020-07-01' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; + function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-15' and '2020-07-01' GROUP BY dt;", "test_mv1", "UNION") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>='2020-06-15' GROUP BY dt;", "test_mv1", "UNION") @@ -164,15 +167,18 @@ function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${u -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('day', dt) ='2020-06-21' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") + function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=true; +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt !='2020-06-15' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' and dt < '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>'2020-06-15' and dt <= '2020-07-22' GROUP BY dt;", "test_mv1") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where (dt>'2020-06-15' and dt <= '2020-06-22') or dt>'2020-07-01' GROUP BY dt;", "test_mv1") +function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") +set enable_materialized_view_transparent_union_rewrite=false; function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt in ('2020-06-21', '2020-07-25') GROUP BY dt;", "test_mv1") -- cannot partition prune function: check_no_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where date_trunc('month', dt) ='2020-07-01' GROUP BY dt;", "test_mv1") -function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-16' and '2020-07-01' GROUP BY dt;", "test_mv1") -- function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt between '2020-06-15' and '2020-07-01' GROUP BY dt;", "test_mv1", "UNION") function: check_hit_materialized_view("SELECT dt,sum(num) FROM mv_iceberg_${uuid0}.mv_ice_db_${uuid0}.t2 where dt>='2020-06-15' GROUP BY dt;", "test_mv1", "UNION")