From f6de537e727b3edf3993538200102b609decda02 Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 1 Apr 2024 14:35:08 +0800 Subject: [PATCH 1/4] [Feature](materialized-view) support ignore not slot is null when count(slot) not has key in mv (#32912) support ignore not slot is null when count(slot) not has key in mv --- .../AbstractSelectMaterializedIndexRule.java | 95 +++++++++++++++++-- .../SelectMaterializedIndexWithAggregate.java | 24 +++-- .../mv_ignore_predicate.out | 15 +++ .../mv_ignore_predicate.groovy | 59 ++++++++++++ .../test_dup_mv_repeat.groovy | 2 +- 5 files changed, 178 insertions(+), 17 deletions(-) create mode 100644 regression-test/data/mv_p0/mv_ignore_predicate/mv_ignore_predicate.out create mode 100644 regression-test/suites/mv_p0/mv_ignore_predicate/mv_ignore_predicate.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 9633c6cb36a162..e69bffb301b31e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite.mv; import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndexMeta; @@ -33,12 +34,14 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -62,6 +65,7 @@ import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -72,6 +76,7 @@ import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Base class for selecting materialized index rules. @@ -109,6 +114,45 @@ protected boolean shouldSelectIndexWithoutAgg(LogicalOlapScan scan) { } } + // get the predicates that can be ignored when all aggregate functions are sum + protected static List getPrunedPredicatesWithAllSumAgg(List aggExpressions, + Set predicateExpr) { + List prunedExpr = new ArrayList<>(); + + Set sumSlots = aggExpressions.stream().map(e -> e.child(0).toSql()) + .collect(Collectors.toCollection(() -> new TreeSet(String.CASE_INSENSITIVE_ORDER))); + for (Expression expr : predicateExpr) { + if (expr instanceof Not && expr.child(0) instanceof IsNull) { + Expression slot = expr.child(0).child(0); + String countColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.SUM, + CreateMaterializedViewStmt.mvColumnBuilder(slotToCaseWhen(slot).toSql()))); + if (sumSlots.contains(countColumn)) { + prunedExpr.add(expr); + } + } + } + return prunedExpr; + } + + // we can prune some predicates when there is no group-by column + protected static List getPrunedPredicates(List aggExpressions, + Set predicateExpr) { + List prunedExpr = new ArrayList<>(); + + boolean isAllSumAgg = true; + for (Expression expr : aggExpressions) { + if (!(expr instanceof Sum)) { + isAllSumAgg = false; + break; + } + } + if (isAllSumAgg) { + prunedExpr.addAll(getPrunedPredicatesWithAllSumAgg(aggExpressions, predicateExpr)); + } + + return prunedExpr; + } + protected static boolean containAllRequiredColumns(MaterializedIndex index, LogicalOlapScan scan, Set requiredScanOutput, Set requiredExpr, Set predicateExpr) { OlapTable table = scan.getTable(); @@ -121,12 +165,14 @@ protected static boolean containAllRequiredColumns(MaterializedIndex index, Logi .map(e -> { e.setDisableTableName(true); return e; - }) - .map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet()); - Set commonConjuncts = indexConjuncts.stream().filter(predicateExprSql::contains) - .collect(Collectors.toSet()); - if (commonConjuncts.size() != indexConjuncts.size()) { - return false; + }).map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet()); + + for (String indexConjunct : indexConjuncts) { + if (predicateExprSql.contains(indexConjunct)) { + predicateExprSql.remove(indexConjunct); + } else { + return false; + } } Set requiredMvColumnNames = requiredScanOutput.stream() @@ -138,10 +184,24 @@ protected static boolean containAllRequiredColumns(MaterializedIndex index, Logi .collect(Collectors.toCollection(() -> new TreeSet(String.CASE_INSENSITIVE_ORDER))); mvColNames.addAll(indexConjuncts); - return mvColNames.containsAll(requiredMvColumnNames) - && (indexConjuncts.isEmpty() || commonConjuncts.size() == predicateExprSql.size()) - || requiredExpr.stream().filter(e -> !containsAllColumn(e, mvColNames)).collect(Collectors.toSet()) - .isEmpty(); + if (mvColNames.containsAll(requiredMvColumnNames) && predicateExprSql.isEmpty()) { + return true; + } + + Set remained = requiredExpr.stream().filter(e -> !containsAllColumn(e, mvColNames)) + .collect(Collectors.toSet()); + if (remained.isEmpty()) { + return true; + } + + if (!scan.getGroupExpression().isPresent()) { + Set prunedExpr = getPrunedPredicates( + requiredExpr.stream().filter(e -> e instanceof AggregateFunction).collect(Collectors.toList()), + predicateExpr).stream().collect(Collectors.toSet()); + remained = remained.stream().filter(e -> !prunedExpr.contains(e)).collect(Collectors.toSet()); + } + + return remained.isEmpty(); } public static String parseMvColumnToSql(String mvName) { @@ -433,6 +493,21 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { .collect(Collectors.toSet())); } + // Call this generateBaseScanExprToMvExpr only when we have both agg and filter + protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan, Set requiredExpr, + Set predicateExpr) { + SlotContext context = generateBaseScanExprToMvExpr(mvPlan); + if (mvPlan.getGroupExpression().isPresent()) { + return context; + } + Set pruned = getPrunedPredicates( + requiredExpr.stream().filter(e -> e instanceof AggregateFunction).collect(Collectors.toList()), + predicateExpr).stream().collect(Collectors.toSet()); + + return new SlotContext(context.baseSlotToMvSlot, context.mvNameToMvSlot, + Stream.concat(pruned.stream(), context.trueExprs.stream()).collect(Collectors.toSet())); + } + /** SlotContext */ protected static class SlotContext { public static final SlotContext EMPTY diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 198a03011fbdf1..b1a06e3875a466 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -166,7 +166,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); return new LogicalProject<>( generateProjectsAlias(agg.getOutputs(), slotContext), @@ -250,7 +252,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); if (result.indexId == scan.getTable().getBaseIndexId()) { LogicalOlapScan mvPlanWithoutAgg = SelectMaterializedIndexWithoutAggregate.select(scan, project::getInputSlots, filter::getConjuncts, @@ -311,7 +315,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); @@ -390,7 +396,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); return new LogicalProject<>( generateProjectsAlias(agg.getOutputs(), slotContext), @@ -481,7 +489,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); @@ -531,7 +541,9 @@ public List buildRules() { ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream() + .map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()), + filter.getConjuncts()); List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); diff --git a/regression-test/data/mv_p0/mv_ignore_predicate/mv_ignore_predicate.out b/regression-test/data/mv_p0/mv_ignore_predicate/mv_ignore_predicate.out new file mode 100644 index 00000000000000..e35122c75d0566 --- /dev/null +++ b/regression-test/data/mv_p0/mv_ignore_predicate/mv_ignore_predicate.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +\N 4 \N d +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c +5 \N \N \N + +-- !select_mv -- +5 + +-- !select_mv -- +5 + diff --git a/regression-test/suites/mv_p0/mv_ignore_predicate/mv_ignore_predicate.groovy b/regression-test/suites/mv_p0/mv_ignore_predicate/mv_ignore_predicate.groovy new file mode 100644 index 00000000000000..974dabda4668b5 --- /dev/null +++ b/regression-test/suites/mv_p0/mv_ignore_predicate/mv_ignore_predicate.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("mv_ignore_predicate") { + + sql """ DROP TABLE IF EXISTS d_table; """ + + sql """ + create table d_table( + k1 int null, + k2 int null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" + sql "insert into d_table select 3,-3,null,'c';" + + createMV("create materialized view kign as select k1,count(k2) from d_table group by k1;") + + sql "insert into d_table select -4,-4,-4,'d';" + sql "insert into d_table(k4,k2) values('d',4);" + sql "insert into d_table select 5,null,null,null;" + + qt_select_star "select * from d_table order by k1;" + + explain { + sql("select count(k2) from d_table;") + contains "(kign)" + } + qt_select_mv "select count(k2) from d_table;" + + explain { + sql("select count(k2) from d_table where k2 is not null;") + contains "(kign)" + } + qt_select_mv "select count(k2) from d_table where k2 is not null;" +} diff --git a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy index 0a40c3cb050577..f8672eaa259629 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite ("test_dup_mv_repeat") { - sql """ DROP TABLE IF EXISTS d_table; """ + sql """ DROP TABLE IF EXISTS db1; """ sql """ CREATE TABLE `db1` ( From fb703f91acd1cc5b3d1f97a3119e5eaff931da07 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:53:41 +0800 Subject: [PATCH 2/4] [Fix](nereids) fix qualifier problem that affects delete stmt in another catalog (#32853) --- .../nereids/rules/analysis/BindRelation.java | 30 +++++---- .../plans/logical/LogicalCatalogRelation.java | 31 +++++----- .../physical/PhysicalCatalogRelation.java | 22 ++++--- .../rules/analysis/BindRelationTest.java | 4 +- .../rules/rewrite/ColumnPruningTest.java | 62 +++++++++---------- 5 files changed, 81 insertions(+), 68 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index b1b0fe1045c299..33fba7f6ec3227 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -66,6 +66,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; @@ -189,7 +190,7 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List tabletIds = unboundRelation.getTabletIds(); if (!CollectionUtils.isEmpty(partIds)) { scan = new LogicalOlapScan(unboundRelation.getRelationId(), - (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, + (OlapTable) table, tableQualifier, partIds, tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); } else { Optional indexName = unboundRelation.getIndexName(); @@ -206,11 +207,11 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, : PreAggStatus.off("For direct index scan."); scan = new LogicalOlapScan(unboundRelation.getRelationId(), - (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, indexId, + (OlapTable) table, tableQualifier, tabletIds, indexId, preAggStatus, unboundRelation.getHints(), unboundRelation.getTableSample()); } else { scan = new LogicalOlapScan(unboundRelation.getRelationId(), - (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, unboundRelation.getHints(), + (OlapTable) table, tableQualifier, tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); } } @@ -238,10 +239,12 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List tableQualifier, CascadesContext cascadesContext) { + List qualifierWithoutTableName = Lists.newArrayList(); + qualifierWithoutTableName.addAll(tableQualifier.subList(0, tableQualifier.size() - 1)); switch (table.getType()) { case OLAP: case MATERIALIZED_VIEW: - return makeOlapScan(table, unboundRelation, tableQualifier); + return makeOlapScan(table, unboundRelation, qualifierWithoutTableName); case VIEW: View view = (View) table; String inlineViewDef = view.getInlineViewDef(); @@ -257,25 +260,26 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); } hmsTable.setScanParams(unboundRelation.getScanParams()); - return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, tableQualifier, - unboundRelation.getTableSample()); + return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, + qualifierWithoutTableName, unboundRelation.getTableSample()); case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: case TRINO_CONNECTOR_EXTERNAL_TABLE: - return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, tableQualifier, - unboundRelation.getTableSample()); + return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, + qualifierWithoutTableName, unboundRelation.getTableSample()); case SCHEMA: - return new LogicalSchemaScan(unboundRelation.getRelationId(), table, tableQualifier); + return new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case JDBC_EXTERNAL_TABLE: case JDBC: - return new LogicalJdbcScan(unboundRelation.getRelationId(), table, tableQualifier); + return new LogicalJdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case ODBC: - return new LogicalOdbcScan(unboundRelation.getRelationId(), table, tableQualifier); + return new LogicalOdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case ES_EXTERNAL_TABLE: - return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, tableQualifier); + return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, + qualifierWithoutTableName); case TEST_EXTERNAL_TABLE: - return new LogicalTestScan(unboundRelation.getRelationId(), table, tableQualifier); + return new LogicalTestScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); default: throw new AnalysisException("Unsupported tableType " + table.getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index b4dbc9444604da..5f7982aae46cab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -39,11 +39,11 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; @@ -57,7 +57,7 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements CatalogRelation { protected final TableIf table; - // [catalogName, databaseName, tableName] + // [catalogName, databaseName] protected final ImmutableList qualifier; public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier) { @@ -82,13 +82,20 @@ public TableIf getTable() { public DatabaseIf getDatabase() throws AnalysisException { Preconditions.checkArgument(!qualifier.isEmpty(), "qualifier can not be empty"); try { - CatalogIf catalog = qualifier.size() == 3 - ? Env.getCurrentEnv().getCatalogMgr().getCatalogOrException(qualifier.get(0), - s -> new Exception("Catalog [" + qualifier.get(0) + "] does not exist.")) - : Env.getCurrentEnv().getCurrentCatalog(); - return catalog.getDbOrException(qualifier.size() == 3 ? qualifier.get(1) : qualifier.get(0), - s -> new Exception("Database [" + qualifier.get(1) + "] does not exist in catalog [" - + qualifier.get(0) + "].")); + int len = qualifier.size(); + if (2 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(qualifier.get(0)); + return catalog.getDbOrAnalysisException(qualifier.get(1)); + } else if (1 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCurrentCatalog(); + return catalog.getDbOrAnalysisException(qualifier.get(0)); + } else if (0 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCurrentCatalog(); + ConnectContext ctx = ConnectContext.get(); + return catalog.getDb(ctx.getDatabase()).get(); + } + return null; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } @@ -110,9 +117,6 @@ public List getQualifier() { * Full qualified name parts, i.e., concat qualifier and name into a list. */ public List qualified() { - if (qualifier.size() == 3) { - return qualifier; - } return Utils.qualifiedNameParts(qualifier, table.getName()); } @@ -120,9 +124,6 @@ public List qualified() { * Full qualified table name, concat qualifier and name with `.` as separator. */ public String qualifiedName() { - if (qualifier.size() == 3) { - return StringUtils.join(qualifier, "."); - } return Utils.qualifiedName(qualifier, table.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java index 70ac9aaa645566..ddc26f1041c1fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; @@ -85,13 +86,20 @@ public TableIf getTable() { public DatabaseIf getDatabase() throws AnalysisException { Preconditions.checkArgument(!qualifier.isEmpty(), "qualifier can not be empty"); try { - CatalogIf catalog = qualifier.size() == 3 - ? Env.getCurrentEnv().getCatalogMgr().getCatalogOrException(qualifier.get(0), - s -> new Exception("Catalog [" + qualifier.get(0) + "] does not exist.")) - : Env.getCurrentEnv().getCurrentCatalog(); - return catalog.getDbOrException(qualifier.size() == 3 ? qualifier.get(1) : qualifier.get(0), - s -> new Exception("Database [" + qualifier.get(1) + "] does not exist in catalog [" - + qualifier.get(0) + "].")); + int len = qualifier.size(); + if (2 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(qualifier.get(0)); + return catalog.getDbOrAnalysisException(qualifier.get(1)); + } else if (1 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCurrentCatalog(); + return catalog.getDbOrAnalysisException(qualifier.get(0)); + } else if (0 == len) { + CatalogIf catalog = Env.getCurrentEnv().getCurrentCatalog(); + ConnectContext ctx = ConnectContext.get(); + return catalog.getDb(ctx.getDatabase()).get(); + } + return null; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index d719f3a8f89da8..25d0bb706bef68 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -64,7 +64,7 @@ void bindInCurrentDb() { Assertions.assertTrue(plan instanceof LogicalOlapScan); Assertions.assertEquals( - ImmutableList.of(DEFAULT_CLUSTER_PREFIX + DB1, "t"), + ImmutableList.of("internal", DEFAULT_CLUSTER_PREFIX + DB1, "t"), ((LogicalOlapScan) plan).qualified()); } @@ -76,7 +76,7 @@ void bindByDbQualifier() { Assertions.assertTrue(plan instanceof LogicalOlapScan); Assertions.assertEquals( - ImmutableList.of(DEFAULT_CLUSTER_PREFIX + DB1, "t"), + ImmutableList.of("internal", DEFAULT_CLUSTER_PREFIX + DB1, "t"), ((LogicalOlapScan) plan).qualified()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java index c19fd68a72b518..e6f2bb6838d9b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java @@ -70,18 +70,18 @@ public void testPruneColumns1() { logicalProject() .when(p -> getOutputQualifiedNames(p).containsAll( ImmutableList.of( - "test.student.id", - "test.student.name"))), + "internal.test.student.id", + "internal.test.student.name"))), logicalProject().when( p -> getOutputQualifiedNames(p).containsAll( ImmutableList.of( - "test.score.sid", - "test.score.grade"))) + "internal.test.score.sid", + "internal.test.score.grade"))) )) .when(p -> getOutputQualifiedNames(p) .containsAll( - ImmutableList.of("test.student.name", - "test.student.id"))) + ImmutableList.of("internal.test.student.name", + "internal.test.student.id"))) ) ) ); @@ -102,18 +102,18 @@ public void testPruneColumns2() { logicalProject() .when(p -> getOutputQualifiedNames(p).containsAll( ImmutableList.of( - "test.student.id", - "test.student.name", - "test.student.sex"))), + "internal.test.student.id", + "internal.test.student.name", + "internal.test.student.sex"))), logicalRelation() )) .when(p -> getOutputQualifiedNames(p) .containsAll( - ImmutableList.of("test.student.name", - "test.score.cid", - "test.score.grade", - "test.student.sex"))) + ImmutableList.of("internal.test.student.name", + "internal.test.score.cid", + "internal.test.score.grade", + "internal.test.student.sex"))) ) ) ); @@ -129,9 +129,9 @@ public void testPruneColumns3() { logicalFilter( logicalProject().when(p -> getOutputQualifiedNames(p) .containsAll(ImmutableList.of( - "test.student.name", - "test.student.id", - "test.student.age"))) + "internal.test.student.name", + "internal.test.student.id", + "internal.test.student.age"))) ) ) ); @@ -155,25 +155,25 @@ public void testPruneColumns4() { logicalProject(logicalRelation()) .when(p -> getOutputQualifiedNames( p).containsAll(ImmutableList.of( - "test.student.id", - "test.student.name"))), + "internal.test.student.id", + "internal.test.student.name"))), logicalRelation() )).when(p -> getOutputQualifiedNames(p) .containsAll(ImmutableList.of( - "test.student.name", - "test.score.cid", - "test.score.grade"))), + "internal.test.student.name", + "internal.test.score.cid", + "internal.test.score.grade"))), logicalProject(logicalRelation()) .when(p -> getOutputQualifiedNames(p) .containsAll(ImmutableList.of( - "test.course.cid", - "test.course.cname"))) + "internal.test.course.cid", + "internal.test.course.cname"))) ) ).when(p -> getOutputQualifiedNames(p).containsAll(ImmutableList.of( - "test.student.name", - "test.course.cname", - "test.score.grade"))) + "internal.test.student.name", + "internal.test.course.cname", + "internal.test.score.grade"))) ) ) ); @@ -280,12 +280,12 @@ public void pruneColumnForOneSideOnCrossJoin() { logicalProject(logicalRelation()) .when(p -> getOutputQualifiedNames(p) .containsAll(ImmutableList.of( - "test.student.id", - "test.student.name"))), + "internal.test.student.id", + "internal.test.student.name"))), logicalProject(logicalRelation()) .when(p -> getOutputQualifiedNames(p) .containsAll(ImmutableList.of( - "test.score.sid"))) + "internal.test.score.sid"))) ) ) ); @@ -304,10 +304,10 @@ public void pruneAggregateOutput() { logicalProject( logicalOlapScan() ).when(p -> getOutputQualifiedNames(p).equals( - ImmutableList.of("test.student.id") + ImmutableList.of("internal.test.student.id") )) ).when(agg -> getOutputQualifiedNames(agg.getOutputs()).equals( - ImmutableList.of("test.student.id") + ImmutableList.of("internal.test.student.id") ))) ) ) From cfc8954d6c64ac2a267a76a78657bef0d92eec92 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:06:44 +0800 Subject: [PATCH 3/4] [Fix](test) add sync to ensure data synchronization in test_set_operater (#32993) --- .../suites/query_p0/operator/test_set_operator.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/query_p0/operator/test_set_operator.groovy b/regression-test/suites/query_p0/operator/test_set_operator.groovy index ac091a9d8bb9bc..1bc9cc29e4c535 100644 --- a/regression-test/suites/query_p0/operator/test_set_operator.groovy +++ b/regression-test/suites/query_p0/operator/test_set_operator.groovy @@ -74,6 +74,7 @@ suite("test_set_operators", "query,p0,arrow_flight_sql") { sql """insert into t2 values('6',5,3,4);""" sql """insert into t2 values('9',8,0,7);""" + sql 'sync' order_qt_select """ select col1 @@ -88,4 +89,4 @@ suite("test_set_operators", "query,p0,arrow_flight_sql") { t3 on t2.col1=t3.col1; """ -} \ No newline at end of file +} From e53806b71e15e408508016e28e3e754ee8c0b215 Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 1 Apr 2024 15:46:54 +0800 Subject: [PATCH 4/4] [Chore](runtime-filter) add check is broadcast on nlj (#33088) add check is broadcast on nlj --- be/src/pipeline/exec/nested_loop_join_build_operator.cpp | 5 +++++ .../main/java/org/apache/doris/planner/RuntimeFilter.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp index f074afce374c08..52b27f03b5f001 100644 --- a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp +++ b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp @@ -44,6 +44,11 @@ Status NestedLoopJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkSta for (size_t i = 0; i < p._runtime_filter_descs.size(); i++) { RETURN_IF_ERROR(state->register_producer_runtime_filter( p._runtime_filter_descs[i], p._need_local_merge, &_runtime_filters[i], false)); + if (!_runtime_filters[i]->is_broadcast_join()) { + return Status::InternalError( + "runtime filter({}) on NestedLoopJoin should be set to is_broadcast_join,", + _runtime_filters[i]->get_name()); + } } return Status::OK(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 00117beae72899..a049079ccd4059 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -685,7 +685,7 @@ public void registerToPlan(Analyzer analyzer) { if (node instanceof HashJoinNode) { setIsBroadcast(((HashJoinNode) node).getDistributionMode() == HashJoinNode.DistributionMode.BROADCAST); } else { - setIsBroadcast(false); + setIsBroadcast(true); } if (LOG.isTraceEnabled()) { LOG.trace("Runtime filter: " + debugString());