From c6f702de4c1aa5e4f4bed1b79d9350e709e4c322 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Wed, 28 Jul 2021 17:58:18 +0800 Subject: [PATCH] combine projects (#1282) --- src/optimizer/CMakeLists.txt | 1 + src/optimizer/rule/CollapseProjectRule.cpp | 121 ++++++++++++++++++ src/optimizer/rule/CollapseProjectRule.h | 47 +++++++ .../rule/PushFilterDownAggregateRule.cpp | 2 +- .../optimizer/CollapseProjectRule.feature | 54 ++++++++ .../PushFilterDownLeftJoinRule.feature | 37 +++--- 6 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 src/optimizer/rule/CollapseProjectRule.cpp create mode 100644 src/optimizer/rule/CollapseProjectRule.h create mode 100644 tests/tck/features/optimizer/CollapseProjectRule.feature diff --git a/src/optimizer/CMakeLists.txt b/src/optimizer/CMakeLists.txt index 7c7aba1ae..06061dc13 100644 --- a/src/optimizer/CMakeLists.txt +++ b/src/optimizer/CMakeLists.txt @@ -14,6 +14,7 @@ nebula_add_library( rule/PushFilterDownGetNbrsRule.cpp rule/RemoveNoopProjectRule.cpp rule/CombineFilterRule.cpp + rule/CollapseProjectRule.cpp rule/MergeGetVerticesAndDedupRule.cpp rule/MergeGetVerticesAndProjectRule.cpp rule/MergeGetNbrsAndDedupRule.cpp diff --git a/src/optimizer/rule/CollapseProjectRule.cpp b/src/optimizer/rule/CollapseProjectRule.cpp new file mode 100644 index 000000000..481fce762 --- /dev/null +++ b/src/optimizer/rule/CollapseProjectRule.cpp @@ -0,0 +1,121 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "optimizer/rule/CollapseProjectRule.h" + +#include "optimizer/OptContext.h" +#include "optimizer/OptGroup.h" +#include "planner/plan/PlanNode.h" +#include "planner/plan/Query.h" + +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr CollapseProjectRule::kInstance = + std::unique_ptr(new CollapseProjectRule()); + +CollapseProjectRule::CollapseProjectRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern& CollapseProjectRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kProject, + {Pattern::create(graph::PlanNode::Kind::kProject)}); + return pattern; +} + +StatusOr CollapseProjectRule::transform( + OptContext* octx, + const MatchedResult& matched) const { + const auto* groupNodeAbove = matched.node; + const auto* projAbove = groupNodeAbove->node(); + DCHECK_EQ(projAbove->kind(), PlanNode::Kind::kProject); + const auto& deps = matched.dependencies; + DCHECK_EQ(deps.size(), 1); + const auto* groupNodeBelow = deps[0].node; + const auto* projBelow = groupNodeBelow->node(); + DCHECK_EQ(projBelow->kind(), PlanNode::Kind::kProject); + std::vector colsBelow = + static_cast(projBelow)->columns()->columns(); + const auto* projGroup = groupNodeAbove->group(); + + auto* newProj = static_cast(projAbove->clone()); + std::vector colsAbove = newProj->columns()->columns(); + + // 1. collect all property reference + std::vector allPropRefNames; + for (auto col : colsAbove) { + std::vector propRefs = graph::ExpressionUtils::collectAll( + col->expr(), {Expression::Kind::kVarProperty, Expression::Kind::kInputProperty}); + for (auto* expr : propRefs) { + DCHECK(expr->kind() == Expression::Kind::kVarProperty || + expr->kind() == Expression::Kind::kInputProperty); + allPropRefNames.emplace_back(static_cast(expr)->prop()); + } + } + + // disable this case to avoid the expression in ProjBelow being eval multiple times + std::unordered_set uniquePropRefNames; + for (auto p : allPropRefNames) { + if (!uniquePropRefNames.insert(p).second) { + return TransformResult::noTransform(); + } + } + + // 2. find link according to propRefNames and colNames in ProjBelow + std::unordered_map rewriteMap; + auto colNames = projBelow->colNames(); + for (size_t i = 0; i < colNames.size(); ++i) { + if (uniquePropRefNames.count(colNames[i])) { + rewriteMap[colNames[i]] = colsBelow[i]->expr(); + } + } + + // 3. rewrite YieldColumns + auto matcher = [&rewriteMap](const Expression* e) -> bool { + if (e->kind() != Expression::Kind::kVarProperty && + e->kind() != Expression::Kind::kInputProperty) { + return false; + } + auto& propName = static_cast(e)->prop(); + return rewriteMap.find(propName) != rewriteMap.end(); + }; + auto rewriter = [&rewriteMap](const Expression* e) -> Expression* { + DCHECK(e->kind() == Expression::Kind::kVarProperty || + e->kind() == Expression::Kind::kInputProperty); + auto& propName = static_cast(e)->prop(); + return rewriteMap[propName]->clone(); + }; + for (auto col : colsAbove) { + auto* newColExpr = graph::RewriteVisitor::transform(col->expr(), matcher, rewriter); + col->setExpr(newColExpr); + } + + // 4. rebuild OptGroupNode + newProj->setInputVar(projBelow->inputVar()); + auto* newGroupNode = OptGroupNode::create(octx, newProj, projGroup); + newGroupNode->setDeps(groupNodeBelow->dependencies()); + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newGroupNode); + + return result; +} + +bool CollapseProjectRule::match(OptContext* octx, const MatchedResult& matched) const { + return OptRule::match(octx, matched); +} + +std::string CollapseProjectRule::toString() const { + return "CollapseProjectRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/optimizer/rule/CollapseProjectRule.h b/src/optimizer/rule/CollapseProjectRule.h new file mode 100644 index 000000000..fb37c5d7d --- /dev/null +++ b/src/optimizer/rule/CollapseProjectRule.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef OPTIMIZER_RULE_COLLAPSEPROJECTRULE_H_ +#define OPTIMIZER_RULE_COLLAPSEPROJECTRULE_H_ + +#include + +#include "optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +/** + * Combines two [[Project]] nodes into one + * Required conditions: + * 1. Match the pattern + * 2. Expressions between nodes cannot be referenced more than once + * Benefits: + * 1. reduce the copy of memory between nodes + * 2. reduces expression overhead in some cases(similar to column pruning) + */ + +class CollapseProjectRule final : public OptRule { +public: + const Pattern &pattern() const override; + + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + + std::string toString() const override; + +private: + CollapseProjectRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif // OPTIMIZER_RULE_COLLAPSEPROJECTRULE_H_ diff --git a/src/optimizer/rule/PushFilterDownAggregateRule.cpp b/src/optimizer/rule/PushFilterDownAggregateRule.cpp index 929df355c..4cec82d3a 100644 --- a/src/optimizer/rule/PushFilterDownAggregateRule.cpp +++ b/src/optimizer/rule/PushFilterDownAggregateRule.cpp @@ -79,7 +79,7 @@ StatusOr PushFilterDownAggregateRule::transform( return false; } auto& propName = static_cast(e)->prop(); - return rewriteMap[propName]; + return rewriteMap.find(propName) != rewriteMap.end(); }; auto rewriter = [&rewriteMap](const Expression* e) -> Expression* { DCHECK_EQ(e->kind(), Expression::Kind::kVarProperty); diff --git a/tests/tck/features/optimizer/CollapseProjectRule.feature b/tests/tck/features/optimizer/CollapseProjectRule.feature new file mode 100644 index 000000000..a2cd2298e --- /dev/null +++ b/tests/tck/features/optimizer/CollapseProjectRule.feature @@ -0,0 +1,54 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Collapse Project Rule + + Background: + Given a graph with space named "nba" + + Scenario: Collapse Project + When profiling query: + """ + MATCH (v:player) + WHERE v.age > 38 + WITH v, v.age AS age, v.age/10 AS iage, v.age%10 AS mage,v.name AS name + RETURN iage + """ + Then the result should be, in any order: + | iage | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 4 | + | 3 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 16 | Project | 14 | | + | 14 | Filter | 7 | | + | 7 | Project | 6 | | + | 6 | Project | 5 | | + | 5 | Filter | 18 | | + | 18 | GetVertices | 12 | | + | 12 | IndexScan | 0 | | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON player + WHERE player.name=='Tim Duncan' + | YIELD $-.VertexID AS vid + """ + Then the result should be, in any order: + | vid | + | "Tim Duncan" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 6 | Project | 5 | | + | 5 | TagIndexPrefixScan | 0 | | + | 0 | Start | | | diff --git a/tests/tck/features/optimizer/PushFilterDownLeftJoinRule.feature b/tests/tck/features/optimizer/PushFilterDownLeftJoinRule.feature index 25c79158f..23e3208eb 100644 --- a/tests/tck/features/optimizer/PushFilterDownLeftJoinRule.feature +++ b/tests/tck/features/optimizer/PushFilterDownLeftJoinRule.feature @@ -23,25 +23,24 @@ Feature: Push Filter down LeftJoin rule | "Tim Duncan" | And the execution plan should be: | id | name | dependencies | operator info | - | 22 | Project | 21 | | - | 21 | Filter | 20 | | - | 20 | InnerJoin | 19 | | - | 19 | LeftJoin | 18 | | - | 18 | Project | 17 | | - | 17 | GetVertices | 16 | | - | 16 | Project | 28 | | - | 28 | GetNeighbors | 12 | | - | 12 | Project | 11 | | - | 11 | Filter | 10 | | - | 10 | InnerJoin | 9 | | - | 9 | LeftJoin | 8 | | - | 8 | Project | 7 | | - | 7 | GetVertices | 6 | | - | 6 | Project | 27 | | - | 27 | GetNeighbors | 2 | | - | 2 | Project | 3 | | - | 3 | Project | 30 | | - | 30 | TagIndexPrefixScan | 0 | | + | 24 | Project | 23 | | + | 23 | Filter | 22 | | + | 22 | InnerJoin | 21 | | + | 21 | LeftJoin | 20 | | + | 20 | Project | 19 | | + | 19 | GetVertices | 18 | | + | 18 | Project | 31 | | + | 31 | GetNeighbors | 14 | | + | 14 | Project | 13 | | + | 13 | Filter | 12 | | + | 12 | InnerJoin | 11 | | + | 11 | LeftJoin | 10 | | + | 10 | Project | 9 | | + | 9 | GetVertices | 8 | | + | 8 | Project | 30 | | + | 30 | GetNeighbors | 27 | | + | 27 | Project | 25 | | + | 25 | TagIndexPrefixScan | 0 | | | 0 | Start | | | When profiling query: """