diff --git a/src/common/expression/PredicateExpression.cpp b/src/common/expression/PredicateExpression.cpp index 16390619f24..88f71a058c4 100644 --- a/src/common/expression/PredicateExpression.cpp +++ b/src/common/expression/PredicateExpression.cpp @@ -19,7 +19,8 @@ std::unordered_map PredicateExpression:: const Value& PredicateExpression::evalExists(ExpressionContext& ctx) { DCHECK(collection_->kind() == Expression::Kind::kAttribute || collection_->kind() == Expression::Kind::kSubscript || - collection_->kind() == Expression::Kind::kLabelTagProperty); + collection_->kind() == Expression::Kind::kLabelTagProperty) + << "actual kind: " << collection_->kind() << ", toString: " << toString(); if (collection_->kind() == Expression::Kind::kLabelTagProperty) { result_ = !collection_->eval(ctx).isNull(); diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index cae7300d687..b27c20e97aa 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -54,6 +54,7 @@ nebula_add_library( rule/EliminateAppendVerticesRule.cpp rule/PushLimitDownScanEdgesRule.cpp rule/RemoveProjectDedupBeforeGetDstBySrcRule.cpp + rule/PushFilterDownTraverseRule.cpp ) nebula_add_subdirectory(test) diff --git a/src/graph/optimizer/OptGroup.cpp b/src/graph/optimizer/OptGroup.cpp index 569b5ec43c8..340881b224d 100644 --- a/src/graph/optimizer/OptGroup.cpp +++ b/src/graph/optimizer/OptGroup.cpp @@ -119,7 +119,9 @@ void OptGroup::addGroupNode(OptGroupNode *groupNode) { if (outputVar_.empty()) { outputVar_ = groupNode->node()->outputVar(); } else { - DCHECK_EQ(outputVar_, groupNode->node()->outputVar()); + DCHECK_EQ(outputVar_, groupNode->node()->outputVar()) + << "outputVar_: " << outputVar_ << "groupNode plan: " << groupNode->node()->toString() + << ", groupNode->node()->outputVar(): " << groupNode->node()->outputVar(); } groupNodes_.emplace_back(groupNode); groupNode->node()->updateSymbols(); diff --git a/src/graph/optimizer/rule/PushFilterDownTraverseRule.cpp b/src/graph/optimizer/rule/PushFilterDownTraverseRule.cpp new file mode 100644 index 00000000000..0ad74b1dd01 --- /dev/null +++ b/src/graph/optimizer/rule/PushFilterDownTraverseRule.cpp @@ -0,0 +1,152 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/PushFilterDownTraverseRule.h" + +#include "common/expression/ConstantExpression.h" +#include "common/expression/Expression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" +#include "graph/visitor/ExtractFilterExprVisitor.h" + +using nebula::Expression; +using nebula::graph::Filter; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; +using nebula::graph::Traverse; + +namespace nebula { +namespace opt { + +std::unique_ptr PushFilterDownTraverseRule::kInstance = + std::unique_ptr(new PushFilterDownTraverseRule()); + +PushFilterDownTraverseRule::PushFilterDownTraverseRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern& PushFilterDownTraverseRule::pattern() const { + static Pattern pattern = + Pattern::create(PlanNode::Kind::kFilter, + {Pattern::create(PlanNode::Kind::kAppendVertices, + {Pattern::create(PlanNode::Kind::kTraverse)})}); + return pattern; +} + +bool PushFilterDownTraverseRule::match(OptContext* ctx, const MatchedResult& matched) const { + if (!OptRule::match(ctx, matched)) { + return false; + } + DCHECK_EQ(matched.dependencies[0].dependencies[0].node->node()->kind(), + PlanNode::Kind::kTraverse); + auto traverse = + static_cast(matched.dependencies[0].dependencies[0].node->node()); + return traverse->isOneStep(); +} + +StatusOr PushFilterDownTraverseRule::transform( + OptContext* ctx, const MatchedResult& matched) const { + auto* filterGroupNode = matched.node; + auto* filterGroup = filterGroupNode->group(); + auto* filter = static_cast(filterGroupNode->node()); + auto* condition = filter->condition(); + + auto* avGroupNode = matched.dependencies[0].node; + auto* av = static_cast(avGroupNode->node()); + + auto* tvGroupNode = matched.dependencies[0].dependencies[0].node; + auto* tv = static_cast(tvGroupNode->node()); + auto& edgeAlias = tv->edgeAlias(); + + auto qctx = ctx->qctx(); + auto pool = qctx->objPool(); + + // Pick the expr looks like `$-.e[0].likeness + auto picker = [&edgeAlias](const Expression* e) -> bool { + // TODO(jie): Handle the strange exists expr. e.g. exists(e.likeness) + auto exprs = graph::ExpressionUtils::collectAll(e, {Expression::Kind::kPredicate}); + for (auto* expr : exprs) { + if (static_cast(expr)->name() == "exists") { + return false; + } + } + + auto varProps = graph::ExpressionUtils::collectAll( + e, {Expression::Kind::kInputProperty, Expression::Kind::kVarProperty}); + if (varProps.empty()) { + return false; + } + for (auto* expr : varProps) { + DCHECK(graph::ExpressionUtils::isPropertyExpr(expr)); + auto& propName = static_cast(expr)->prop(); + if (propName != edgeAlias) return false; + } + return true; + }; + Expression* filterPicked = nullptr; + Expression* filterUnpicked = nullptr; + graph::ExpressionUtils::splitFilter(condition, picker, &filterPicked, &filterUnpicked); + + if (!filterPicked) { + return TransformResult::noTransform(); + } + auto* newFilterPicked = + graph::ExpressionUtils::rewriteEdgePropertyFilter(pool, edgeAlias, filterPicked->clone()); + + Filter* newFilter = nullptr; + OptGroupNode* newFilterGroupNode = nullptr; + if (filterUnpicked) { + newFilter = Filter::make(qctx, nullptr, filterUnpicked); + newFilter->setOutputVar(filter->outputVar()); + newFilter->setColNames(filter->colNames()); + newFilterGroupNode = OptGroupNode::create(ctx, newFilter, filterGroup); + } + + auto* newAv = static_cast(av->clone()); + + OptGroupNode* newAvGroupNode = nullptr; + if (newFilterGroupNode) { + auto* newAvGroup = OptGroup::create(ctx); + newAvGroupNode = newAvGroup->makeGroupNode(newAv); + newFilterGroupNode->dependsOn(newAvGroup); + newFilter->setInputVar(newAv->outputVar()); + } else { + newAvGroupNode = OptGroupNode::create(ctx, newAv, filterGroup); + newAv->setOutputVar(filter->outputVar()); + } + + auto* eFilter = tv->eFilter(); + Expression* newEFilter = eFilter + ? LogicalExpression::makeAnd(pool, newFilterPicked, eFilter->clone()) + : newFilterPicked; + + auto* newTv = static_cast(tv->clone()); + newAv->setInputVar(newTv->outputVar()); + newTv->setEdgeFilter(newEFilter); + + auto* newTvGroup = OptGroup::create(ctx); + newAvGroupNode->dependsOn(newTvGroup); + auto* newTvGroupNode = newTvGroup->makeGroupNode(newTv); + + for (auto dep : tvGroupNode->dependencies()) { + newTvGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseCurr = true; + result.newGroupNodes.emplace_back(newFilterGroupNode ? newFilterGroupNode : newAvGroupNode); + + return result; +} + +std::string PushFilterDownTraverseRule::toString() const { + return "PushFilterDownTraverseRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushFilterDownTraverseRule.h b/src/graph/optimizer/rule/PushFilterDownTraverseRule.h new file mode 100644 index 00000000000..459773a2017 --- /dev/null +++ b/src/graph/optimizer/rule/PushFilterDownTraverseRule.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#pragma once + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +/* + * Before: + * Filter(e.likeness > 78) + * | + * AppendVertices + * | + * Traverse + * + * After : + * AppendVertices + * | + * Traverse(eFilter_: *.likeness > 78) + */ +class PushFilterDownTraverseRule final : public OptRule { + public: + const Pattern &pattern() const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + + StatusOr transform(OptContext *ctx, const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + PushFilterDownTraverseRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index b39108d9f7c..24b3c8d03d5 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -1538,6 +1538,10 @@ class Traverse final : public GetNeighbors { return range_; } + bool isOneStep() const { + return !range_; + } + // Contains zero step bool zeroStep() const { return range_ != nullptr && range_->min() == 0; @@ -1555,6 +1559,17 @@ class Traverse final : public GetNeighbors { return trackPrevPath_; } + const std::string& nodeAlias() const { + auto& cols = this->colNames(); + DCHECK_GE(cols.size(), 2); + return cols[cols.size() - 2]; + } + + const std::string& edgeAlias() const { + DCHECK(!this->colNames().empty()); + return this->colNames().back(); + } + void setStepRange(MatchStepRange* range) { range_ = range; } @@ -1618,6 +1633,11 @@ class AppendVertices final : public GetVertices { return trackPrevPath_; } + const std::string nodeAlias() const { + DCHECK(!this->colNames().empty()); + return this->colNames().back(); + } + void setVertexFilter(Expression* vFilter) { vFilter_ = vFilter; } diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index d8ce7d8953b..2975681b372 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -350,6 +350,7 @@ Expression *ExpressionUtils::rewriteInExpr(const Expression *expr) { return const_cast(expr); } std::vector operands; + operands.reserve(values.size()); for (const auto &v : values) { operands.emplace_back( RelationalExpression::makeEQ(pool, inExpr->left(), ConstantExpression::make(pool, v))); @@ -1431,5 +1432,67 @@ bool ExpressionUtils::checkExprDepth(const Expression *expr) { return true; } +/*static*/ +bool ExpressionUtils::isSingleLenExpandExpr(const std::string &edgeAlias, const Expression *expr) { + if (expr->kind() != Expression::Kind::kAttribute) { + return false; + } + auto attributeExpr = static_cast(expr); + auto *left = attributeExpr->left(); + auto *right = attributeExpr->right(); + + if (left->kind() != Expression::Kind::kSubscript) return false; + if (right->kind() != Expression::Kind::kConstant || + !static_cast(right)->value().isStr()) + return false; + + auto subscriptExpr = static_cast(left); + auto *listExpr = subscriptExpr->left(); + auto *idxExpr = subscriptExpr->right(); + if (listExpr->kind() != Expression::Kind::kInputProperty && + listExpr->kind() != Expression::Kind::kVarProperty) { + return false; + } + if (static_cast(listExpr)->prop() != edgeAlias) { + return false; + } + + // NOTE(jie): Just handled `$-.e[0].likeness` for now, whileas the traverse is single length + // expand. + // TODO(jie): Handle `ALL(i IN e WHERE i.likeness > 78)`, whileas the traverse is var len + // expand. + if (idxExpr->kind() != Expression::Kind::kConstant || + static_cast(idxExpr)->value() != 0) { + return false; + } + return true; +} + +// Transform expression `$-.e[0].likeness` to EdgePropertyExpression `like.likeness` +// for more friendly to push down +// \param pool object pool to hold ownership of objects alloacted +// \param edgeAlias the name of edge. e.g. e in pattern -[e]-> +// \param expr the filter expression +/*static*/ Expression *ExpressionUtils::rewriteEdgePropertyFilter(ObjectPool *pool, + const std::string &edgeAlias, + Expression *expr) { + graph::RewriteVisitor::Matcher matcher = [&edgeAlias](const Expression *e) -> bool { + return isSingleLenExpandExpr(edgeAlias, e); + }; + graph::RewriteVisitor::Rewriter rewriter = [pool](const Expression *e) -> Expression * { + DCHECK_EQ(e->kind(), Expression::Kind::kAttribute); + auto attributeExpr = static_cast(e); + auto *right = attributeExpr->right(); + DCHECK_EQ(right->kind(), Expression::Kind::kConstant); + + auto &prop = static_cast(right)->value().getStr(); + + auto *edgePropExpr = EdgePropertyExpression::make(pool, "*", prop); + return edgePropExpr; + }; + + return graph::RewriteVisitor::transform(expr, matcher, rewriter); +} + } // namespace graph } // namespace nebula diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h index 75af5021080..5a69cef1998 100644 --- a/src/graph/util/ExpressionUtils.h +++ b/src/graph/util/ExpressionUtils.h @@ -228,6 +228,13 @@ class ExpressionUtils { // Whether the whole expression is vertex id predication // e.g. id(v) == 1, id(v) IN [...] static bool isVidPredication(const Expression* expr); + + // Check if the expr looks like `$-.e[0].likeness` + static bool isSingleLenExpandExpr(const std::string& edgeAlias, const Expression* expr); + + static Expression* rewriteEdgePropertyFilter(ObjectPool* pool, + const std::string& edgeAlias, + Expression* expr); }; } // namespace graph diff --git a/src/storage/query/QueryBaseProcessor-inl.h b/src/storage/query/QueryBaseProcessor-inl.h index e1291920fce..c459b76d1dd 100644 --- a/src/storage/query/QueryBaseProcessor-inl.h +++ b/src/storage/query/QueryBaseProcessor-inl.h @@ -407,9 +407,11 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp( if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } - ret = checkExp(predExp->filter(), returned, filtered, updated, allowNoexistentProp); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; + if (predExp->hasFilter()) { + ret = checkExp(predExp->filter(), returned, filtered, updated, allowNoexistentProp); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } } return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/tests/tck/features/optimizer/PushFilterDownHashInnerJoinRule.feature b/tests/tck/features/optimizer/PushFilterDownHashInnerJoinRule.feature index 43c8c956c38..e7c09c3b2d8 100644 --- a/tests/tck/features/optimizer/PushFilterDownHashInnerJoinRule.feature +++ b/tests/tck/features/optimizer/PushFilterDownHashInnerJoinRule.feature @@ -32,21 +32,19 @@ Feature: Push Filter down HashInnerJoin rule | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | And the execution plan should be: - | id | name | dependencies | operator info | - | 30 | Sort | 14 | | - | 14 | Project | 19 | | - | 19 | HashInnerJoin | 6,22 | | - | 6 | Project | 20 | | - | 20 | AppendVertices | 2 | | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | - | 22 | Project | 21 | | - | 21 | Filter | 10 | { "condition": "($-.e[0].likeness>0)" } | - | 10 | AppendVertices | 9 | | - | 9 | Traverse | 7 | | - | 7 | Argument | 8 | | - | 8 | Start | | | + | id | name | dependencies | operator info | + | 30 | Sort | 14 | | + | 14 | Project | 19 | | + | 19 | HashInnerJoin | 6,22 | | + | 6 | Project | 20 | | + | 20 | AppendVertices | 2 | | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 22 | Project | 10 | | + | 10 | AppendVertices | 9 | | + | 9 | Traverse | 7 | {"edge filter": "(*.likeness>0)"} | + | 7 | Argument | | | When profiling query: """ MATCH (v:player) @@ -112,22 +110,20 @@ Feature: Push Filter down HashInnerJoin rule | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | And the execution plan should be: - | id | name | dependencies | operator info | - | 30 | Sort | 14 | | - | 14 | Project | 20 | | - | 20 | HashInnerJoin | 23,25 | | - | 23 | Project | 22 | | - | 22 | Filter | 21 | {"condition": "(v.player.age>0)"} | - | 21 | AppendVertices | 2 | | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | - | 25 | Project | 24 | | - | 24 | Filter | 10 | {"condition": "($-.e[0].likeness>0)"} | - | 10 | AppendVertices | 9 | | - | 9 | Traverse | 7 | | - | 7 | Argument | 8 | | - | 8 | Start | | | + | id | name | dependencies | operator info | + | 30 | Sort | 14 | | + | 14 | Project | 20 | | + | 20 | HashInnerJoin | 23,25 | | + | 23 | Project | 22 | | + | 22 | Filter | 21 | {"condition": "(v.player.age>0)"} | + | 21 | AppendVertices | 2 | | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 25 | Project | 10 | | + | 10 | AppendVertices | 9 | | + | 9 | Traverse | 7 | {"edge filter": "(*.likeness>0)"} | + | 7 | Argument | | | When profiling query: """ MATCH (v:player) diff --git a/tests/tck/features/optimizer/PushFilterDownTraverseRule.feature b/tests/tck/features/optimizer/PushFilterDownTraverseRule.feature new file mode 100644 index 00000000000..56aed295543 --- /dev/null +++ b/tests/tck/features/optimizer/PushFilterDownTraverseRule.feature @@ -0,0 +1,97 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Push Filter down Traverse rule + + Background: + Given a graph with space named "nba" + + Scenario: push filter down Traverse + When profiling query: + """ + MATCH (v:player)-[e:like]->(v2) WHERE e.likeness > 99 RETURN e.likeness, v2.player.age + """ + Then the result should be, in any order: + | e.likeness | v2.player.age | + | 100 | 31 | + | 100 | 43 | + # The filter `e.likeness>99` is first pushed down to the `eFilter_` of `Traverese` by rule `PushFilterDownTraverse`, + # and then pushed down to the `filter_` of `Traverse` by rule `PushEFilterDown`. + And the execution plan should be: + | id | name | dependencies | operator info | + | 6 | Project | 5 | | + | 5 | AppendVertices | 10 | | + | 10 | Traverse | 2 | {"filter": "(like.likeness>99)"} | + | 2 | Dedup | 9 | | + | 9 | IndexScan | 3 | | + | 3 | Start | | | + When profiling query: + """ + MATCH (person:player)-[:like*1..2]-(friend:player)-[served:serve]->(friendTeam:team) + WHERE id(person) == "Tony Parker" AND id(friend) != "Tony Parker" AND served.start_year > 2010 + WITH DISTINCT friend, friendTeam + OPTIONAL MATCH (friend)<-[:like]-(friend2:player)<-[:like]-(friendTeam) + WITH friendTeam, count(friend2) AS numFriends + RETURN + friendTeam.team.name AS teamName, + numFriends + ORDER BY teamName DESC + LIMIT 8 + """ + Then the result should be, in any order, with relax comparison: + | teamName | numFriends | + | "Warriors" | 0 | + | "Trail Blazers" | 0 | + | "Spurs" | 0 | + | "Rockets" | 0 | + | "Raptors" | 0 | + | "Pistons" | 0 | + | "Lakers" | 0 | + | "Kings" | 0 | + # The filter `served.start_year` is first pushed down to the `eFilter_` of `Traverese` by rule `PushFilterDownTraverse`, + # and then pushed down to the `filter_` of `Traverse` by rule `PushEFilterDown`. + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 21 | TopN | 18 | | | + | 18 | Project | 17 | | | + | 17 | Aggregate | 16 | | | + | 16 | HashLeftJoin | 10,15 | | | + | 10 | Dedup | 28 | | | + | 28 | Project | 22 | | | + | 22 | Filter | 26 | | | + | 26 | AppendVertices | 25 | | | + | 25 | Traverse | 24 | | {"filter": "(serve.start_year>2010)"} | + | 24 | Traverse | 2 | | | + | 2 | Dedup | 1 | | | + | 1 | PassThrough | 3 | | | + | 3 | Start | | | | + | 15 | Project | 15 | | | + | 30 | AppendVertices | 14 | | | + | 14 | Traverse | 12 | | | + | 12 | Traverse | 11 | | | + | 11 | Argument | | | | + + Scenario: push filter down Traverse with complex filter + When profiling query: + """ + MATCH (v:player)-[e:like]->(v2) WHERE v.player.age != 35 and (e.likeness + 100) != 199 + RETURN e.likeness, v2.player.age as age + ORDER BY age + LIMIT 3 + """ + Then the result should be, in any order: + | e.likeness | age | + | 90 | 20 | + | 80 | 22 | + | 90 | 23 | + # The filter `e.likeness+100!=99` is first pushed down to the `eFilter_` of `Traverese` by rule `PushFilterDownTraverse`, + # and then pushed down to the `filter_` of `Traverse` by rule `PushEFilterDown`. + And the execution plan should be: + | id | name | dependencies | operator info | + | 11 | TopN | 10 | | + | 10 | Project | 9 | | + | 9 | Filter | 4 | {"condition": "(-.v.player.age!=35)" } | + | 4 | AppendVertices | 12 | | + | 12 | Traverse | 8 | {"filter": "((like.likeness+100)!=199)"} | + | 8 | IndexScan | 2 | | + | 2 | Start | | |