Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

property pruner #1

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

property pruner #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d14272f
property pruner
jievince Jan 17, 2022
6c3c89e
postprocess works.
jievince Jan 18, 2022
d73fb11
add Travese::pruneProperties
jievince Jan 18, 2022
527c0d8
make vertexprops of travese be null when node alias is not used
jievince Jan 18, 2022
26982c4
fix extractPropsFromExps for tagPropExpr and edgePropexpr, vFilter an…
jievince Jan 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/common/expression/Expression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,14 @@ class Expression {

std::ostream& operator<<(std::ostream& os, Expression::Kind kind);

struct PropertyTracker {
std::unordered_map<std::string, std::unordered_map<TagID, std::unordered_set<std::string>>>
vertexPropsMap;
std::unordered_map<std::string, std::unordered_map<EdgeType, std::unordered_set<std::string>>>
edgePropsMap;
std::unordered_set<std::string> colsSet;
};

} // namespace nebula

#endif // COMMON_EXPRESSION_EXPRESSION_H_
11 changes: 5 additions & 6 deletions src/common/expression/PropertyExpression.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ std::string VariablePropertyExpression::toString() const {
}

const Value& LabelTagPropertyExpression::eval(ExpressionContext& ctx) {
const auto& var = label_->eval(ctx);
const auto& var = ctx.getVarProp("", label_);
if (var.type() != Value::Type::VERTEX) {
return Value::kNullBadType;
}
Expand All @@ -195,26 +195,25 @@ void LabelTagPropertyExpression::accept(ExprVisitor* visitor) {
}

std::string LabelTagPropertyExpression::toString() const {
std::string labelStr = label_ != nullptr ? label_->toString().erase(0, 1) : "";
return labelStr + "." + sym_ + "." + prop_;
return label_ + "." + sym_ + "." + prop_;
}

bool LabelTagPropertyExpression::operator==(const Expression& rhs) const {
if (kind_ != rhs.kind()) {
return false;
}
const auto& expr = static_cast<const LabelTagPropertyExpression&>(rhs);
return *label_ == *expr.label_ && sym_ == expr.sym_ && prop_ == expr.prop_;
return label_ == expr.label_ && sym_ == expr.sym_ && prop_ == expr.prop_;
}

void LabelTagPropertyExpression::writeTo(Encoder& encoder) const {
PropertyExpression::writeTo(encoder);
encoder << *label_;
encoder << label_;
}

void LabelTagPropertyExpression::resetFrom(Decoder& decoder) {
PropertyExpression::resetFrom(decoder);
label_ = decoder.readExpression(pool_);
label_ = decoder.readStr();
}

} // namespace nebula
18 changes: 7 additions & 11 deletions src/common/expression/PropertyExpression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ class LabelTagPropertyExpression final : public PropertyExpression {
LabelTagPropertyExpression& operator=(LabelTagPropertyExpression&&) = delete;

static LabelTagPropertyExpression* make(ObjectPool* pool,
Expression* label = nullptr,
const std::string& label = "",
const std::string& tag = "",
const std::string& prop = "") {
return pool->add(new LabelTagPropertyExpression(pool, label, tag, prop));
Expand All @@ -147,26 +147,22 @@ class LabelTagPropertyExpression final : public PropertyExpression {
return LabelTagPropertyExpression::make(pool_, label_, sym(), prop());
}

const Expression* label() const {
return label_;
}

Expression* label() {
const std::string& label() const {
return label_;
}

private:
LabelTagPropertyExpression(ObjectPool* pool,
Expression* label = nullptr,
const std::string& tag = "",
const std::string& prop = "")
explicit LabelTagPropertyExpression(ObjectPool* pool,
const std::string& label = "",
const std::string& tag = "",
const std::string& prop = "")
: PropertyExpression(pool, Kind::kLabelTagProperty, "", tag, prop), label_(label) {}

void writeTo(Encoder& encoder) const override;
void resetFrom(Decoder& decoder) override;

private:
Expression* label_{nullptr};
std::string label_;
};

// $-.any_prop_name
Expand Down
21 changes: 17 additions & 4 deletions src/graph/optimizer/Optimizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,25 @@ StatusOr<const PlanNode *> Optimizer::findBestPlan(QueryContext *qctx) {
auto optCtx = std::make_unique<OptContext>(qctx);

auto root = qctx->plan()->root();
auto status = prepare(optCtx.get(), root);
NG_RETURN_IF_ERROR(status);
auto rootGroup = std::move(status).value();
auto spaceID = qctx->rctx()->session()->space().id;

// auto status = preprocess(root, qctx, spaceID);
// NG_RETURN_IF_ERROR(status);

auto ret = prepare(optCtx.get(), root);
NG_RETURN_IF_ERROR(ret);
auto rootGroup = std::move(ret).value();

NG_RETURN_IF_ERROR(doExploration(optCtx.get(), rootGroup));
return rootGroup->getPlan();
auto *newRoot = rootGroup->getPlan();
auto status = preprocess(const_cast<PlanNode *>(newRoot), qctx, spaceID);
NG_RETURN_IF_ERROR(status);
return newRoot;
}

Status Optimizer::preprocess(PlanNode *root, graph::QueryContext *qctx, GraphSpaceID spaceID) {
PropertyTracker propsUsed;
return root->pruneProperties(propsUsed, qctx, spaceID);
}

StatusOr<OptGroup *> Optimizer::prepare(OptContext *ctx, PlanNode *root) {
Expand Down
4 changes: 4 additions & 0 deletions src/graph/optimizer/Optimizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#include "common/base/Base.h"
#include "common/base/StatusOr.h"
#include "common/thrift/ThriftTypes.h"

namespace nebula {
namespace graph {
Expand All @@ -30,7 +31,10 @@ class Optimizer final {
StatusOr<const graph::PlanNode *> findBestPlan(graph::QueryContext *qctx);

private:
Status preprocess(graph::PlanNode *root, graph::QueryContext *qctx, GraphSpaceID spaceID);

StatusOr<OptGroup *> prepare(OptContext *ctx, graph::PlanNode *root);

Status doExploration(OptContext *octx, OptGroup *rootGroup);

OptGroup *convertToGroup(OptContext *ctx,
Expand Down
3 changes: 1 addition & 2 deletions src/graph/planner/match/LabelIndexSeek.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,7 @@ StatusOr<SubPlan> LabelIndexSeek::transformNode(NodeContext* nodeCtx) {
bool matched = true;
for (auto* expr : exprs) {
auto tagPropExpr = static_cast<const LabelTagPropertyExpression*>(expr);
if (static_cast<const PropertyExpression*>(tagPropExpr->label())->prop() != nodeAlias ||
tagPropExpr->sym() != schemaName) {
if (tagPropExpr->label() != nodeAlias || tagPropExpr->sym() != schemaName) {
matched = false;
break;
}
Expand Down
3 changes: 1 addition & 2 deletions src/graph/planner/match/MatchSolver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,7 @@ Expression* MatchSolver::makeIndexFilter(const std::string& label,
} else {
continue;
}
if (static_cast<const PropertyExpression*>(la->label())->prop() != alias ||
la->sym() != label) {
if (la->label() != alias || la->sym() != label) {
continue;
}
propName = la->prop();
Expand Down
16 changes: 16 additions & 0 deletions src/graph/planner/plan/PlanNode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,22 @@ void PlanNode::releaseSymbols() {
}
}

Status PlanNode::pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) {
NG_RETURN_IF_ERROR(depsPruneProperties(propsUsed, qctx, spaceID));
return Status::OK();
}

Status PlanNode::depsPruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) {
for (const auto* dep : dependencies_) {
NG_RETURN_IF_ERROR(const_cast<PlanNode*>(dep)->pruneProperties(propsUsed, qctx, spaceID));
}
return Status::OK();
}

std::ostream& operator<<(std::ostream& os, PlanNode::Kind kind) {
os << PlanNode::toString(kind);
return os;
Expand Down
8 changes: 8 additions & 0 deletions src/graph/planner/plan/PlanNode.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,19 @@ class PlanNode {
return cost_;
}

virtual Status pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID);

protected:
PlanNode(QueryContext* qctx, Kind kind);

virtual ~PlanNode() = default;

Status depsPruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID);

static void addDescription(std::string key, std::string value, PlanNodeDescription* desc);

void readVariable(const std::string& varname);
Expand Down
95 changes: 95 additions & 0 deletions src/graph/planner/plan/Query.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <folly/String.h>
#include <folly/dynamic.h>
#include <folly/json.h>
#include <glog/logging.h>
#include <thrift/lib/cpp/util/EnumUtils.h>

#include "graph/util/ExpressionUtils.h"
Expand Down Expand Up @@ -351,6 +352,20 @@ void Project::cloneMembers(const Project& p) {
}
}

Status Project::pruneProperties(PropertyTracker& propsUsed,
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rewrite PropertyTracker's symbols according to Project alias?
Add test case:

MATCH (a:player)-->(b) WHERE a.player.age>40
with a AS b, b AS a where a.player.age>30
RETURN a.player.name, b.player.age

graph::QueryContext* qctx,
GraphSpaceID spaceID) {
if (cols_) {
for (auto* col : cols_->columns()) {
DCHECK_NOTNULL(col);
auto* expr = col->expr();
NG_RETURN_IF_ERROR(ExpressionUtils::extractPropsFromExprs(expr, propsUsed, qctx, spaceID));
}
}

return depsPruneProperties(propsUsed, qctx, spaceID);
}

std::unique_ptr<PlanNodeDescription> Unwind::explain() const {
auto desc = SingleInputNode::explain();
addDescription("alias", alias(), desc.get());
Expand Down Expand Up @@ -734,6 +749,46 @@ std::unique_ptr<PlanNodeDescription> Traverse::explain() const {
return desc;
}

Status Traverse::pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) {
auto& colNames = this->colNames();
DCHECK_GE(colNames.size(), 2);
auto& nodeAlias = colNames[colNames.size() - 2];

auto it = propsUsed.colsSet.find(nodeAlias);
if (it != propsUsed.colsSet.end()) { // All properties are used
propsUsed.colsSet.erase(it);
return depsPruneProperties(propsUsed, qctx, spaceID);
}

if (vFilter_ != nullptr) {
NG_RETURN_IF_ERROR(
ExpressionUtils::extractPropsFromExprs(vFilter_, propsUsed, qctx, spaceID, nodeAlias));
}

auto* vertexProps = this->vertexProps();
if (vertexProps != nullptr) {
auto it2 = propsUsed.vertexPropsMap.find(nodeAlias);
if (it2 == propsUsed.vertexPropsMap.end()) { // nodeAlias is not used
setVertexProps(nullptr);
} else {
auto prunedVertexProps = std::make_unique<std::vector<VertexProp>>();
auto& usedVertexProps = it2->second;
prunedVertexProps->reserve(usedVertexProps.size());
for (auto& [tagId, vProps] : usedVertexProps) {
VertexProp newVProp;
newVProp.tag_ref() = tagId;
newVProp.props_ref() = std::vector<std::string>(vProps.begin(), vProps.end());
prunedVertexProps->emplace_back(std::move(newVProp));
}
setVertexProps(std::move(prunedVertexProps));
}
}

return depsPruneProperties(propsUsed, qctx, spaceID);
}

AppendVertices* AppendVertices::clone() const {
auto newAV = AppendVertices::make(qctx_, nullptr, space_);
newAV->cloneMembers(*this);
Expand All @@ -760,6 +815,46 @@ std::unique_ptr<PlanNodeDescription> AppendVertices::explain() const {
return desc;
}

Status AppendVertices::pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) {
auto& colNames = this->colNames();
DCHECK(!colNames.empty());
auto& nodeAlias = colNames.back();
auto it = propsUsed.colsSet.find(nodeAlias);
if (it != propsUsed.colsSet.end()) { // All properties are used
propsUsed.colsSet.erase(it);
return depsPruneProperties(propsUsed, qctx, spaceID);
}

if (vFilter_ != nullptr) {
NG_RETURN_IF_ERROR(
ExpressionUtils::extractPropsFromExprs(vFilter_, propsUsed, qctx, spaceID, nodeAlias));
}
auto* vertexProps = props();
if (vertexProps != nullptr) {
auto prunedVertexProps = std::make_unique<std::vector<VertexProp>>();
auto it2 = propsUsed.vertexPropsMap.find(nodeAlias);
if (it2 != propsUsed.vertexPropsMap.end()) {
auto& usedVertexProps = it2->second;
prunedVertexProps->reserve(usedVertexProps.size());
for (auto& [tagId, vProps] : usedVertexProps) {
VertexProp newVProp;
newVProp.tag_ref() = tagId;
newVProp.props_ref() = std::vector<std::string>(vProps.begin(), vProps.end());
prunedVertexProps->emplace_back(std::move(newVProp));
}
} else {
// AppendVertices should be deleted
// Mark the node as toBeDeleted
// It should be done by ColumnPruner
}
setVertexProps(std::move(prunedVertexProps));
}

return depsPruneProperties(propsUsed, qctx, spaceID);
}

std::unique_ptr<PlanNodeDescription> BiJoin::explain() const {
auto desc = BinaryInputNode::explain();
addDescription("hashKeys", folly::toJson(util::toJson(hashKeys_)), desc.get());
Expand Down
12 changes: 12 additions & 0 deletions src/graph/planner/plan/Query.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,10 @@ class Project final : public SingleInputNode {
PlanNode* clone() const override;
std::unique_ptr<PlanNodeDescription> explain() const override;

Status pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) override;

private:
Project(QueryContext* qctx, PlanNode* input, YieldColumns* cols);

Expand Down Expand Up @@ -1503,6 +1507,10 @@ class Traverse final : public GetNeighbors {
trackPrevPath_ = track;
}

Status pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) override;

private:
Traverse(QueryContext* qctx, PlanNode* input, GraphSpaceID space)
: GetNeighbors(qctx, Kind::kTraverse, input, space) {
Expand Down Expand Up @@ -1544,6 +1552,10 @@ class AppendVertices final : public GetVertices {
trackPrevPath_ = track;
}

Status pruneProperties(PropertyTracker& propsUsed,
graph::QueryContext* qctx,
GraphSpaceID spaceID) override;

private:
AppendVertices(QueryContext* qctx, PlanNode* input, GraphSpaceID space)
: GetVertices(qctx,
Expand Down
Loading