diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 21880d788ae..7576b0f9649 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -3116,102 +3116,6 @@ folly::Future>> MetaClient::listZones() { return future; } -folly::Future> MetaClient::addGroup(std::string groupName, - std::vector zoneNames) { - cpp2::AddGroupReq req; - req.set_group_name(std::move(groupName)); - req.set_zone_names(std::move(zoneNames)); - - folly::Promise> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_addGroup(request); }, - [](cpp2::ExecResp&& resp) -> bool { - return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; - }, - std::move(promise)); - return future; -} - -folly::Future> MetaClient::dropGroup(std::string groupName) { - cpp2::DropGroupReq req; - req.set_group_name(std::move(groupName)); - - folly::Promise> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_dropGroup(request); }, - [](cpp2::ExecResp&& resp) -> bool { - return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; - }, - std::move(promise)); - return future; -} - -folly::Future> MetaClient::addZoneIntoGroup(std::string zoneName, - std::string groupName) { - cpp2::AddZoneIntoGroupReq req; - req.set_zone_name(zoneName); - req.set_group_name(groupName); - - folly::Promise> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_addZoneIntoGroup(request); }, - [](cpp2::ExecResp&& resp) -> bool { - return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; - }, - std::move(promise)); - return future; -} - -folly::Future> MetaClient::dropZoneFromGroup(std::string zoneName, - std::string groupName) { - cpp2::DropZoneFromGroupReq req; - req.set_zone_name(zoneName); - req.set_group_name(groupName); - - folly::Promise> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_dropZoneFromGroup(request); }, - [](cpp2::ExecResp&& resp) -> bool { - return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; - }, - std::move(promise)); - return future; -} - -folly::Future>> MetaClient::getGroup(std::string groupName) { - cpp2::GetGroupReq req; - req.set_group_name(std::move(groupName)); - - folly::Promise>> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_getGroup(request); }, - [](cpp2::GetGroupResp&& resp) -> decltype(auto) { return resp.get_zone_names(); }, - std::move(promise)); - return future; -} - -folly::Future>> MetaClient::listGroups() { - cpp2::ListGroupsReq req; - folly::Promise>> promise; - auto future = promise.getFuture(); - getResponse( - std::move(req), - [](auto client, auto request) { return client->future_listGroups(request); }, - [](cpp2::ListGroupsResp&& resp) -> decltype(auto) { return resp.get_groups(); }, - std::move(promise)); - return future; -} - folly::Future> MetaClient::getStats(GraphSpaceID spaceId) { cpp2::GetStatsReq req; req.set_space_id(spaceId); diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 4e19f2e79ee..0bb794c6178 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -607,18 +607,6 @@ class MetaClient { folly::Future>> listZones(); - folly::Future> addGroup(std::string groupName, std::vector zoneNames); - - folly::Future> dropGroup(std::string groupName); - - folly::Future> addZoneIntoGroup(std::string zoneName, std::string groupName); - - folly::Future> dropZoneFromGroup(std::string zoneName, std::string groupName); - - folly::Future>> getGroup(std::string groupName); - - folly::Future>> listGroups(); - Status refreshCache(); folly::Future> getStats(GraphSpaceID spaceId); diff --git a/src/graph/executor/CMakeLists.txt b/src/graph/executor/CMakeLists.txt index 6830ad7ac5e..5bb8064ce3c 100644 --- a/src/graph/executor/CMakeLists.txt +++ b/src/graph/executor/CMakeLists.txt @@ -65,7 +65,6 @@ nebula_add_library( admin/DownloadExecutor.cpp admin/IngestExecutor.cpp admin/ConfigExecutor.cpp - admin/GroupExecutor.cpp admin/ZoneExecutor.cpp admin/ShowTSClientsExecutor.cpp admin/SignInTSServiceExecutor.cpp diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 4c8237b9e1c..447e25acb31 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -24,7 +24,6 @@ #include "graph/executor/admin/DownloadExecutor.h" #include "graph/executor/admin/DropUserExecutor.h" #include "graph/executor/admin/GrantRoleExecutor.h" -#include "graph/executor/admin/GroupExecutor.h" #include "graph/executor/admin/IngestExecutor.h" #include "graph/executor/admin/KillQueryExecutor.h" #include "graph/executor/admin/ListRolesExecutor.h" @@ -435,24 +434,6 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) { case PlanNode::Kind::kSubgraph: { return pool->add(new SubgraphExecutor(node, qctx)); } - case PlanNode::Kind::kAddGroup: { - return pool->add(new AddGroupExecutor(node, qctx)); - } - case PlanNode::Kind::kDropGroup: { - return pool->add(new DropGroupExecutor(node, qctx)); - } - case PlanNode::Kind::kDescribeGroup: { - return pool->add(new DescribeGroupExecutor(node, qctx)); - } - case PlanNode::Kind::kAddZoneIntoGroup: { - return pool->add(new AddZoneIntoGroupExecutor(node, qctx)); - } - case PlanNode::Kind::kDropZoneFromGroup: { - return pool->add(new DropZoneFromGroupExecutor(node, qctx)); - } - case PlanNode::Kind::kShowGroups: { - return pool->add(new ListGroupsExecutor(node, qctx)); - } case PlanNode::Kind::kAddZone: { return pool->add(new AddZoneExecutor(node, qctx)); } diff --git a/src/graph/executor/admin/GroupExecutor.cpp b/src/graph/executor/admin/GroupExecutor.cpp deleted file mode 100644 index 462690f8f27..00000000000 --- a/src/graph/executor/admin/GroupExecutor.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "graph/executor/admin/GroupExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/planner/plan/Admin.h" - -namespace nebula { -namespace graph { - -folly::Future AddGroupExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *agNode = asNode(node()); - return qctx() - ->getMetaClient() - ->addGroup(agNode->groupName(), agNode->zoneNames()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Add Group Failed: " << resp.status(); - return resp.status(); - } - return Status::OK(); - }); -} - -folly::Future DropGroupExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *dgNode = asNode(node()); - return qctx() - ->getMetaClient() - ->dropGroup(dgNode->groupName()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Drop Group Failed: " << resp.status(); - return resp.status(); - } - return Status::OK(); - }); -} - -folly::Future DescribeGroupExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *dgNode = asNode(node()); - return qctx() - ->getMetaClient() - ->getGroup(dgNode->groupName()) - .via(runner()) - .thenValue([this](StatusOr> resp) { - if (!resp.ok()) { - LOG(ERROR) << "Describe Group Failed: " << resp.status(); - return resp.status(); - } - - auto zones = std::move(resp).value(); - DataSet dataSet({"Zone"}); - for (auto &zone : zones) { - Row row({zone}); - dataSet.rows.emplace_back(std::move(row)); - } - return finish(ResultBuilder() - .value(Value(std::move(dataSet))) - .iter(Iterator::Kind::kDefault) - .build()); - }); -} - -folly::Future AddZoneIntoGroupExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *azNode = asNode(node()); - return qctx() - ->getMetaClient() - ->addZoneIntoGroup(azNode->zoneName(), azNode->groupName()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Add Zone Into Group Failed: " << resp.status(); - return resp.status(); - } - return Status::OK(); - }); -} - -folly::Future DropZoneFromGroupExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *dzNode = asNode(node()); - return qctx() - ->getMetaClient() - ->dropZoneFromGroup(dzNode->zoneName(), dzNode->groupName()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Drop Zone From Group Failed: " << resp.status(); - return resp.status(); - } - return Status::OK(); - }); -} - -folly::Future ListGroupsExecutor::execute() { - SCOPED_TIMER(&execTime_); - return qctx()->getMetaClient()->listGroups().via(runner()).thenValue( - [this](StatusOr> resp) { - if (!resp.ok()) { - LOG(ERROR) << "List Groups Failed: " << resp.status(); - return resp.status(); - } - - auto groups = std::move(resp).value(); - DataSet dataSet({"Name", "Zone"}); - for (auto &group : groups) { - for (auto &zone : group.get_zone_names()) { - Row row({*group.group_name_ref(), zone}); - dataSet.rows.emplace_back(std::move(row)); - } - } - return finish(ResultBuilder() - .value(Value(std::move(dataSet))) - .iter(Iterator::Kind::kDefault) - .build()); - }); -} - -} // namespace graph -} // namespace nebula diff --git a/src/graph/executor/admin/GroupExecutor.h b/src/graph/executor/admin/GroupExecutor.h deleted file mode 100644 index 0cb38a3fd7f..00000000000 --- a/src/graph/executor/admin/GroupExecutor.h +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef GRAPH_EXECUTOR_ADMIN_GROUPEXECUTOR_H_ -#define GRAPH_EXECUTOR_ADMIN_GROUPEXECUTOR_H_ - -#include "graph/executor/Executor.h" - -namespace nebula { -namespace graph { - -class AddGroupExecutor final : public Executor { - public: - AddGroupExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("AddGroupExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -class DropGroupExecutor final : public Executor { - public: - DropGroupExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DropGroupExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -class DescribeGroupExecutor final : public Executor { - public: - DescribeGroupExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DescribeGroupExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -class AddZoneIntoGroupExecutor final : public Executor { - public: - AddZoneIntoGroupExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("AddZoneIntoGroupExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -class DropZoneFromGroupExecutor final : public Executor { - public: - DropZoneFromGroupExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DropZoneFromGroupExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -class ListGroupsExecutor final : public Executor { - public: - ListGroupsExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("ListGroupsExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -} // namespace graph -} // namespace nebula - -#endif // GRAPH_EXECUTOR_ADMIN_GROUPEXECUTOR_H_ diff --git a/src/graph/optimizer/rule/TopNRule.cpp b/src/graph/optimizer/rule/TopNRule.cpp index 194ec2788bd..17171fc4872 100644 --- a/src/graph/optimizer/rule/TopNRule.cpp +++ b/src/graph/optimizer/rule/TopNRule.cpp @@ -4,6 +4,7 @@ */ #include "graph/optimizer/rule/TopNRule.h" + #include "graph/optimizer/OptContext.h" #include "graph/optimizer/OptGroup.h" #include "graph/planner/plan/PlanNode.h" diff --git a/src/graph/planner/plan/Admin.h b/src/graph/planner/plan/Admin.h index a8313c5fd68..1f5b4c24215 100644 --- a/src/graph/planner/plan/Admin.h +++ b/src/graph/planner/plan/Admin.h @@ -772,82 +772,6 @@ class ShowCollation final : public SingleDependencyNode { : SingleDependencyNode(qctx, Kind::kShowCollation, input) {} }; -class AddGroup final : public SingleDependencyNode { - public: - static AddGroup* make(QueryContext* qctx, - PlanNode* input, - std::string groupName, - std::vector zoneNames) { - return qctx->objPool()->add( - new AddGroup(qctx, input, std::move(groupName), std::move(zoneNames))); - } - - const std::string& groupName() const { return groupName_; } - - const std::vector& zoneNames() const { return zoneNames_; } - - private: - AddGroup(QueryContext* qctx, - PlanNode* input, - std::string groupName, - std::vector zoneNames) - : SingleDependencyNode(qctx, Kind::kAddGroup, input) { - groupName_ = std::move(groupName); - zoneNames_ = std::move(zoneNames); - } - - private: - std::string groupName_; - std::vector zoneNames_; -}; - -class DropGroup final : public SingleDependencyNode { - public: - static DropGroup* make(QueryContext* qctx, PlanNode* input, std::string groupName) { - return qctx->objPool()->add(new DropGroup(qctx, input, std::move(groupName))); - } - - const std::string& groupName() const { return groupName_; } - - private: - DropGroup(QueryContext* qctx, PlanNode* input, std::string groupName) - : SingleDependencyNode(qctx, Kind::kDropGroup, input) { - groupName_ = std::move(groupName); - } - - private: - std::string groupName_; -}; - -class DescribeGroup final : public SingleDependencyNode { - public: - static DescribeGroup* make(QueryContext* qctx, PlanNode* input, std::string groupName) { - return qctx->objPool()->add(new DescribeGroup(qctx, input, std::move(groupName))); - } - - const std::string& groupName() const { return groupName_; } - - private: - DescribeGroup(QueryContext* qctx, PlanNode* input, std::string groupName) - : SingleDependencyNode(qctx, Kind::kDescribeGroup, input) { - groupName_ = std::move(groupName); - } - - private: - std::string groupName_; -}; - -class ListGroups final : public SingleDependencyNode { - public: - static ListGroups* make(QueryContext* qctx, PlanNode* input) { - return qctx->objPool()->add(new ListGroups(qctx, input)); - } - - private: - ListGroups(QueryContext* qctx, PlanNode* input) - : SingleDependencyNode(qctx, Kind::kShowGroups, input) {} -}; - class AddHostIntoZone final : public SingleDependencyNode { public: static AddHostIntoZone* make(QueryContext* qctx, @@ -998,72 +922,6 @@ class ListZones final : public SingleDependencyNode { : SingleDependencyNode(qctx, Kind::kShowZones, input) {} }; -class AddZoneIntoGroup final : public SingleDependencyNode { - public: - static AddZoneIntoGroup* make(QueryContext* qctx, - PlanNode* input, - std::string groupName, - std::string zoneName) { - return qctx->objPool()->add( - new AddZoneIntoGroup(qctx, input, std::move(zoneName), std::move(groupName))); - } - - const std::string& zoneName() const { return zoneName_; } - - const std::string& groupName() const { return groupName_; } - - private: - AddZoneIntoGroup(QueryContext* qctx, PlanNode* input, std::string zoneName, std::string groupName) - : SingleDependencyNode(qctx, Kind::kAddZoneIntoGroup, input) { - zoneName_ = std::move(zoneName); - groupName_ = std::move(groupName); - } - - private: - std::string zoneName_; - std::string groupName_; -}; - -class DropZoneFromGroup final : public SingleDependencyNode { - public: - static DropZoneFromGroup* make(QueryContext* qctx, - PlanNode* input, - std::string groupName, - std::string zoneName) { - return qctx->objPool()->add( - new DropZoneFromGroup(qctx, input, std::move(zoneName), std::move(groupName))); - } - - const std::string& zoneName() const { return zoneName_; } - - const std::string& groupName() const { return groupName_; } - - private: - DropZoneFromGroup(QueryContext* qctx, - PlanNode* input, - std::string zoneName, - std::string groupName) - : SingleDependencyNode(qctx, Kind::kDropZoneFromGroup, input) { - zoneName_ = std::move(zoneName); - groupName_ = std::move(groupName); - } - - private: - std::string zoneName_; - std::string groupName_; -}; - -class ShowGroups final : public SingleDependencyNode { - public: - static ShowGroups* make(QueryContext* qctx, PlanNode* input) { - return qctx->objPool()->add(new ShowGroups(qctx, input)); - } - - private: - ShowGroups(QueryContext* qctx, PlanNode* input) - : SingleDependencyNode(qctx, Kind::kShowGroups, input) {} -}; - class ShowZones final : public SingleDependencyNode { public: static ShowZones* make(QueryContext* qctx, PlanNode* input) { diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp index 4b663865702..b1a43a2836c 100644 --- a/src/graph/planner/plan/PlanNode.cpp +++ b/src/graph/planner/plan/PlanNode.cpp @@ -233,28 +233,16 @@ const char* PlanNode::toString(PlanNode::Kind kind) { case Kind::kSubgraph: return "Subgraph"; // Group and Zone - case Kind::kAddGroup: - return "AddGroup"; - case Kind::kDropGroup: - return "DropGroup"; case Kind::kAddZone: return "AddZone"; case Kind::kDropZone: return "DropZone"; - case Kind::kDescribeGroup: - return "DescribeGroup"; - case Kind::kAddZoneIntoGroup: - return "AddZoneIntoGroup"; - case Kind::kDropZoneFromGroup: - return "DropZoneFromGroup"; case Kind::kDescribeZone: return "DescribeZone"; case Kind::kAddHostIntoZone: return "AddHostIntoZone"; case Kind::kDropHostFromZone: return "DropHostFromZone"; - case Kind::kShowGroups: - return "ShowGroups"; case Kind::kShowZones: return "ShowZones"; case Kind::kAddListener: diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h index 3da8520d477..053272ce01b 100644 --- a/src/graph/planner/plan/PlanNode.h +++ b/src/graph/planner/plan/PlanNode.h @@ -146,13 +146,7 @@ class PlanNode { kShowMetaLeader, // zone related - kShowGroups, kShowZones, - kAddGroup, - kDropGroup, - kDescribeGroup, - kAddZoneIntoGroup, - kDropZoneFromGroup, kAddZone, kDropZone, kDescribeZone, diff --git a/src/graph/service/PermissionCheck.cpp b/src/graph/service/PermissionCheck.cpp index 0617e22a8cd..ccc3ebdcf36 100644 --- a/src/graph/service/PermissionCheck.cpp +++ b/src/graph/service/PermissionCheck.cpp @@ -55,12 +55,6 @@ Status PermissionCheck::permissionCheck(ClientSession *session, case Sentence::Kind::kDropSpace: case Sentence::Kind::kCreateSnapshot: case Sentence::Kind::kDropSnapshot: - case Sentence::Kind::kAddGroup: - case Sentence::Kind::kDropGroup: - case Sentence::Kind::kDescribeGroup: - case Sentence::Kind::kListGroups: - case Sentence::Kind::kAddZoneIntoGroup: - case Sentence::Kind::kDropZoneFromGroup: case Sentence::Kind::kAddZone: case Sentence::Kind::kDropZone: case Sentence::Kind::kDescribeZone: diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index aff0c9f14ce..ee3c7913ad0 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -63,6 +63,16 @@ std::vector ExpressionUtils::collectAll( return std::move(visitor).results(); } +bool ExpressionUtils::checkVarExprIfExist(const Expression *expr) { + auto vars = ExpressionUtils::collectAll(expr, {Expression::Kind::kVar}); + for (auto *var : vars) { + if (!static_cast(var)->isInner()) { + return true; + } + } + return false; +} + std::vector ExpressionUtils::findAllStorage(const Expression *expr) { return collectAll(expr, {Expression::Kind::kTagProperty, diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h index 484f2db4127..c6d1fa5b87b 100644 --- a/src/graph/util/ExpressionUtils.h +++ b/src/graph/util/ExpressionUtils.h @@ -44,6 +44,8 @@ class ExpressionUtils { static std::vector collectAll( const Expression* self, const std::unordered_set& expected); + static bool checkVarExprIfExist(const Expression* expr); + static std::vector findAllStorage(const Expression* expr); static std::vector findAllInputVariableProp(const Expression* expr); diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp index 73483e092ac..609479c1348 100644 --- a/src/graph/validator/MaintainValidator.cpp +++ b/src/graph/validator/MaintainValidator.cpp @@ -460,74 +460,6 @@ Status ShowEdgeIndexStatusValidator::toPlan() { return Status::OK(); } -Status AddGroupValidator::validateImpl() { - auto sentence = static_cast(sentence_); - if (*sentence->groupName() == "default") { - return Status::SemanticError("Group default conflict"); - } - return Status::OK(); -} - -Status AddGroupValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = - AddGroup::make(qctx_, nullptr, *sentence->groupName(), sentence->zoneNames()->zoneNames()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status DropGroupValidator::validateImpl() { return Status::OK(); } - -Status DropGroupValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = DropGroup::make(qctx_, nullptr, *sentence->groupName()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status DescribeGroupValidator::validateImpl() { return Status::OK(); } - -Status DescribeGroupValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = DescribeGroup::make(qctx_, nullptr, *sentence->groupName()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status ListGroupsValidator::validateImpl() { return Status::OK(); } - -Status ListGroupsValidator::toPlan() { - auto *doNode = ListGroups::make(qctx_, nullptr); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status AddZoneIntoGroupValidator::validateImpl() { return Status::OK(); } - -Status AddZoneIntoGroupValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = - AddZoneIntoGroup::make(qctx_, nullptr, *sentence->groupName(), *sentence->zoneName()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status DropZoneFromGroupValidator::validateImpl() { return Status::OK(); } - -Status DropZoneFromGroupValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = - DropZoneFromGroup::make(qctx_, nullptr, *sentence->groupName(), *sentence->zoneName()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - Status AddZoneValidator::validateImpl() { return Status::OK(); } Status AddZoneValidator::toPlan() { diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h index 095f4266d4a..865994424ad 100644 --- a/src/graph/validator/MaintainValidator.h +++ b/src/graph/validator/MaintainValidator.h @@ -310,80 +310,6 @@ class ShowEdgeIndexStatusValidator final : public Validator { Status toPlan() override; }; -class AddGroupValidator final : public Validator { - public: - AddGroupValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - -class DropGroupValidator final : public Validator { - public: - DropGroupValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - -class DescribeGroupValidator final : public Validator { - public: - DescribeGroupValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - -class ListGroupsValidator final : public Validator { - public: - ListGroupsValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - -class AddZoneIntoGroupValidator final : public Validator { - public: - AddZoneIntoGroupValidator(Sentence* sentence, QueryContext* context) - : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - -class DropZoneFromGroupValidator final : public Validator { - public: - DropZoneFromGroupValidator(Sentence* sentence, QueryContext* context) - : Validator(sentence, context) { - setNoSpaceRequired(); - } - - private: - Status validateImpl() override; - - Status toPlan() override; -}; - class AddZoneValidator final : public Validator { public: AddZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index 7513a7ac93a..687d4731492 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -202,18 +202,6 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon return std::make_unique(sentence, context); case Sentence::Kind::kLookup: return std::make_unique(sentence, context); - case Sentence::Kind::kAddGroup: - return std::make_unique(sentence, context); - case Sentence::Kind::kDropGroup: - return std::make_unique(sentence, context); - case Sentence::Kind::kDescribeGroup: - return std::make_unique(sentence, context); - case Sentence::Kind::kListGroups: - return std::make_unique(sentence, context); - case Sentence::Kind::kAddZoneIntoGroup: - return std::make_unique(sentence, context); - case Sentence::Kind::kDropZoneFromGroup: - return std::make_unique(sentence, context); case Sentence::Kind::kAddZone: return std::make_unique(sentence, context); case Sentence::Kind::kDropZone: diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 166d738623c..50afb2462fc 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -853,49 +853,6 @@ struct ListZonesResp { 3: list zones, } -struct AddGroupReq { - 1: binary group_name, - 2: list zone_names, -} - -struct DropGroupReq { - 1: binary group_name, -} - -struct AddZoneIntoGroupReq { - 1: binary zone_name, - 2: binary group_name, -} - -struct DropZoneFromGroupReq { - 1: binary zone_name, - 2: binary group_name, -} - -struct GetGroupReq { - 1: binary group_name, -} - -struct GetGroupResp { - 1: common.ErrorCode code, - 2: common.HostAddr leader, - 3: list zone_names, -} - -struct ListGroupsReq { -} - -struct Group { - 1: binary group_name, - 2: list zone_names, -} - -struct ListGroupsResp { - 1: common.ErrorCode code, - 2: common.HostAddr leader, - 3: list groups, -} - enum ListenerType { UNKNOWN = 0x00, ELASTICSEARCH = 0x01, @@ -1226,13 +1183,6 @@ service MetaService { GetZoneResp getZone(1: GetZoneReq req); ListZonesResp listZones(1: ListZonesReq req); - ExecResp addGroup(1: AddGroupReq req); - ExecResp dropGroup(1: DropGroupReq req); - ExecResp addZoneIntoGroup(1: AddZoneIntoGroupReq req); - ExecResp dropZoneFromGroup(1: DropZoneFromGroupReq req); - GetGroupResp getGroup(1: GetGroupReq req); - ListGroupsResp listGroups(1: ListGroupsReq req); - CreateBackupResp createBackup(1: CreateBackupReq req); ExecResp restoreMeta(1: RestoreMetaReq req); ExecResp addListener(1: AddListenerReq req); diff --git a/src/interface/raftex.thrift b/src/interface/raftex.thrift index 8f1a8ea4f47..4c7dde896a9 100644 --- a/src/interface/raftex.thrift +++ b/src/interface/raftex.thrift @@ -50,6 +50,7 @@ struct AskForVoteRequest { 5: TermID term; // Proposed term 6: LogID last_log_id; // The last received log id 7: TermID last_log_term; // The term receiving the last log + 8: bool is_pre_vote; // Is pre vote or not } diff --git a/src/kvstore/Listener.cpp b/src/kvstore/Listener.cpp index f99ad2476e7..65d2e632cbd 100644 --- a/src/kvstore/Listener.cpp +++ b/src/kvstore/Listener.cpp @@ -49,7 +49,7 @@ void Listener::start(std::vector&& peers, bool) { lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); - term_ = proposedTerm_ = lastLogTerm_; + term_ = lastLogTerm_; // Set the quorum number quorum_ = (peers.size() + 1) / 2; @@ -273,10 +273,9 @@ void Listener::resetListener() { reset(); VLOG(1) << folly::sformat( "The listener has been reset : leaderCommitId={}," - "proposedTerm={}, lastLogTerm={}, term={}," + "lastLogTerm={}, term={}," "lastApplyLogId={}", leaderCommitId_, - proposedTerm_, lastLogTerm_, term_, lastApplyLogId_); diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index 17b666488cb..de2aa7692c0 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -211,8 +211,7 @@ RaftPart::RaftPart( executor_(executor), snapshot_(snapshotMan), clientMan_(clientMan), - diskMan_(diskMan), - weight_(1) { + diskMan_(diskMan) { FileBasedWalPolicy policy; policy.fileSize = FLAGS_wal_file_size; policy.bufferSize = FLAGS_wal_buffer_size; @@ -262,7 +261,7 @@ void RaftPart::start(std::vector&& peers, bool asLearner) { lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); - term_ = proposedTerm_ = lastLogTerm_; + term_ = lastLogTerm_; // Set the quorum number quorum_ = (peers.size() + 1) / 2; @@ -384,7 +383,8 @@ void RaftPart::preProcessTransLeader(const HostAddr& target) { self->role_ = Role::CANDIDATE; self->leader_ = HostAddr("", 0); } - self->leaderElection().get(); + // skip prevote for transfer leader + self->leaderElection(false).get(); }); } break; @@ -719,6 +719,7 @@ void RaftPart::appendLogsInternal(AppendLogsIterator iter, TermID termId) { if (!wal_->appendLogs(iter)) { LOG_EVERY_N(WARNING, 100) << idStr_ << "Failed to write into WAL"; res = AppendLogResult::E_WAL_FAILURE; + wal_->rollbackToLog(lastLogId_); break; } lastId = wal_->lastLogId(); @@ -754,6 +755,7 @@ void RaftPart::replicateLogs(folly::EventBase* eb, std::lock_guard g(raftLock_); res = canAppendLogs(currTerm); if (res != AppendLogResult::SUCCEEDED) { + wal_->rollbackToLog(lastLogId_); break; } hosts = hosts_; @@ -943,6 +945,7 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, // Not enough hosts accepted the log, re-try LOG_EVERY_N(WARNING, 100) << idStr_ << "Only " << numSucceeded << " hosts succeeded, Need to try again"; + usleep(1000); replicateLogs(eb, std::move(iter), currTerm, lastLogId, committedId, prevLogTerm, prevLogId); } } @@ -955,7 +958,7 @@ bool RaftPart::needToSendHeartbeat() { bool RaftPart::needToStartElection() { std::lock_guard g(raftLock_); if (status_ == Status::RUNNING && role_ == Role::FOLLOWER && - (lastMsgRecvDur_.elapsedInMSec() >= weight_ * FLAGS_raft_heartbeat_interval_secs * 1000 || + (lastMsgRecvDur_.elapsedInMSec() >= FLAGS_raft_heartbeat_interval_secs * 1000 || isBlindFollower_)) { LOG(INFO) << idStr_ << "Start leader election, reason: lastMsgDur " << lastMsgRecvDur_.elapsedInMSec() << ", term " << term_; @@ -967,7 +970,8 @@ bool RaftPart::needToStartElection() { } bool RaftPart::prepareElectionRequest(cpp2::AskForVoteRequest& req, - std::vector>& hosts) { + std::vector>& hosts, + bool isPreVote) { std::lock_guard g(raftLock_); // Make sure the partition is running @@ -982,14 +986,17 @@ bool RaftPart::prepareElectionRequest(cpp2::AskForVoteRequest& req, return false; } - // Before start a new election, reset the votedAddr - votedAddr_ = HostAddr("", 0); - req.set_space(spaceId_); req.set_part(partId_); req.set_candidate_addr(addr_.host); req.set_candidate_port(addr_.port); - req.set_term(++proposedTerm_); // Bump up the proposed term + // Use term_ + 1 to check if peers would vote for me in prevote. + // Only increase the term when prevote succeeeded. + if (isPreVote) { + req.set_term(term_ + 1); + } else { + req.set_term(++term_); + } req.set_last_log_id(lastLogId_); req.set_last_log_term(lastLogTerm_); @@ -998,62 +1005,73 @@ bool RaftPart::prepareElectionRequest(cpp2::AskForVoteRequest& req, return true; } -typename RaftPart::Role RaftPart::processElectionResponses( - const RaftPart::ElectionResponses& results, - std::vector> hosts, - TermID proposedTerm) { +bool RaftPart::processElectionResponses(const RaftPart::ElectionResponses& results, + std::vector> hosts, + TermID proposedTerm, + bool isPreVote) { std::lock_guard g(raftLock_); if (UNLIKELY(status_ == Status::STOPPED)) { LOG(INFO) << idStr_ << "The part has been stopped, skip the request"; - return role_; + return false; } if (UNLIKELY(status_ == Status::STARTING)) { LOG(INFO) << idStr_ << "The partition is still starting"; - return role_; + return false; } if (UNLIKELY(status_ == Status::WAITING_SNAPSHOT)) { LOG(INFO) << idStr_ << "The partition is still waiting snapshot"; - return role_; + return false; } if (role_ != Role::CANDIDATE) { LOG(INFO) << idStr_ << "Partition's role has changed to " << roleStr(role_) << " during the election, so discard the results"; - return role_; + return false; + } + + // term changed during actual leader election + if (!isPreVote && proposedTerm != term_) { + LOG(INFO) << idStr_ << "Partition's term has changed during election, " + << "so just ignore the respsonses, " + << "expected " << proposedTerm << ", actual " << term_; + return false; } size_t numSucceeded = 0; for (auto& r : results) { if (r.second.get_error_code() == cpp2::ErrorCode::SUCCEEDED) { ++numSucceeded; - } else if (r.second.get_error_code() == cpp2::ErrorCode::E_LOG_STALE) { - LOG(INFO) << idStr_ << "My last log id is less than " << hosts[r.first]->address() - << ", double my election interval."; - uint64_t curWeight = weight_.load(); - weight_.store(curWeight * 2); } else { - LOG(ERROR) << idStr_ << "Receive response about askForVote from " << hosts[r.first]->address() - << ", error code is " - << apache::thrift::util::enumNameSafe(r.second.get_error_code()); + LOG(WARNING) << idStr_ << "Receive response about askForVote from " + << hosts[r.first]->address() << ", error code is " + << apache::thrift::util::enumNameSafe(r.second.get_error_code()) + << ", isPreVote = " << isPreVote; } } CHECK(role_ == Role::CANDIDATE); if (numSucceeded >= quorum_) { - LOG(INFO) << idStr_ << "Partition is elected as the new leader for term " << proposedTerm; - term_ = proposedTerm; - role_ = Role::LEADER; - isBlindFollower_ = false; + if (isPreVote) { + LOG(INFO) << idStr_ << "Partition win prevote of term " << proposedTerm; + } else { + LOG(INFO) << idStr_ << "Partition is elected as the new leader for term " << proposedTerm; + term_ = proposedTerm; + role_ = Role::LEADER; + isBlindFollower_ = false; + } + return true; } - return role_; + LOG(INFO) << idStr_ << "Did not get enough votes from election of term " << proposedTerm + << ", isPreVote = " << isPreVote; + return false; } -folly::Future RaftPart::leaderElection() { +folly::Future RaftPart::leaderElection(bool isPreVote) { VLOG(2) << idStr_ << "Start leader election..."; using namespace folly; // NOLINT since the fancy overload of | operator @@ -1065,7 +1083,7 @@ folly::Future RaftPart::leaderElection() { cpp2::AskForVoteRequest voteReq; decltype(hosts_) hosts; - if (!prepareElectionRequest(voteReq, hosts)) { + if (!prepareElectionRequest(voteReq, hosts, isPreVote)) { // Suppose we have three replicas A(leader), B, C, after A crashed, // B, C will begin the election. B win, and send hb, C has gap with B // and need the snapshot from B. Meanwhile C begin the election, @@ -1086,12 +1104,13 @@ folly::Future RaftPart::leaderElection() { << ", term = " << voteReq.get_term() << ", lastLogId = " << voteReq.get_last_log_id() << ", lastLogTerm = " << voteReq.get_last_log_term() << ", candidateIP = " << voteReq.get_candidate_addr() - << ", candidatePort = " << voteReq.get_candidate_port() << ")"; + << ", candidatePort = " << voteReq.get_candidate_port() << ")" + << ", isPreVote = " << isPreVote; auto proposedTerm = voteReq.get_term(); auto resps = ElectionResponses(); if (hosts.empty()) { - auto ret = handleElectionResponses(resps, hosts, proposedTerm); + auto ret = handleElectionResponses(resps, hosts, proposedTerm, isPreVote); inElection_ = false; return ret; } else { @@ -1114,13 +1133,14 @@ folly::Future RaftPart::leaderElection() { return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED && !hosts[idx]->isLearner(); }) .via(executor_.get()) - .then([self = shared_from_this(), pro = std::move(promise), hosts, proposedTerm]( + .then([self = shared_from_this(), pro = std::move(promise), hosts, proposedTerm, isPreVote]( auto&& t) mutable { VLOG(2) << self->idStr_ << "AskForVoteRequest has been sent to all peers, waiting for responses"; CHECK(!t.hasException()); - pro.setValue(self->handleElectionResponses(t.value(), std::move(hosts), proposedTerm)); self->inElection_ = false; + pro.setValue( + self->handleElectionResponses(t.value(), std::move(hosts), proposedTerm, isPreVote)); }); return future; } @@ -1128,50 +1148,30 @@ folly::Future RaftPart::leaderElection() { bool RaftPart::handleElectionResponses(const ElectionResponses& resps, const std::vector>& peers, - TermID proposedTerm) { + TermID proposedTerm, + bool isPreVote) { // Process the responses - switch (processElectionResponses(resps, std::move(peers), proposedTerm)) { - case Role::LEADER: { - // Elected - LOG(INFO) << idStr_ << "The partition is elected as the leader"; - std::vector> hosts; - { - std::lock_guard g(raftLock_); - if (status_ == Status::RUNNING) { - leader_ = addr_; - hosts = hosts_; - bgWorkers_->addTask( - [self = shared_from_this(), proposedTerm] { self->onElected(proposedTerm); }); - lastMsgAcceptedTime_ = 0; - } - weight_ = 1; - commitInThisTerm_ = false; - } - // reset host can't be executed with raftLock_, otherwise it may encounter deadlock - for (auto& host : hosts) { - host->reset(); + auto elected = processElectionResponses(resps, std::move(peers), proposedTerm, isPreVote); + if (!isPreVote && elected) { + std::vector> hosts; + { + std::lock_guard g(raftLock_); + if (status_ == Status::RUNNING) { + leader_ = addr_; + hosts = hosts_; + bgWorkers_->addTask( + [self = shared_from_this(), proposedTerm] { self->onElected(proposedTerm); }); + lastMsgAcceptedTime_ = 0; } - sendHeartbeat(); - return true; - } - case Role::FOLLOWER: { - // Someone was elected - LOG(INFO) << idStr_ << "Someone else was elected"; - return true; + commitInThisTerm_ = false; } - case Role::CANDIDATE: { - // No one has been elected - LOG(INFO) << idStr_ << "No one is elected, continue the election"; - return false; - } - case Role::LEARNER: { - LOG(FATAL) << idStr_ << " Impossible! There must be some bugs!"; - return false; + // reset host can't be executed with raftLock_, otherwise it may encounter deadlock + for (auto& host : hosts) { + host->reset(); } + sendHeartbeat(); } - - LOG(FATAL) << "Should not reach here"; - return false; + return elected; } void RaftPart::statusPolling(int64_t startTime) { @@ -1184,15 +1184,15 @@ void RaftPart::statusPolling(int64_t startTime) { return; } } - size_t delay = FLAGS_raft_heartbeat_interval_secs * 1000 / 3; + size_t delay = FLAGS_raft_heartbeat_interval_secs * 1000 / 3 + +folly::Random::rand32(500); if (needToStartElection()) { - if (leaderElection().get()) { - VLOG(2) << idStr_ << "Stop the election"; + if (leaderElection(true).get() && leaderElection(false).get()) { + // elected as leader } else { // No leader has been elected, need to continue // (After sleeping a random period between [500ms, 2s]) VLOG(2) << idStr_ << "Wait for a while and continue the leader election"; - delay = (folly::Random::rand32(1500) + 500) * weight_; + delay = (folly::Random::rand32(1500) + 500); } } else if (needToSendHeartbeat()) { VLOG(2) << idStr_ << "Need to send heartbeat"; @@ -1243,7 +1243,8 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, << ": space = " << req.get_space() << ", partition = " << req.get_part() << ", candidateAddr = " << req.get_candidate_addr() << ":" << req.get_candidate_port() << ", term = " << req.get_term() << ", lastLogId = " << req.get_last_log_id() - << ", lastLogTerm = " << req.get_last_log_term(); + << ", lastLogTerm = " << req.get_last_log_term() + << ", isPreVote = " << req.get_is_pre_vote(); std::lock_guard g(raftLock_); @@ -1275,19 +1276,27 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, } auto candidate = HostAddr(req.get_candidate_addr(), req.get_candidate_port()); + auto code = checkPeer(candidate); + if (code != cpp2::ErrorCode::SUCCEEDED) { + resp.set_error_code(code); + return; + } + // Check term id - auto term = term_; - if (req.get_term() <= term) { - LOG(INFO) << idStr_ - << (role_ == Role::CANDIDATE ? "The partition is currently proposing term " - : "The partition currently is on term ") - << term - << ". The term proposed by the candidate is" - " no greater, so it will be rejected"; + if (req.get_term() < term_) { + LOG(INFO) << idStr_ << "The partition currently is on term " << term_ + << ", the term proposed by the candidate is " << req.get_term() + << ", so it will be rejected"; resp.set_error_code(cpp2::ErrorCode::E_TERM_OUT_OF_DATE); return; } + auto oldTerm = term_; + // req.get_term() >= term_, we won't update term in prevote + if (!req.get_is_pre_vote()) { + term_ = req.get_term(); + } + // Check the last term to receive a log if (req.get_last_log_term() < lastLogTerm_) { LOG(INFO) << idStr_ << "The partition's last term to receive a log is " << lastLogTerm_ @@ -1308,27 +1317,31 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, } } - // If we have voted for somebody, we will reject other candidates under the - // proposedTerm. - if (votedAddr_ != HostAddr("", 0)) { - if (proposedTerm_ > req.get_term() || - (proposedTerm_ == req.get_term() && votedAddr_ != candidate)) { - LOG(INFO) << idStr_ << "We have voted " << votedAddr_ << " on term " << proposedTerm_ - << ", so we should reject the candidate " << candidate << " request on term " - << req.get_term(); - resp.set_error_code(cpp2::ErrorCode::E_TERM_OUT_OF_DATE); - return; - } + /* + check if we have voted some one in the candidate's proposed term + 1. if this is a prevote: + * not enough votes: the candidate will trigger another round election + * majority votes: the candidate will start formal election (I'll reject the formal one as well) + 2. if this is a formal election: + * not enough votes: the candidate will trigger another round election + * majority votes: the candidate will be leader + */ + if (votedTerm_ == req.get_term() && votedAddr_ != candidate) { + LOG(INFO) << idStr_ << "We have voted " << votedAddr_ << " on term " << votedTerm_ + << ", so we should reject the candidate " << candidate << " request on term " + << req.get_term(); + resp.set_error_code(cpp2::ErrorCode::E_TERM_OUT_OF_DATE); + return; } - - auto code = checkPeer(candidate); - if (code != cpp2::ErrorCode::SUCCEEDED) { - resp.set_error_code(code); + if (req.get_is_pre_vote()) { + // return succeed if it is prevote, do not change any state + resp.set_error_code(cpp2::ErrorCode::SUCCEEDED); return; } + // Ok, no reason to refuse, we will vote for the candidate - LOG(INFO) << idStr_ << "The partition will vote for the candidate " << candidate; - resp.set_error_code(cpp2::ErrorCode::SUCCEEDED); + LOG(INFO) << idStr_ << "The partition will vote for the candidate " << candidate + << ", isPreVote = " << req.get_is_pre_vote(); // Before change role from leader to follower, check the logs locally. if (role_ == Role::LEADER && wal_->lastLogId() > lastLogId_) { @@ -1337,16 +1350,15 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, wal_->rollbackToLog(lastLogId_); } if (role_ == Role::LEADER) { - bgWorkers_->addTask([self = shared_from_this(), term] { self->onLostLeadership(term); }); + bgWorkers_->addTask([self = shared_from_this(), oldTerm] { self->onLostLeadership(oldTerm); }); } role_ = Role::FOLLOWER; votedAddr_ = candidate; - proposedTerm_ = req.get_term(); + votedTerm_ = req.get_term(); leader_ = HostAddr("", 0); // Reset the last message time lastMsgRecvDur_.reset(); - weight_ = 1; isBlindFollower_ = false; return; } @@ -1561,8 +1573,8 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, template cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { DCHECK(!raftLock_.try_lock()); - auto candidate = HostAddr(req.get_leader_addr(), req.get_leader_port()); - auto code = checkPeer(candidate); + auto peer = HostAddr(req.get_leader_addr(), req.get_leader_port()); + auto code = checkPeer(peer); if (code != cpp2::ErrorCode::SUCCEEDED) { return code; } @@ -1574,29 +1586,22 @@ cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { << ". The local term is " << term_ << ". The remote term is not newer"; return cpp2::ErrorCode::E_TERM_OUT_OF_DATE; } else if (req.get_current_term() > term_) { - // Leader stickiness, no matter the term in Request is larger or not. - // TODO(heng) Maybe we should reconsider the logic - if (leader_ != HostAddr("", 0) && leader_ != candidate && - lastMsgRecvDur_.elapsedInMSec() < FLAGS_raft_heartbeat_interval_secs * 1000) { - LOG_EVERY_N(INFO, 100) << idStr_ << "I believe the leader " << leader_ << " exists. " - << "Refuse to append logs of " << candidate; - return cpp2::ErrorCode::E_WRONG_LEADER; - } + // found new leader with higher term } else { // req.get_current_term() == term_ do { - if (role_ != Role::LEADER && leader_ == HostAddr("", 0)) { - LOG_EVERY_N(INFO, 100) << idStr_ << "I dont know who is leader for current term " << term_ - << ", so accept the candidate " << candidate; + if (UNLIKELY(role_ == Role::LEADER)) { + LOG(ERROR) << idStr_ << "Split brain happens, will follow the new leader " << peer + << " on term " << req.get_current_term(); break; - } - // Same leader - if (role_ != Role::LEADER && candidate == leader_) { - return cpp2::ErrorCode::SUCCEEDED; } else { - LOG_EVERY_N(INFO, 100) << idStr_ << "The local term is same as remote term " << term_ - << ", my role is " << roleStr(role_) << ", reject it!"; - return cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + if (LIKELY(leader_ == peer)) { + // Same leader + return cpp2::ErrorCode::SUCCEEDED; + } else if (UNLIKELY(leader_ == HostAddr("", 0))) { + // I don't know who is the leader, will accept it as new leader + break; + } } } while (false); } @@ -1606,16 +1611,13 @@ cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { TermID oldTerm = term_; // Ok, no reason to refuse, just follow the leader LOG(INFO) << idStr_ << "The current role is " << roleStr(role_) << ". Will follow the new leader " - << req.get_leader_addr() << ":" << req.get_leader_port() - << " [Term: " << req.get_current_term() << "]"; + << peer << " on term " << req.get_current_term(); if (role_ != Role::LEARNER) { role_ = Role::FOLLOWER; } - leader_ = candidate; - term_ = proposedTerm_ = req.get_current_term(); - votedAddr_ = HostAddr("", 0); - weight_ = 1; + leader_ = peer; + term_ = req.get_current_term(); isBlindFollower_ = false; // Before accept the logs from the new leader, check the logs locally. if (wal_->lastLogId() > lastLogId_) { @@ -1735,7 +1737,7 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, committedLogId_ = req.get_committed_log_id(); lastLogId_ = committedLogId_; lastLogTerm_ = req.get_committed_log_term(); - term_ = proposedTerm_ = lastLogTerm_; + term_ = lastLogTerm_; // there should be no wal after state converts to WAITING_SNAPSHOT, the RaftPart has been reset DCHECK_EQ(wal_->firstLogId(), 0); DCHECK_EQ(wal_->lastLogId(), 0); diff --git a/src/kvstore/raftex/RaftPart.h b/src/kvstore/raftex/RaftPart.h index a67fc0123c6..dfcf818a12a 100644 --- a/src/kvstore/raftex/RaftPart.h +++ b/src/kvstore/raftex/RaftPart.h @@ -344,25 +344,27 @@ class RaftPart : public std::enable_shared_from_this { void cleanupSnapshot(); // The method sends out AskForVote request - // Return true if a leader is elected (the leader could be self or others), - // otherwise returns false - folly::Future leaderElection(); + // Return true if I have been granted majority votes on proposedTerm, no matter isPreVote or not + folly::Future leaderElection(bool isPreVote); // The method will fill up the request object and return TRUE // if the election should continue. Otherwise the method will // return FALSE bool prepareElectionRequest(cpp2::AskForVoteRequest& req, - std::vector>& hosts); + std::vector>& hosts, + bool isPreVote); - // return true if elected as the leader, else return false + // Return true if I have been granted majority votes on proposedTerm, no matter isPreVote or not bool handleElectionResponses(const ElectionResponses& resps, const std::vector>& hosts, - TermID proposedTerm); + TermID proposedTerm, + bool isPreVote); - // The method returns the partition's role after the election - Role processElectionResponses(const ElectionResponses& results, + // Return true if I have been granted majority votes on proposedTerm, no matter isPreVote or not + bool processElectionResponses(const ElectionResponses& results, std::vector> hosts, - TermID proposedTerm); + TermID proposedTerm, + bool isPreVote); // Check whether new logs can be appended // Pre-condition: The caller needs to hold the raftLock_ @@ -513,27 +515,15 @@ class RaftPart : public std::enable_shared_from_this { HostAddr leader_; // After voted for somebody, it will not be empty anymore. - // And it will be reset to empty after current election finished. HostAddr votedAddr_; // The current term id // the term id proposed by that candidate TermID term_{0}; - // During normal operation, proposedTerm_ is equal to term_, - // when the partition becomes a candidate, proposedTerm_ will be - // bumped up by 1 every time when sending out the AskForVote - // Request - - // If voted for somebody, the proposeTerm will be reset to the candidate - // propose term. So we could use it to prevent revote if someone else ask for - // vote for current proposedTerm. - - // TODO(heng) We should persist it on the disk in the future - // Otherwise, after restart the whole cluster, maybe the stale - // leader still has the unsend log with larger term, and after other - // replicas elected the new leader, the stale one will not join in the - // Raft group any more. - TermID proposedTerm_{0}; + + // Once we have voted some one in formal election, we will set votedTerm_ and votedAddr_. + // To prevent we have voted more than once in a same term + TermID votedTerm_{0}; // The id and term of the last-sent log LogID lastLogId_{0}; @@ -582,8 +572,6 @@ class RaftPart : public std::enable_shared_from_this { // Used to bypass the stale command int64_t startTimeMs_ = 0; - std::atomic weight_; - std::atomic blocking_{false}; }; diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index f5ecbe7bcd8..a02e1bd3073 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -85,11 +85,6 @@ nebula_add_library( processors/zone/GetZoneProcessor.cpp processors/zone/ListZonesProcessor.cpp processors/zone/UpdateZoneProcessor.cpp - processors/zone/AddGroupProcessor.cpp - processors/zone/DropGroupProcessor.cpp - processors/zone/GetGroupProcessor.cpp - processors/zone/ListGroupsProcessor.cpp - processors/zone/UpdateGroupProcessor.cpp processors/listener/ListenerProcessor.cpp processors/session/SessionManagerProcessor.cpp ) diff --git a/src/meta/MetaServiceHandler.cpp b/src/meta/MetaServiceHandler.cpp index 4e58544af0b..58a26b8c038 100644 --- a/src/meta/MetaServiceHandler.cpp +++ b/src/meta/MetaServiceHandler.cpp @@ -61,15 +61,10 @@ #include "meta/processors/schema/ListTagsProcessor.h" #include "meta/processors/session/SessionManagerProcessor.h" #include "meta/processors/user/AuthenticationProcessor.h" -#include "meta/processors/zone/AddGroupProcessor.h" #include "meta/processors/zone/AddZoneProcessor.h" -#include "meta/processors/zone/DropGroupProcessor.h" #include "meta/processors/zone/DropZoneProcessor.h" -#include "meta/processors/zone/GetGroupProcessor.h" #include "meta/processors/zone/GetZoneProcessor.h" -#include "meta/processors/zone/ListGroupsProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateGroupProcessor.h" #include "meta/processors/zone/UpdateZoneProcessor.h" #define RETURN_FUTURE(processor) \ @@ -455,40 +450,6 @@ folly::Future MetaServiceHandler::future_dropHostFromZone( RETURN_FUTURE(processor); } -folly::Future MetaServiceHandler::future_addGroup(const cpp2::AddGroupReq& req) { - auto* processor = AddGroupProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_dropGroup(const cpp2::DropGroupReq& req) { - auto* processor = DropGroupProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_getGroup( - const cpp2::GetGroupReq& req) { - auto* processor = GetGroupProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_listGroups( - const cpp2::ListGroupsReq& req) { - auto* processor = ListGroupsProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_addZoneIntoGroup( - const cpp2::AddZoneIntoGroupReq& req) { - auto* processor = AddZoneIntoGroupProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_dropZoneFromGroup( - const cpp2::DropZoneFromGroupReq& req) { - auto* processor = DropZoneFromGroupProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - folly::Future MetaServiceHandler::future_addListener( const cpp2::AddListenerReq& req) { auto* processor = AddListenerProcessor::instance(kvstore_); diff --git a/src/meta/MetaServiceHandler.h b/src/meta/MetaServiceHandler.h index 2ea463e7c89..1a66097c08c 100644 --- a/src/meta/MetaServiceHandler.h +++ b/src/meta/MetaServiceHandler.h @@ -190,20 +190,6 @@ class MetaServiceHandler final : public cpp2::MetaServiceSvIf { folly::Future future_dropHostFromZone( const cpp2::DropHostFromZoneReq& req) override; - folly::Future future_addGroup(const cpp2::AddGroupReq& req) override; - - folly::Future future_dropGroup(const cpp2::DropGroupReq& req) override; - - folly::Future future_getGroup(const cpp2::GetGroupReq& req) override; - - folly::Future future_listGroups(const cpp2::ListGroupsReq& req) override; - - folly::Future future_addZoneIntoGroup( - const cpp2::AddZoneIntoGroupReq& req) override; - - folly::Future future_dropZoneFromGroup( - const cpp2::DropZoneFromGroupReq& req) override; - // listener folly::Future future_addListener(const cpp2::AddListenerReq& req) override; diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index 2d15e262d3b..ca2c116e719 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -66,6 +66,10 @@ JobManager::~JobManager() { shutDown(); } nebula::cpp2::ErrorCode JobManager::handleRemainingJobs() { std::unique_ptr iter; auto retCode = kvStore_->prefix(kDefaultSpaceId, kDefaultPartId, JobUtil::jobPrefix(), &iter); + if (retCode == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + LOG(INFO) << "Not leader, skip reading remaining jobs"; + return nebula::cpp2::ErrorCode::SUCCEEDED; + } if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Can't find jobs, error: " << apache::thrift::util::enumNameSafe(retCode); return retCode; diff --git a/src/meta/processors/zone/AddGroupProcessor.cpp b/src/meta/processors/zone/AddGroupProcessor.cpp deleted file mode 100644 index 316ca7a3a37..00000000000 --- a/src/meta/processors/zone/AddGroupProcessor.cpp +++ /dev/null @@ -1,126 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/AddGroupProcessor.h" - -namespace nebula { -namespace meta { - -void AddGroupProcessor::process(const cpp2::AddGroupReq& req) { - folly::SharedMutex::WriteHolder wHolder(LockUtils::groupLock()); - auto groupName = req.get_group_name(); - auto zoneNames = req.get_zone_names(); - if (zoneNames.empty()) { - LOG(ERROR) << "The zone names should not be empty."; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - std::set zoneSet(zoneNames.begin(), zoneNames.end()); - if (zoneNames.size() != zoneSet.size()) { - LOG(ERROR) << "Conflict zone found in the group."; - handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT); - onFinished(); - return; - } - - // check the zone existed - const auto& prefix = MetaKeyUtils::zonePrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "Get zones failed: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto iter = nebula::value(iterRet).get(); - std::vector zones; - while (iter->valid()) { - auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); - zones.emplace_back(std::move(zoneName)); - iter->next(); - } - - for (auto name = zoneNames.begin(); name != zoneNames.end(); name++) { - if (std::find(zones.begin(), zones.end(), *name) == zones.end()) { - LOG(ERROR) << "Zone: " << *name << " not existed"; - handleErrorCode(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); - onFinished(); - return; - } - } - - auto retCode = checkGroupRedundancy(zoneNames); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(retCode); - onFinished(); - return; - } - - auto groupRet = getGroupId(groupName); - if (nebula::ok(groupRet)) { - LOG(ERROR) << "Group " << groupName << " already existed"; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } else { - retCode = nebula::error(groupRet); - if (retCode != nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND) { - LOG(ERROR) << "Create Group failed, group name " << groupName - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - } - - auto groupIdRet = autoIncrementId(); - if (!nebula::ok(groupIdRet)) { - LOG(ERROR) << "Create Group failed"; - handleErrorCode(nebula::error(groupIdRet)); - onFinished(); - return; - } - - std::vector data; - auto groupId = nebula::value(groupIdRet); - data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), - std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); - - LOG(INFO) << "Create Group: " << groupName; - doSyncPutAndUpdate(std::move(data)); -} - -nebula::cpp2::ErrorCode AddGroupProcessor::checkGroupRedundancy(std::vector zones) { - const auto& prefix = MetaKeyUtils::groupPrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "Get groups failed, error: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - auto iter = nebula::value(iterRet).get(); - - std::sort(zones.begin(), zones.end()); - while (iter->valid()) { - auto groupName = MetaKeyUtils::parseGroupName(iter->key()); - auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); - std::sort(zoneNames.begin(), zoneNames.end()); - if (zones == zoneNames) { - LOG(ERROR) << "Group " << groupName - << " have created, although the zones order maybe not the same"; - return nebula::cpp2::ErrorCode::E_EXISTED; - } - iter->next(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/AddGroupProcessor.h b/src/meta/processors/zone/AddGroupProcessor.h deleted file mode 100644 index 868331f8f5c..00000000000 --- a/src/meta/processors/zone/AddGroupProcessor.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_ADDGROUPPROCESSOR_H -#define META_ADDGROUPPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class AddGroupProcessor : public BaseProcessor { - public: - static AddGroupProcessor* instance(kvstore::KVStore* kvstore) { - return new AddGroupProcessor(kvstore); - } - - void process(const cpp2::AddGroupReq& req); - - private: - explicit AddGroupProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - - nebula::cpp2::ErrorCode checkGroupRedundancy(std::vector zones); -}; - -} // namespace meta -} // namespace nebula - -#endif // META_ADDGROUPPROCESSOR_H diff --git a/src/meta/processors/zone/DropGroupProcessor.cpp b/src/meta/processors/zone/DropGroupProcessor.cpp deleted file mode 100644 index 50f40385fa4..00000000000 --- a/src/meta/processors/zone/DropGroupProcessor.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/DropGroupProcessor.h" - -namespace nebula { -namespace meta { - -void DropGroupProcessor::process(const cpp2::DropGroupReq& req) { - folly::SharedMutex::WriteHolder wHolder(LockUtils::groupLock()); - auto groupName = req.get_group_name(); - auto groupIdRet = getGroupId(groupName); - if (!nebula::ok(groupIdRet)) { - auto retCode = nebula::error(groupIdRet); - if (retCode == nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND) { - LOG(ERROR) << "Drop Group Failed, Group " << groupName << " not found."; - } else { - LOG(ERROR) << "Drop Group Failed, error: " << apache::thrift::util::enumNameSafe(retCode); - } - handleErrorCode(retCode); - onFinished(); - return; - } - - // If any space rely on this group, it should not be dropped. - auto retCode = checkSpaceDependency(groupName); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(retCode); - onFinished(); - return; - } - - std::vector keys; - keys.emplace_back(MetaKeyUtils::indexGroupKey(groupName)); - keys.emplace_back(MetaKeyUtils::groupKey(groupName)); - LOG(INFO) << "Drop Group: " << groupName; - doSyncMultiRemoveAndUpdate(std::move(keys)); -} - -nebula::cpp2::ErrorCode DropGroupProcessor::checkSpaceDependency(const std::string& groupName) { - const auto& prefix = MetaKeyUtils::spacePrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "List spaces failed, error: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - auto iter = nebula::value(iterRet).get(); - - while (iter->valid()) { - auto properties = MetaKeyUtils::parseSpace(iter->val()); - if (properties.group_name_ref().has_value() && *properties.group_name_ref() == groupName) { - LOG(ERROR) << "Space " << properties.get_space_name() << " is bind to the group " - << groupName; - return nebula::cpp2::ErrorCode::E_NOT_DROP; - } - iter->next(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/DropGroupProcessor.h b/src/meta/processors/zone/DropGroupProcessor.h deleted file mode 100644 index 975aee33b61..00000000000 --- a/src/meta/processors/zone/DropGroupProcessor.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_DROPGROUPPROCESSOR_H -#define META_DROPGROUPPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class DropGroupProcessor : public BaseProcessor { - public: - static DropGroupProcessor* instance(kvstore::KVStore* kvstore) { - return new DropGroupProcessor(kvstore); - } - - void process(const cpp2::DropGroupReq& req); - - private: - explicit DropGroupProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - - nebula::cpp2::ErrorCode checkSpaceDependency(const std::string& groupName); -}; - -} // namespace meta -} // namespace nebula - -#endif // META_DROPGROUPPROCESSOR_H diff --git a/src/meta/processors/zone/GetGroupProcessor.cpp b/src/meta/processors/zone/GetGroupProcessor.cpp deleted file mode 100644 index 24fcde38317..00000000000 --- a/src/meta/processors/zone/GetGroupProcessor.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/GetGroupProcessor.h" - -namespace nebula { -namespace meta { - -void GetGroupProcessor::process(const cpp2::GetGroupReq& req) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::groupLock()); - auto& groupName = req.get_group_name(); - auto groupIdRet = getGroupId(groupName); - if (!nebula::ok(groupIdRet)) { - auto retCode = nebula::error(groupIdRet); - if (retCode == nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND) { - LOG(ERROR) << "Get Group Failed, Group " << groupName << " not found."; - } else { - LOG(ERROR) << "Get Group Failed, error: " << apache::thrift::util::enumNameSafe(retCode); - } - handleErrorCode(retCode); - onFinished(); - return; - } - - auto groupKey = MetaKeyUtils::groupKey(groupName); - auto groupValueRet = doGet(std::move(groupKey)); - if (!nebula::ok(groupValueRet)) { - auto retCode = nebula::error(groupValueRet); - LOG(ERROR) << "Get group " << groupName << " failed, error " - << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); - LOG(INFO) << "Get Group: " << groupName << " zone size: " << zoneNames.size(); - resp_.set_zone_names(std::move(zoneNames)); - handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); - onFinished(); -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/GetGroupProcessor.h b/src/meta/processors/zone/GetGroupProcessor.h deleted file mode 100644 index 2129d1923c9..00000000000 --- a/src/meta/processors/zone/GetGroupProcessor.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_GETGROUPPROCESSOR_H -#define META_GETGROUPPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class GetGroupProcessor : public BaseProcessor { - public: - static GetGroupProcessor* instance(kvstore::KVStore* kvstore) { - return new GetGroupProcessor(kvstore); - } - - void process(const cpp2::GetGroupReq& req); - - private: - explicit GetGroupProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -} // namespace meta -} // namespace nebula -#endif // META_GETGROUPPROCESSOR_H diff --git a/src/meta/processors/zone/ListGroupsProcessor.cpp b/src/meta/processors/zone/ListGroupsProcessor.cpp deleted file mode 100644 index 17bbfad38c7..00000000000 --- a/src/meta/processors/zone/ListGroupsProcessor.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/ListGroupsProcessor.h" - -namespace nebula { -namespace meta { - -void ListGroupsProcessor::process(const cpp2::ListGroupsReq&) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::groupLock()); - const auto& prefix = MetaKeyUtils::groupPrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "List groups failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - auto iter = nebula::value(iterRet).get(); - - std::vector groups; - while (iter->valid()) { - auto groupName = MetaKeyUtils::parseGroupName(iter->key()); - auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); - cpp2::Group group; - group.set_group_name(std::move(groupName)); - group.set_zone_names(std::move(zoneNames)); - groups.emplace_back(std::move(group)); - iter->next(); - } - - handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); - resp_.set_groups(std::move(groups)); - onFinished(); -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/ListGroupsProcessor.h b/src/meta/processors/zone/ListGroupsProcessor.h deleted file mode 100644 index 77bf00f293d..00000000000 --- a/src/meta/processors/zone/ListGroupsProcessor.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_LISTGROUPSPROCESSOR_H -#define META_LISTGROUPSPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class ListGroupsProcessor : public BaseProcessor { - public: - static ListGroupsProcessor* instance(kvstore::KVStore* kvstore) { - return new ListGroupsProcessor(kvstore); - } - - void process(const cpp2::ListGroupsReq& req); - - private: - explicit ListGroupsProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -} // namespace meta -} // namespace nebula -#endif // META_LISTGROUPSPROCESSOR_H diff --git a/src/meta/processors/zone/UpdateGroupProcessor.cpp b/src/meta/processors/zone/UpdateGroupProcessor.cpp deleted file mode 100644 index 5b3db6a6b83..00000000000 --- a/src/meta/processors/zone/UpdateGroupProcessor.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/UpdateGroupProcessor.h" - -namespace nebula { -namespace meta { - -void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::groupLock()); - auto groupName = req.get_group_name(); - auto groupIdRet = getGroupId(groupName); - if (!nebula::ok(groupIdRet)) { - auto retCode = nebula::error(groupIdRet); - LOG(ERROR) << "Get Group failed, group " << groupName - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto groupKey = MetaKeyUtils::groupKey(groupName); - auto groupValueRet = doGet(std::move(groupKey)); - if (!nebula::ok(groupValueRet)) { - auto retCode = nebula::error(groupValueRet); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND; - } - LOG(ERROR) << "Get group " << groupName << " failed, error " - << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneName = req.get_zone_name(); - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); - auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); - if (iter != zoneNames.end()) { - LOG(ERROR) << "Zone " << zoneName << " already exist in the group " << groupName; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } - - const auto& zonePrefix = MetaKeyUtils::zonePrefix(); - auto iterRet = doPrefix(zonePrefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "Get zones failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - auto zoneIter = nebula::value(iterRet).get(); - - bool found = false; - while (zoneIter->valid()) { - auto name = MetaKeyUtils::parseZoneName(zoneIter->key()); - if (name == zoneName) { - found = true; - break; - } - zoneIter->next(); - } - - if (!found) { - LOG(ERROR) << "Zone " << zoneName << " not found"; - handleErrorCode(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); - onFinished(); - return; - } - - zoneNames.emplace_back(zoneName); - std::vector data; - data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); - LOG(INFO) << "Add Zone " << zoneName << " Into Group " << groupName; - doSyncPutAndUpdate(std::move(data)); -} - -void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::groupLock()); - auto groupName = req.get_group_name(); - auto groupIdRet = getGroupId(groupName); - if (!nebula::ok(groupIdRet)) { - auto retCode = nebula::error(groupIdRet); - LOG(ERROR) << " Get Group " << groupName - << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto groupKey = MetaKeyUtils::groupKey(groupName); - auto groupValueRet = doGet(groupKey); - if (!nebula::ok(groupValueRet)) { - auto retCode = nebula::error(groupValueRet); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND; - } - LOG(ERROR) << "Get group " << groupName - << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneName = req.get_zone_name(); - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); - auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); - if (iter == zoneNames.end()) { - LOG(ERROR) << "Zone " << zoneName << " not exist in the group " << groupName; - handleErrorCode(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); - onFinished(); - return; - } - - const auto& spacePrefix = MetaKeyUtils::spacePrefix(); - auto spaceRet = doPrefix(spacePrefix); - if (!nebula::ok(spaceRet)) { - auto retCode = nebula::error(spaceRet); - LOG(ERROR) << "List spaces failed, error " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - nebula::cpp2::ErrorCode spaceCode = nebula::cpp2::ErrorCode::SUCCEEDED; - auto spaceIter = nebula::value(spaceRet).get(); - while (spaceIter->valid()) { - auto properties = MetaKeyUtils::parseSpace(spaceIter->val()); - if (properties.group_name_ref().has_value() && *properties.group_name_ref() == groupName) { - LOG(ERROR) << "Space is bind to the group " << *properties.group_name_ref(); - spaceCode = nebula::cpp2::ErrorCode::E_CONFLICT; - } - spaceIter->next(); - } - - if (spaceCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(spaceCode); - onFinished(); - return; - } - - zoneNames.erase(iter); - std::vector data; - data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); - LOG(INFO) << "Drop Zone " << zoneName << " From Group " << groupName; - doSyncPutAndUpdate(std::move(data)); -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/UpdateGroupProcessor.h b/src/meta/processors/zone/UpdateGroupProcessor.h deleted file mode 100644 index a0367f8a898..00000000000 --- a/src/meta/processors/zone/UpdateGroupProcessor.h +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_UPDATEGROUPPROCESSOR_H -#define META_UPDATEGROUPPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class AddZoneIntoGroupProcessor : public BaseProcessor { - public: - static AddZoneIntoGroupProcessor* instance(kvstore::KVStore* kvstore) { - return new AddZoneIntoGroupProcessor(kvstore); - } - - void process(const cpp2::AddZoneIntoGroupReq& req); - - private: - explicit AddZoneIntoGroupProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -class DropZoneFromGroupProcessor : public BaseProcessor { - public: - static DropZoneFromGroupProcessor* instance(kvstore::KVStore* kvstore) { - return new DropZoneFromGroupProcessor(kvstore); - } - - void process(const cpp2::DropZoneFromGroupReq& req); - - private: - explicit DropZoneFromGroupProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -} // namespace meta -} // namespace nebula -#endif // META_UPDATEGROUPPROCESSOR_H diff --git a/src/meta/test/GroupZoneTest.cpp b/src/meta/test/GroupZoneTest.cpp index 10af4a722b2..564cb1f7955 100644 --- a/src/meta/test/GroupZoneTest.cpp +++ b/src/meta/test/GroupZoneTest.cpp @@ -9,15 +9,10 @@ #include "common/fs/TempDir.h" #include "meta/processors/parts/CreateSpaceProcessor.h" #include "meta/processors/parts/DropSpaceProcessor.h" -#include "meta/processors/zone/AddGroupProcessor.h" #include "meta/processors/zone/AddZoneProcessor.h" -#include "meta/processors/zone/DropGroupProcessor.h" #include "meta/processors/zone/DropZoneProcessor.h" -#include "meta/processors/zone/GetGroupProcessor.h" #include "meta/processors/zone/GetZoneProcessor.h" -#include "meta/processors/zone/ListGroupsProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateGroupProcessor.h" #include "meta/processors/zone/UpdateZoneProcessor.h" #include "meta/test/TestUtils.h" @@ -280,54 +275,6 @@ TEST(GroupAndZoneTest, GroupAndZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, resp.get_code()); } -// Add Group -{ - cpp2::AddGroupReq req; - req.set_group_name("group_0"); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space"); - properties.set_partition_num(12); - properties.set_replica_factor(3); - properties.set_group_name("group_0"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop host from zone -{ - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"12", 12}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_CONFLICT, resp.get_code()); -} -{ - cpp2::DropSpaceReq req; - req.set_space_name("space"); - req.set_if_exists(false); - auto* processor = DropSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} { cpp2::DropHostFromZoneReq req; req.set_zone_name("zone_0"); @@ -339,255 +286,6 @@ TEST(GroupAndZoneTest, GroupAndZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } -// Add Group which zone not exist -{ - LOG(INFO) << "Add Group which zone not exist"; - cpp2::AddGroupReq req; - req.set_group_name("group_zone_not_exist"); - std::vector zones = {"zone_0", "zone_1", "zone_4"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// Group already existed -{ - cpp2::AddGroupReq req; - req.set_group_name("group_1"); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Group already existed although the order is different -{ - cpp2::AddGroupReq req; - req.set_group_name("group_1"); - std::vector zones = {"zone_2", "zone_1", "zone_0"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Add Group with empty zone name list -{ - cpp2::AddGroupReq req; - req.set_group_name("group_0"); - std::vector zones = {}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -// Add Group with duplicate zone name -{ - cpp2::AddGroupReq req; - req.set_group_name("group_0"); - std::vector zones = {"zone_0", "zone_0", "zone_2"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_CONFLICT, resp.get_code()); -} -// Add Group name already existed -{ - cpp2::AddGroupReq req; - req.set_group_name("group_0"); - std::vector zones = {"zone_0", "zone_1"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -{ - cpp2::AddGroupReq req; - req.set_group_name("group_1"); - std::vector zones = {"zone_0", "zone_1"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Get Group -{ - cpp2::GetGroupReq req; - req.set_group_name("group_0"); - auto* processor = GetGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(3, resp.get_zone_names().size()); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - ASSERT_EQ(zones, resp.get_zone_names()); -} -// Get Group which is not exist -{ - cpp2::GetGroupReq req; - req.set_group_name("group_not_exist"); - auto* processor = GetGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND, resp.get_code()); -} -// List Groups -{ - cpp2::ListGroupsReq req; - auto* processor = ListGroupsProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(2, resp.get_groups().size()); - ASSERT_EQ("group_0", resp.get_groups()[0].get_group_name()); - ASSERT_EQ("group_1", resp.get_groups()[1].get_group_name()); -} -{ - std::vector nodes; - for (int32_t i = 9; i < 12; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_3"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Add zone into group -{ - cpp2::AddZoneIntoGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_3"); - auto* processor = AddZoneIntoGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Add zone into group which group not exist -{ - cpp2::AddZoneIntoGroupReq req; - req.set_group_name("group_not_exist"); - req.set_zone_name("zone_0"); - auto* processor = AddZoneIntoGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND, resp.get_code()); -} -// Add zone into group which zone already exist -{ - cpp2::AddZoneIntoGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_0"); - auto* processor = AddZoneIntoGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Add zone into group which zone not exist -{ - cpp2::AddZoneIntoGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_not_exist"); - auto* processor = AddZoneIntoGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// Drop zone from group -{ - cpp2::DropZoneFromGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_3"); - auto* processor = DropZoneFromGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop zone from group which group not exist -{ - cpp2::DropZoneFromGroupReq req; - req.set_group_name("group_not_exist"); - req.set_zone_name("zone_0"); - auto* processor = DropZoneFromGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND, resp.get_code()); -} -// Drop zone from group which zone not exist -{ - cpp2::DropZoneFromGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_not_exist"); - auto* processor = DropZoneFromGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// Drop Group -{ - cpp2::DropGroupReq req; - req.set_group_name("group_0"); - auto* processor = DropGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop Group which is not exist -{ - cpp2::DropGroupReq req; - req.set_group_name("group_0"); - auto* processor = DropGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND, resp.get_code()); -} -// Drop Zone belong to a group -{ - cpp2::DropZoneReq req; - req.set_zone_name("zone_0"); - auto* processor = DropZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_NOT_DROP, resp.get_code()); -} -{ - cpp2::DropGroupReq req; - req.set_group_name("group_1"); - auto* processor = DropGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} // Drop Zone { cpp2::DropZoneReq req; diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index a9e6d6f4723..1a8c6a9318f 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -397,22 +397,6 @@ TEST(MetaClientTest, SpaceWithGroupTest) { ASSERT_TRUE(result.ok()); ASSERT_EQ(5, result.value().size()); } -// Add Group -{ - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - auto result = client->addGroup("group_0", std::move(zones)).get(); - ASSERT_TRUE(result.ok()); -} -{ - std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; - auto result = client->addGroup("group_1", std::move(zones)).get(); - ASSERT_TRUE(result.ok()); -} -{ - auto result = client->listGroups().get(); - ASSERT_TRUE(result.ok()); - ASSERT_EQ(2, result.value().size()); -} } // namespace meta // Create Space without Group { @@ -426,79 +410,6 @@ TEST(MetaClientTest, SpaceWithGroupTest) { ret = client->createSpace(spaceDesc, true).get(); ASSERT_TRUE(ret.ok()) << ret.status(); } -// Create Space on group_0, replica factor is equal with zone size -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("space_on_group_0_3"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(3); - spaceDesc.set_group_name("group_0"); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); -} -// Drop Group should failed -{ - auto result = client->dropGroup("group_0").get(); - ASSERT_FALSE(result.ok()); -} -// Create Space on group_0, replica factor is less than zone size -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("space_on_group_0_1"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(1); - spaceDesc.set_group_name("group_0"); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); -} -// Create Space on group_0, replica factor is larger than zone size -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("space_on_group_0_4"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(4); - spaceDesc.set_group_name("group_0"); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); -} -{ - auto result = client->addZoneIntoGroup("zone_3", "group_0").get(); - ASSERT_TRUE(result.ok()); -} -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("space_on_group_0_4"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(4); - spaceDesc.set_group_name("group_0"); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); -} -// Create Space on a group which is not exist -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("space_on_group_not_exist"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(4); - spaceDesc.set_group_name("group_not_exist"); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_FALSE(ret.ok()) << ret.status(); -} } // namespace nebula TEST(MetaClientTest, TagTest) { @@ -1317,100 +1228,6 @@ TEST(MetaClientTest, GroupAndZoneTest) { auto result = client->dropHostFromZone(node, "zone_0").get(); ASSERT_FALSE(result.ok()); } - // Add Group - { - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - auto result = client->addGroup("group_0", std::move(zones)).get(); - ASSERT_TRUE(result.ok()); - } - // Add Group with empty zone name list - { - std::vector zones = {}; - auto result = client->addGroup("group_0", std::move(zones)).get(); - ASSERT_FALSE(result.ok()); - } - // Add Group with duplicate zone name - { - std::vector zones = {"zone_0", "zone_0", "zone_2"}; - auto result = client->addGroup("group_0", std::move(zones)).get(); - ASSERT_FALSE(result.ok()); - } - // Add Group already existed - { - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - auto result = client->addGroup("group_0", std::move(zones)).get(); - ASSERT_FALSE(result.ok()); - } - { - std::vector zones = {"zone_1", "zone_2"}; - auto result = client->addGroup("group_1", std::move(zones)).get(); - ASSERT_TRUE(result.ok()); - } - // Get Group - { - auto result = client->getGroup("group_0").get(); - ASSERT_TRUE(result.ok()); - } - // Get Group which is not exist - { - auto result = client->getGroup("group_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // List Groups - { - auto result = client->listGroups().get(); - ASSERT_TRUE(result.ok()); - } - { - std::vector nodes = {{"9", 9}, {"10", 10}, {"11", 11}}; - auto result = client->addZone("zone_3", nodes).get(); - ASSERT_TRUE(result.ok()); - } - // Add zone into group - { - auto result = client->addZoneIntoGroup("zone_3", "group_0").get(); - ASSERT_TRUE(result.ok()); - } - // Add zone into group which group not exist - { - auto result = client->addZoneIntoGroup("zone_0", "group_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // Add zone into group which zone already exist - { - auto result = client->addZoneIntoGroup("zone_0", "group_0").get(); - ASSERT_FALSE(result.ok()); - } - // Add zone into group which zone not exist - { - auto result = client->addZoneIntoGroup("zone_not_exist", "group_0").get(); - ASSERT_FALSE(result.ok()); - } - // Drop zone from group - { - auto result = client->dropZoneFromGroup("zone_3", "group_0").get(); - ASSERT_TRUE(result.ok()); - } - // Drop zone from group which group not exist - { - auto result = client->dropZoneFromGroup("zone_0", "group_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // Drop zone from group which zone not exist - { - auto result = client->dropZoneFromGroup("zone_not_exist", "group_0").get(); - ASSERT_FALSE(result.ok()); - } - // Drop Group - { - auto result = client->dropGroup("group_0").get(); - ASSERT_TRUE(result.ok()); - } - // Drop Group which is not exist - { - auto result = client->dropGroup("group_0").get(); - ASSERT_FALSE(result.ok()); - } // Drop Zone { auto result = client->dropZone("zone_0").get(); diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 49e252d53d4..f42f5d4a5d9 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -30,12 +30,8 @@ #include "meta/processors/schema/ListEdgesProcessor.h" #include "meta/processors/schema/ListTagsProcessor.h" #include "meta/processors/session/SessionManagerProcessor.h" -#include "meta/processors/zone/AddGroupProcessor.h" #include "meta/processors/zone/AddZoneProcessor.h" -#include "meta/processors/zone/DropGroupProcessor.h" -#include "meta/processors/zone/ListGroupsProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateGroupProcessor.h" #include "meta/processors/zone/UpdateZoneProcessor.h" #include "meta/test/TestUtils.h" @@ -467,298 +463,6 @@ TEST(ProcessorTest, SpaceTest) { } } -TEST(ProcessorTest, SpaceWithGroupTest) { - fs::TempDir rootPath("/tmp/SpaceWithGroupTest.XXXXXX"); - std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); - std::vector addresses; - for (int32_t i = 0; i <= 10; i++) { - addresses.emplace_back(std::to_string(i), i); - } - TestUtils::createSomeHosts(kv.get(), std::move(addresses)); - - // Add Zones - {{std::vector nodes; - for (int32_t i = 0; i < 2; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 2; i < 4; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_1"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 4; i < 6; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_2"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 6; i < 8; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_3"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 8; i < 10; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_4"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -} // namespace meta -// List Zones -{ - cpp2::ListZonesReq req; - auto* processor = ListZonesProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(5, resp.get_zones().size()); - ASSERT_EQ("zone_0", resp.get_zones()[0].get_zone_name()); - ASSERT_EQ("zone_1", resp.get_zones()[1].get_zone_name()); - ASSERT_EQ("zone_2", resp.get_zones()[2].get_zone_name()); - ASSERT_EQ("zone_3", resp.get_zones()[3].get_zone_name()); - ASSERT_EQ("zone_4", resp.get_zones()[4].get_zone_name()); -} - -// Add Group -{ - cpp2::AddGroupReq req; - req.set_group_name("group_0"); - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - cpp2::AddGroupReq req; - req.set_group_name("group_1"); - std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// List Groups -{ - cpp2::ListGroupsReq req; - auto* processor = ListGroupsProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(2, resp.get_groups().size()); - ASSERT_EQ("group_0", resp.get_groups()[0].get_group_name()); - ASSERT_EQ("group_1", resp.get_groups()[1].get_group_name()); -} - -// Create Space without Group -{ - cpp2::SpaceDesc properties; - properties.set_space_name("default_space"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Create Space on group_0, replica factor is equal with zone size -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_group_0_3"); - properties.set_partition_num(9); - properties.set_replica_factor(3); - properties.set_group_name("group_0"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop Group should failed -{ - cpp2::DropGroupReq req; - req.set_group_name("group_0"); - auto* processor = DropGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_NOT_DROP, resp.get_code()); -} -// Create Space on group_0, replica factor is less than zone size -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_group_0_1"); - properties.set_partition_num(9); - properties.set_replica_factor(1); - properties.set_group_name("group_0"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Create Space on group_0, replica factor is larger than zone size -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_group_0_4"); - properties.set_partition_num(9); - properties.set_replica_factor(4); - properties.set_group_name("group_0"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -{ - cpp2::AddZoneIntoGroupReq req; - req.set_group_name("group_0"); - req.set_zone_name("zone_3"); - auto* processor = AddZoneIntoGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_group_0_4"); - properties.set_partition_num(9); - properties.set_replica_factor(4); - properties.set_group_name("group_0"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Create Space on a group which is not exist -{ - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_group_not_exist"); - properties.set_partition_num(9); - properties.set_replica_factor(4); - properties.set_group_name("group_not_exist"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND, resp.get_code()); -} -// Create space on empty zone -{ - { - std::vector nodes = {HostAddr("10", 10)}; - cpp2::AddZoneReq req; - req.set_zone_name("zone_5"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - { - cpp2::AddGroupReq req; - req.set_group_name("group_2"); - std::vector zones = {"zone_5"}; - req.set_zone_names(std::move(zones)); - auto* processor = AddGroupProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - { - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_5"); - HostAddr node{"10", 10}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - { - cpp2::SpaceDesc properties; - properties.set_space_name("space_on_empty_hosts"); - properties.set_partition_num(1); - properties.set_replica_factor(1); - properties.set_group_name("group_2"); - cpp2::CreateSpaceReq req; - req.set_properties(std::move(properties)); - auto* processor = CreateSpaceProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); - } -} -} // namespace nebula - TEST(ProcessorTest, CreateTagTest) { fs::TempDir rootPath("/tmp/CreateTagTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index 4b9d5d41028..1a90d068192 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -358,16 +358,6 @@ std::string ShowCreateEdgeIndexSentence::toString() const { return folly::stringPrintf("SHOW CREATE EDGE INDEX %s", indexName_.get()->c_str()); } -std::string AddGroupSentence::toString() const { - std::string buf; - buf.reserve(64); - buf += "ADD GROUP "; - buf += *groupName_; - buf += " "; - buf += zoneNames_->toString(); - return buf; -} - std::string AddZoneSentence::toString() const { std::string buf; buf.reserve(128); @@ -377,31 +367,16 @@ std::string AddZoneSentence::toString() const { return buf; } -std::string DropGroupSentence::toString() const { - return folly::stringPrintf("DROP GROUP %s", groupName_.get()->c_str()); -} - std::string DropZoneSentence::toString() const { return folly::stringPrintf("DROP ZONE %s", zoneName_.get()->c_str()); } -std::string DescribeGroupSentence::toString() const { - return folly::stringPrintf("DESCRIBE GROUP %s", groupName_.get()->c_str()); -} - std::string DescribeZoneSentence::toString() const { return folly::stringPrintf("DESCRIBE ZONE %s", zoneName_.get()->c_str()); } -std::string ListGroupsSentence::toString() const { return folly::stringPrintf("SHOW GROUPS"); } - std::string ListZonesSentence::toString() const { return folly::stringPrintf("SHOW ZONES"); } -std::string AddZoneIntoGroupSentence::toString() const { - return folly::stringPrintf( - "Add Zone %s Into Group %s", zoneName_.get()->c_str(), groupName_.get()->c_str()); -} - std::string AddHostIntoZoneSentence::toString() const { std::string buf; buf.reserve(64); @@ -412,11 +387,6 @@ std::string AddHostIntoZoneSentence::toString() const { return buf; } -std::string DropZoneFromGroupSentence::toString() const { - return folly::stringPrintf( - "Drop Zone %s From Group %s", zoneName_.get()->c_str(), groupName_.get()->c_str()); -} - std::string DropHostFromZoneSentence::toString() const { std::string buf; buf.reserve(64); diff --git a/src/parser/MaintainSentences.h b/src/parser/MaintainSentences.h index 6f7de95bb26..64a8363300c 100644 --- a/src/parser/MaintainSentences.h +++ b/src/parser/MaintainSentences.h @@ -784,25 +784,6 @@ class ZoneNameList final { std::vector> zones_; }; -class AddGroupSentence : public Sentence { - public: - explicit AddGroupSentence(std::string *groupName, ZoneNameList *zoneNames) { - groupName_.reset(groupName); - zoneNames_.reset(zoneNames); - kind_ = Kind::kAddGroup; - } - - std::string toString() const override; - - const std::string *groupName() const { return groupName_.get(); } - - const ZoneNameList *zoneNames() const { return zoneNames_.get(); } - - private: - std::unique_ptr groupName_; - std::unique_ptr zoneNames_; -}; - class AddZoneSentence : public Sentence { public: explicit AddZoneSentence(std::string *zoneName, HostList *hosts) { @@ -822,21 +803,6 @@ class AddZoneSentence : public Sentence { std::unique_ptr hosts_; }; -class DropGroupSentence : public Sentence { - public: - explicit DropGroupSentence(std::string *groupName) { - groupName_.reset(groupName); - kind_ = Kind::kDropGroup; - } - - std::string toString() const override; - - const std::string *groupName() const { return groupName_.get(); } - - private: - std::unique_ptr groupName_; -}; - class DropZoneSentence : public Sentence { public: explicit DropZoneSentence(std::string *zoneName) { @@ -852,21 +818,6 @@ class DropZoneSentence : public Sentence { std::unique_ptr zoneName_; }; -class DescribeGroupSentence : public Sentence { - public: - explicit DescribeGroupSentence(std::string *groupName) { - groupName_.reset(groupName); - kind_ = Kind::kDescribeGroup; - } - - std::string toString() const override; - - const std::string *groupName() const { return groupName_.get(); } - - private: - std::unique_ptr groupName_; -}; - class DescribeZoneSentence : public Sentence { public: explicit DescribeZoneSentence(std::string *zoneName) { @@ -882,13 +833,6 @@ class DescribeZoneSentence : public Sentence { std::unique_ptr zoneName_; }; -class ListGroupsSentence : public Sentence { - public: - ListGroupsSentence() { kind_ = Kind::kListGroups; } - - std::string toString() const override; -}; - class ListZonesSentence : public Sentence { public: ListZonesSentence() { kind_ = Kind::kListZones; } @@ -896,25 +840,6 @@ class ListZonesSentence : public Sentence { std::string toString() const override; }; -class AddZoneIntoGroupSentence : public Sentence { - public: - AddZoneIntoGroupSentence(std::string *zoneName, std::string *groupName) { - zoneName_.reset(zoneName); - groupName_.reset(groupName); - kind_ = Kind::kAddZoneIntoGroup; - } - - const std::string *zoneName() const { return zoneName_.get(); } - - const std::string *groupName() const { return groupName_.get(); } - - std::string toString() const override; - - private: - std::unique_ptr zoneName_; - std::unique_ptr groupName_; -}; - class AddHostIntoZoneSentence : public Sentence { public: explicit AddHostIntoZoneSentence(HostAddr *address, std::string *zoneName) { @@ -934,25 +859,6 @@ class AddHostIntoZoneSentence : public Sentence { std::unique_ptr address_; }; -class DropZoneFromGroupSentence : public Sentence { - public: - DropZoneFromGroupSentence(std::string *zoneName, std::string *groupName) { - zoneName_.reset(zoneName); - groupName_.reset(groupName); - kind_ = Kind::kDropZoneFromGroup; - } - - const std::string *zoneName() const { return zoneName_.get(); } - - const std::string *groupName() const { return groupName_.get(); } - - std::string toString() const override; - - private: - std::unique_ptr zoneName_; - std::unique_ptr groupName_; -}; - class DropHostFromZoneSentence : public Sentence { public: explicit DropHostFromZoneSentence(HostAddr *address, std::string *zoneName) { diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index c28d7a7b0a2..e943f8ed24a 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -112,12 +112,6 @@ class Sentence { kAdminJob, kAdminShowJobs, kGetSubgraph, - kAddGroup, - kDropGroup, - kDescribeGroup, - kListGroups, - kAddZoneIntoGroup, - kDropZoneFromGroup, kAddZone, kDropZone, kDescribeZone, diff --git a/src/parser/parser.yy b/src/parser/parser.yy index b503f23a512..3a5aab1f84b 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -120,7 +120,6 @@ static constexpr size_t kCommentLengthLimit = 256; nebula::GroupClause *group_clause; nebula::HostList *host_list; nebula::HostAddr *host_item; - nebula::ZoneNameList *zone_name_list; std::vector *integer_list; nebula::InBoundClause *in_bound_clause; nebula::OutBoundClause *out_bound_clause; @@ -333,7 +332,6 @@ static constexpr size_t kCommentLengthLimit = 256; %type column_spec %type column_spec_list %type column_name_list -%type zone_name_list %type role_type_clause %type acl_item_clause @@ -354,8 +352,6 @@ static constexpr size_t kCommentLengthLimit = 256; %type drop_tag_index_sentence drop_edge_index_sentence drop_fulltext_index_sentence %type describe_tag_index_sentence describe_edge_index_sentence %type rebuild_tag_index_sentence rebuild_edge_index_sentence rebuild_fulltext_index_sentence -%type add_group_sentence drop_group_sentence desc_group_sentence -%type add_zone_into_group_sentence drop_zone_from_group_sentence %type add_zone_sentence drop_zone_sentence desc_zone_sentence %type add_host_into_zone_sentence drop_host_from_zone_sentence %type create_snapshot_sentence drop_snapshot_sentence @@ -1448,9 +1444,16 @@ yield_column delete $3; } | expression { + if (graph::ExpressionUtils::checkVarExprIfExist($1)) { + throw nebula::GraphParser::syntax_error(@1, "Direct output of variable is prohibited"); + } $$ = new YieldColumn($1); } | expression KW_AS name_label { + if (graph::ExpressionUtils::checkVarExprIfExist($1)) { + delete $3; + throw nebula::GraphParser::syntax_error(@1, "Direct output of variable is prohibited"); + } $$ = new YieldColumn($1, *$3); delete $3; } @@ -2641,38 +2644,6 @@ rebuild_fulltext_index_sentence meta::cpp2::AdminCmd::REBUILD_FULLTEXT_INDEX); } -add_group_sentence - : KW_ADD KW_GROUP name_label zone_name_list { - $$ = new AddGroupSentence($3, $4); - } - ; - -drop_group_sentence - : KW_DROP KW_GROUP name_label { - $$ = new DropGroupSentence($3); - } - ; - -desc_group_sentence - : KW_DESCRIBE KW_GROUP name_label { - $$ = new DescribeGroupSentence($3); - } - | KW_DESC KW_GROUP name_label { - $$ = new DescribeGroupSentence($3); - } - ; - -add_zone_into_group_sentence - : KW_ADD KW_ZONE name_label KW_INTO KW_GROUP name_label { - $$ = new AddZoneIntoGroupSentence($3, $6); - } - ; - -drop_zone_from_group_sentence - : KW_DROP KW_ZONE name_label KW_FROM KW_GROUP name_label { - $$ = new DropZoneFromGroupSentence($3, $6); - } - ; add_zone_sentence : KW_ADD KW_ZONE name_label host_list { @@ -3198,9 +3169,6 @@ show_sentence | KW_SHOW KW_COLLATION { $$ = new ShowCollationSentence(); } - | KW_SHOW KW_GROUPS { - $$ = new ListGroupsSentence(); - } | KW_SHOW KW_ZONES { $$ = new ListZonesSentence(); } @@ -3269,17 +3237,6 @@ show_config_item } ; -zone_name_list - : name_label { - $$ = new ZoneNameList(); - $$->addZone($1); - } - | zone_name_list COMMA name_label { - $$ = $1; - $$->addZone($3); - } - ; - create_space_sentence : KW_CREATE KW_SPACE opt_if_not_exists name_label { auto sentence = new CreateSpaceSentence($4, $3); @@ -3638,11 +3595,6 @@ maintain_sentence | rebuild_tag_index_sentence { $$ = $1; } | rebuild_edge_index_sentence { $$ = $1; } | rebuild_fulltext_index_sentence { $$ = $1; } - | add_group_sentence { $$ = $1; } - | drop_group_sentence { $$ = $1; } - | desc_group_sentence { $$ = $1; } - | add_zone_into_group_sentence { $$ = $1; } - | drop_zone_from_group_sentence { $$ = $1; } | add_zone_sentence { $$ = $1; } | drop_zone_sentence { $$ = $1; } | desc_zone_sentence { $$ = $1; } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 5267339a4c2..f1249614e67 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -2710,11 +2710,6 @@ TEST_F(ParserTest, MatchListSubscriptRange) { } TEST_F(ParserTest, Zone) { - { - std::string query = "SHOW GROUPS"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } { std::string query = "SHOW ZONES"; auto result = parse(query); @@ -2750,36 +2745,6 @@ TEST_F(ParserTest, Zone) { auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } - { - std::string query = "ADD GROUP group_0 zone_0,zone_1,zone_2"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "ADD ZONE zone_3 INTO GROUP group_0"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DROP ZONE zone_3 FROM GROUP group_0"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DESC GROUP group_0"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DESCRIBE GROUP group_0"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "DROP GROUP group_0"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } } TEST_F(ParserTest, FullText) { diff --git a/src/storage/context/StorageExpressionContext.h b/src/storage/context/StorageExpressionContext.h index 65b989594a3..38ef72eba39 100644 --- a/src/storage/context/StorageExpressionContext.h +++ b/src/storage/context/StorageExpressionContext.h @@ -147,7 +147,7 @@ class StorageExpressionContext final : public ExpressionContext { size_t vIdLen_; bool isIntId_; - RowReader* reader_; + RowReader* reader_{nullptr}; std::string key_; // tag or edge name std::string name_; diff --git a/src/storage/exec/ScanNode.h b/src/storage/exec/ScanNode.h index 5af7912f758..22425dcb90b 100644 --- a/src/storage/exec/ScanNode.h +++ b/src/storage/exec/ScanNode.h @@ -13,6 +13,8 @@ namespace storage { using Cursor = std::string; +inline bool vTrue(const Value& v) { return v.isBool() && v.getBool(); } + // Node to scan vertices of one partition class ScanVertexPropNode : public QueryNode { public: @@ -23,13 +25,17 @@ class ScanVertexPropNode : public QueryNode { bool enableReadFollower, int64_t limit, std::unordered_map* cursors, - nebula::DataSet* resultDataSet) + nebula::DataSet* resultDataSet, + StorageExpressionContext* expCtx = nullptr, + Expression* filter = nullptr) : context_(context), tagNodes_(std::move(tagNodes)), enableReadFollower_(enableReadFollower), limit_(limit), cursors_(cursors), - resultDataSet_(resultDataSet) { + resultDataSet_(resultDataSet), + expCtx_(expCtx), + filter_(filter) { name_ = "ScanVertexPropNode"; for (std::size_t i = 0; i < tagNodes_.size(); ++i) { tagNodesIndex_.emplace(tagNodes_[i]->tagId(), i); @@ -110,20 +116,36 @@ class ScanVertexPropNode : public QueryNode { })) { for (auto& tagNode : tagNodes_) { ret = tagNode->collectTagPropsIfValid( - [&row](const std::vector* props) -> nebula::cpp2::ErrorCode { + [&row, tagNode = tagNode.get(), this]( + const std::vector* props) -> nebula::cpp2::ErrorCode { for (const auto& prop : *props) { if (prop.returned_) { row.emplace_back(Value()); } + if (prop.filtered_ && expCtx_ != nullptr) { + expCtx_->setTagProp(tagNode->getTagName(), prop.name_, Value()); + } } return nebula::cpp2::ErrorCode::SUCCEEDED; }, - [&row, vIdLen, isIntId]( + [&row, vIdLen, isIntId, tagNode = tagNode.get(), this]( folly::StringPiece key, RowReader* reader, const std::vector* props) -> nebula::cpp2::ErrorCode { - if (!QueryUtils::collectVertexProps(key, vIdLen, isIntId, reader, props, row).ok()) { - return nebula::cpp2::ErrorCode::E_TAG_PROP_NOT_FOUND; + for (const auto& prop : *props) { + if (prop.returned_ || (prop.filtered_ && expCtx_ != nullptr)) { + auto value = QueryUtils::readVertexProp(key, vIdLen, isIntId, reader, prop); + if (!value.ok()) { + return nebula::cpp2::ErrorCode::E_TAG_PROP_NOT_FOUND; + } + if (prop.filtered_ && expCtx_ != nullptr) { + expCtx_->setTagProp(tagNode->getTagName(), prop.name_, value.value()); + } + if (prop.returned_) { + VLOG(2) << "Collect prop " << prop.name_; + row.emplace_back(std::move(value).value()); + } + } } return nebula::cpp2::ErrorCode::SUCCEEDED; }); @@ -131,9 +153,11 @@ class ScanVertexPropNode : public QueryNode { break; } } - if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { + if (ret == nebula::cpp2::ErrorCode::SUCCEEDED && + (filter_ == nullptr || vTrue(filter_->eval(*expCtx_)))) { resultDataSet_->rows.emplace_back(std::move(row)); } + expCtx_->clear(); for (auto& tagNode : tagNodes_) { tagNode->clear(); } @@ -149,6 +173,8 @@ class ScanVertexPropNode : public QueryNode { // cursors for next scan std::unordered_map* cursors_; nebula::DataSet* resultDataSet_; + StorageExpressionContext* expCtx_{nullptr}; + Expression* filter_{nullptr}; }; // Node to scan edge of one partition @@ -161,13 +187,17 @@ class ScanEdgePropNode : public QueryNode { bool enableReadFollower, int64_t limit, std::unordered_map* cursors, - nebula::DataSet* resultDataSet) + nebula::DataSet* resultDataSet, + StorageExpressionContext* expCtx = nullptr, + Expression* filter = nullptr) : context_(context), edgeNodes_(std::move(edgeNodes)), enableReadFollower_(enableReadFollower), limit_(limit), cursors_(cursors), - resultDataSet_(resultDataSet) { + resultDataSet_(resultDataSet), + expCtx_(expCtx), + filter_(filter) { QueryNode::name_ = "ScanEdgePropNode"; for (std::size_t i = 0; i < edgeNodes_.size(); ++i) { edgeNodesIndex_.emplace(edgeNodes_[i]->edgeType(), i); @@ -230,20 +260,36 @@ class ScanEdgePropNode : public QueryNode { nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; for (auto& edgeNode : edgeNodes_) { ret = edgeNode->collectEdgePropsIfValid( - [&row](const std::vector* props) -> nebula::cpp2::ErrorCode { + [&row, edgeNode = edgeNode.get(), this]( + const std::vector* props) -> nebula::cpp2::ErrorCode { for (const auto& prop : *props) { if (prop.returned_) { row.emplace_back(Value()); } + if (prop.filtered_ && expCtx_ != nullptr) { + expCtx_->setEdgeProp(edgeNode->getEdgeName(), prop.name_, Value()); + } } return nebula::cpp2::ErrorCode::SUCCEEDED; }, - [&row, vIdLen, isIntId]( + [&row, vIdLen, isIntId, edgeNode = edgeNode.get(), this]( folly::StringPiece key, RowReader* reader, const std::vector* props) -> nebula::cpp2::ErrorCode { - if (!QueryUtils::collectEdgeProps(key, vIdLen, isIntId, reader, props, row).ok()) { - return nebula::cpp2::ErrorCode::E_EDGE_PROP_NOT_FOUND; + for (const auto& prop : *props) { + if (prop.returned_ || (prop.filtered_ && expCtx_ != nullptr)) { + auto value = QueryUtils::readEdgeProp(key, vIdLen, isIntId, reader, prop); + if (!value.ok()) { + return nebula::cpp2::ErrorCode::E_EDGE_PROP_NOT_FOUND; + } + if (prop.filtered_ && expCtx_ != nullptr) { + expCtx_->setEdgeProp(edgeNode->getEdgeName(), prop.name_, value.value()); + } + if (prop.returned_) { + VLOG(2) << "Collect prop " << prop.name_; + row.emplace_back(std::move(value).value()); + } + } } return nebula::cpp2::ErrorCode::SUCCEEDED; }); @@ -251,9 +297,11 @@ class ScanEdgePropNode : public QueryNode { break; } } - if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { + if (ret == nebula::cpp2::ErrorCode::SUCCEEDED && + (filter_ == nullptr || vTrue(filter_->eval(*expCtx_)))) { resultDataSet_->rows.emplace_back(std::move(row)); } + expCtx_->clear(); for (auto& edgeNode : edgeNodes_) { edgeNode->clear(); } @@ -268,6 +316,8 @@ class ScanEdgePropNode : public QueryNode { // cursors for next scan std::unordered_map* cursors_; nebula::DataSet* resultDataSet_; + StorageExpressionContext* expCtx_{nullptr}; + Expression* filter_{nullptr}; }; } // namespace storage diff --git a/src/storage/query/GetNeighborsProcessor.cpp b/src/storage/query/GetNeighborsProcessor.cpp index 149a7c9018b..5d08b2ab7e3 100644 --- a/src/storage/query/GetNeighborsProcessor.cpp +++ b/src/storage/query/GetNeighborsProcessor.cpp @@ -284,7 +284,13 @@ nebula::cpp2::ErrorCode GetNeighborsProcessor::checkAndBuildContexts( if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { return code; } - code = buildFilter(req); + code = buildFilter(req, [](const cpp2::GetNeighborsRequest& r) -> const std::string* { + if (r.get_traverse_spec().filter_ref().has_value()) { + return r.get_traverse_spec().get_filter(); + } else { + return nullptr; + } + }); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { return code; } diff --git a/src/storage/query/QueryBaseProcessor-inl.h b/src/storage/query/QueryBaseProcessor-inl.h index f472f265576..47b8f48c54f 100644 --- a/src/storage/query/QueryBaseProcessor-inl.h +++ b/src/storage/query/QueryBaseProcessor-inl.h @@ -129,18 +129,18 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::buildYields(const REQ& re } template -nebula::cpp2::ErrorCode QueryBaseProcessor::buildFilter(const REQ& req) { - const auto& traverseSpec = req.get_traverse_spec(); - if (!traverseSpec.filter_ref().has_value()) { +nebula::cpp2::ErrorCode QueryBaseProcessor::buildFilter( + const REQ& req, std::function&& getFilter) { + const auto* filterStr = getFilter(req); + if (filterStr == nullptr) { return nebula::cpp2::ErrorCode::SUCCEEDED; } - const auto& filterStr = *traverseSpec.filter_ref(); auto pool = &this->planContext_->objPool_; // auto v = env_; - if (!filterStr.empty()) { + if (!filterStr->empty()) { // the filter expression **must** return a bool - filter_ = Expression::decode(pool, filterStr); + filter_ = Expression::decode(pool, *filterStr); if (filter_ == nullptr) { return nebula::cpp2::ErrorCode::E_INVALID_FILTER; } @@ -438,8 +438,9 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp(const Expression } return nebula::cpp2::ErrorCode::SUCCEEDED; } + case Expression::Kind::kTagProperty: case Expression::Kind::kSrcProperty: { - auto* sourceExp = static_cast(exp); + auto* sourceExp = static_cast(exp); const auto& tagName = sourceExp->sym(); const auto& propName = sourceExp->prop(); auto tagRet = this->env_->schemaMan_->toTagID(spaceId_, tagName); @@ -458,6 +459,17 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp(const Expression const auto& tagSchema = iter->second.back(); if (propName == kVid || propName == kTag) { + if (returned || filtered) { + addPropContextIfNotExists(tagContext_.propContexts_, + tagContext_.indexMap_, + tagContext_.tagNames_, + tagId, + tagName, + propName, + nullptr, + returned, + filtered); + } return nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -505,6 +517,17 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp(const Expression const auto& edgeSchema = iter->second.back(); if (propName == kSrc || propName == kType || propName == kRank || propName == kDst) { + if (returned || filtered) { + addPropContextIfNotExists(edgeContext_.propContexts_, + edgeContext_.indexMap_, + edgeContext_.edgeNames_, + edgeType, + edgeName, + propName, + nullptr, + returned, + filtered); + } return nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -561,7 +584,6 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp(const Expression case Expression::Kind::kSubscript: case Expression::Kind::kAttribute: case Expression::Kind::kLabelAttribute: - case Expression::Kind::kTagProperty: case Expression::Kind::kVertex: case Expression::Kind::kEdge: case Expression::Kind::kLabel: @@ -597,6 +619,9 @@ void QueryBaseProcessor::addPropContextIfNotExists( bool filtered, const std::pair* statInfo) { auto idxIter = indexMap.find(entryId); + if (idxIter == indexMap.end()) { // for edge type + idxIter = indexMap.find(-entryId); + } if (idxIter == indexMap.end()) { // if no property of tag/edge has been add to propContexts PropContext ctx(propName.c_str(), field, returned, filtered, statInfo); diff --git a/src/storage/query/QueryBaseProcessor.h b/src/storage/query/QueryBaseProcessor.h index 3f75b0c5335..00dd8e97247 100644 --- a/src/storage/query/QueryBaseProcessor.h +++ b/src/storage/query/QueryBaseProcessor.h @@ -154,7 +154,8 @@ class QueryBaseProcessor : public BaseProcessor { // build edgeContexts_ according to return props nebula::cpp2::ErrorCode handleEdgeProps(std::vector& edgeProps); - nebula::cpp2::ErrorCode buildFilter(const REQ& req); + nebula::cpp2::ErrorCode buildFilter( + const REQ& req, std::function&& getFilter); nebula::cpp2::ErrorCode buildYields(const REQ& req); // build ttl info map diff --git a/src/storage/query/ScanEdgeProcessor.cpp b/src/storage/query/ScanEdgeProcessor.cpp index 5da9b6425e6..94a6a03a1b2 100644 --- a/src/storage/query/ScanEdgeProcessor.cpp +++ b/src/storage/query/ScanEdgeProcessor.cpp @@ -64,6 +64,13 @@ nebula::cpp2::ErrorCode ScanEdgeProcessor::checkAndBuildContexts(const cpp2::Sca std::vector returnProps = {*req.return_columns_ref()}; ret = handleEdgeProps(returnProps); buildEdgeColName(returnProps); + ret = buildFilter(req, [](const cpp2::ScanEdgeRequest& r) -> const std::string* { + if (r.filter_ref().has_value()) { + return r.get_filter(); + } else { + return nullptr; + } + }); return ret; } @@ -85,7 +92,8 @@ void ScanEdgeProcessor::onProcessFinished() { StoragePlan ScanEdgeProcessor::buildPlan( RuntimeContext* context, nebula::DataSet* result, - std::unordered_map* cursors) { + std::unordered_map* cursors, + StorageExpressionContext* expCtx) { StoragePlan plan; std::vector> edges; for (const auto& ec : edgeContext_.propContexts_) { @@ -93,7 +101,7 @@ StoragePlan ScanEdgeProcessor::buildPlan( std::make_unique(context, &edgeContext_, ec.first, &ec.second)); } auto output = std::make_unique( - context, std::move(edges), enableReadFollower_, limit_, cursors, result); + context, std::move(edges), enableReadFollower_, limit_, cursors, result, expCtx, filter_); plan.addNode(std::move(output)); return plan; @@ -104,10 +112,11 @@ folly::Future> ScanEdgeProcessor nebula::DataSet* result, std::unordered_map* cursors, PartitionID partId, - Cursor cursor) { + Cursor cursor, + StorageExpressionContext* expCtx) { return folly::via(executor_, - [this, context, result, cursors, partId, input = std::move(cursor)]() { - auto plan = buildPlan(context, result, cursors); + [this, context, result, cursors, partId, input = std::move(cursor), expCtx]() { + auto plan = buildPlan(context, result, cursors, expCtx); auto ret = plan.go(partId, input); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -119,8 +128,9 @@ folly::Future> ScanEdgeProcessor void ScanEdgeProcessor::runInSingleThread(const cpp2::ScanEdgeRequest& req) { contexts_.emplace_back(RuntimeContext(planContext_.get())); + expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; - auto plan = buildPlan(&contexts_.front(), &resultDataSet_, &cursors_); + auto plan = buildPlan(&contexts_.front(), &resultDataSet_, &cursors_, &expCtxs_.front()); for (const auto& partEntry : req.get_parts()) { auto partId = partEntry.first; auto cursor = partEntry.second; @@ -142,6 +152,7 @@ void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); contexts_.emplace_back(RuntimeContext(planContext_.get())); + expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); } size_t i = 0; std::vector>> futures; @@ -150,7 +161,8 @@ void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { &results_[i], &cursorsOfPart_[i], partId, - cursor.get_has_next() ? *cursor.get_next_cursor() : "")); + cursor.get_has_next() ? *cursor.get_next_cursor() : "", + &expCtxs_[i])); i++; } diff --git a/src/storage/query/ScanEdgeProcessor.h b/src/storage/query/ScanEdgeProcessor.h index f1931cd881b..40c5186975d 100644 --- a/src/storage/query/ScanEdgeProcessor.h +++ b/src/storage/query/ScanEdgeProcessor.h @@ -39,14 +39,16 @@ class ScanEdgeProcessor : public QueryBaseProcessor buildPlan(RuntimeContext* context, nebula::DataSet* result, - std::unordered_map* cursors); + std::unordered_map* cursors, + StorageExpressionContext* expCtx); folly::Future> runInExecutor( RuntimeContext* context, nebula::DataSet* result, std::unordered_map* cursors, PartitionID partId, - Cursor cursor); + Cursor cursor, + StorageExpressionContext* expCtx); void runInSingleThread(const cpp2::ScanEdgeRequest& req); @@ -55,6 +57,7 @@ class ScanEdgeProcessor : public QueryBaseProcessor contexts_; + std::vector expCtxs_; std::vector results_; std::vector> cursorsOfPart_; diff --git a/src/storage/query/ScanVertexProcessor.cpp b/src/storage/query/ScanVertexProcessor.cpp index 032ec103660..bb9b3a705ad 100644 --- a/src/storage/query/ScanVertexProcessor.cpp +++ b/src/storage/query/ScanVertexProcessor.cpp @@ -65,6 +65,13 @@ nebula::cpp2::ErrorCode ScanVertexProcessor::checkAndBuildContexts( std::vector returnProps = *req.return_columns_ref(); ret = handleVertexProps(returnProps); buildTagColName(returnProps); + ret = buildFilter(req, [](const cpp2::ScanVertexRequest& r) -> const std::string* { + if (r.filter_ref().has_value()) { + return r.get_filter(); + } else { + return nullptr; + } + }); return ret; } @@ -87,14 +94,15 @@ void ScanVertexProcessor::onProcessFinished() { StoragePlan ScanVertexProcessor::buildPlan( RuntimeContext* context, nebula::DataSet* result, - std::unordered_map* cursors) { + std::unordered_map* cursors, + StorageExpressionContext* expCtx) { StoragePlan plan; std::vector> tags; for (const auto& tc : tagContext_.propContexts_) { tags.emplace_back(std::make_unique(context, &tagContext_, tc.first, &tc.second)); } auto output = std::make_unique( - context, std::move(tags), enableReadFollower_, limit_, cursors, result); + context, std::move(tags), enableReadFollower_, limit_, cursors, result, expCtx, filter_); plan.addNode(std::move(output)); return plan; @@ -105,23 +113,26 @@ folly::Future> ScanVertexProcess nebula::DataSet* result, std::unordered_map* cursorsOfPart, PartitionID partId, - Cursor cursor) { - return folly::via(executor_, - [this, context, result, cursorsOfPart, partId, input = std::move(cursor)]() { - auto plan = buildPlan(context, result, cursorsOfPart); - - auto ret = plan.go(partId, input); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return std::make_pair(ret, partId); - } - return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); - }); + Cursor cursor, + StorageExpressionContext* expCtx) { + return folly::via( + executor_, + [this, context, result, cursorsOfPart, partId, input = std::move(cursor), expCtx]() { + auto plan = buildPlan(context, result, cursorsOfPart, expCtx); + + auto ret = plan.go(partId, input); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return std::make_pair(ret, partId); + } + return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); + }); } void ScanVertexProcessor::runInSingleThread(const cpp2::ScanVertexRequest& req) { contexts_.emplace_back(RuntimeContext(planContext_.get())); + expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; - auto plan = buildPlan(&contexts_.front(), &resultDataSet_, &cursors_); + auto plan = buildPlan(&contexts_.front(), &resultDataSet_, &cursors_, &expCtxs_.front()); for (const auto& partEntry : req.get_parts()) { auto partId = partEntry.first; auto cursor = partEntry.second; @@ -143,6 +154,7 @@ void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); contexts_.emplace_back(RuntimeContext(planContext_.get())); + expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); } size_t i = 0; std::vector>> futures; @@ -151,7 +163,8 @@ void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req &results_[i], &cursorsOfPart_[i], partId, - cursor.get_has_next() ? *cursor.get_next_cursor() : "")); + cursor.get_has_next() ? *cursor.get_next_cursor() : "", + &expCtxs_[i])); i++; } diff --git a/src/storage/query/ScanVertexProcessor.h b/src/storage/query/ScanVertexProcessor.h index 987c77e2edd..39b34aedae9 100644 --- a/src/storage/query/ScanVertexProcessor.h +++ b/src/storage/query/ScanVertexProcessor.h @@ -40,14 +40,16 @@ class ScanVertexProcessor StoragePlan buildPlan(RuntimeContext* context, nebula::DataSet* result, - std::unordered_map* cursors); + std::unordered_map* cursors, + StorageExpressionContext* expCtx); folly::Future> runInExecutor( RuntimeContext* context, nebula::DataSet* result, std::unordered_map* cursors, PartitionID partId, - Cursor cursor); + Cursor cursor, + StorageExpressionContext* expCtx); void runInSingleThread(const cpp2::ScanVertexRequest& req); @@ -57,6 +59,7 @@ class ScanVertexProcessor private: std::vector contexts_; + std::vector expCtxs_; std::vector results_; std::vector> cursorsOfPart_; diff --git a/src/storage/test/ScanEdgeTest.cpp b/src/storage/test/ScanEdgeTest.cpp index 3ed3d41a9dc..381b0df6c33 100644 --- a/src/storage/test/ScanEdgeTest.cpp +++ b/src/storage/test/ScanEdgeTest.cpp @@ -289,6 +289,49 @@ TEST(ScanEdgeTest, LimitTest) { } } +TEST(ScanEdgeTest, FilterTest) { + fs::TempDir rootPath("/tmp/ScanVertexTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto totalParts = cluster.getTotalParts(); + ASSERT_EQ(true, QueryTestUtils::mockVertexData(env, totalParts)); + ASSERT_EQ(true, QueryTestUtils::mockEdgeData(env, totalParts)); + + EdgeType serve = 101; + ObjectPool pool; + + { + LOG(INFO) << "Scan one edge with some properties in one batch"; + constexpr std::size_t limit = 3; + auto edge = std::make_pair( + serve, + std::vector{kSrc, kType, kRank, kDst, "teamName", "startYear", "endYear"}); + auto req = buildRequest({1}, {""}, edge, limit); + Expression* filter = EdgePropertyExpression::make(&pool, "101", kSrc); + filter = RelationalExpression::makeEQ( + &pool, filter, ConstantExpression::make(&pool, "Damian Lillard")); + req.set_filter(filter->encode()); + auto* processor = ScanEdgeProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + + DataSet expected({"101._src", + "101._type", + "101._rank", + "101._dst", + "101.teamName", + "101.startYear", + "101.endYear"}); + expected.emplace_back( + List({"Damian Lillard", 101, 2012, "Trail Blazers", "Trail Blazers", 2012, 2020})); + EXPECT_EQ(*resp.edge_data_ref(), expected); + } +} + } // namespace storage } // namespace nebula diff --git a/src/storage/test/ScanVertexTest.cpp b/src/storage/test/ScanVertexTest.cpp index f582848ad92..ea972dd39e6 100644 --- a/src/storage/test/ScanVertexTest.cpp +++ b/src/storage/test/ScanVertexTest.cpp @@ -437,6 +437,71 @@ TEST(ScanVertexTest, MultipleTagsTest) { } } +TEST(ScanVertexTest, FilterTest) { + fs::TempDir rootPath("/tmp/ScanVertexTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto totalParts = cluster.getTotalParts(); + ASSERT_EQ(true, QueryTestUtils::mockVertexData(env, totalParts)); + ASSERT_EQ(true, QueryTestUtils::mockEdgeData(env, totalParts)); + + TagID player = 1; + TagID team = 2; + ObjectPool pool; + + { + LOG(INFO) << "Scan one tag with some properties in one batch"; + // size_t totalRowCount = 0; + auto playerTag = + std::make_pair(player, std::vector{kVid, kTag, "name", "age", "avgScore"}); + auto teamTag = std::make_pair(team, std::vector{kTag, "name"}); + auto req = buildRequest({1}, {""}, {playerTag, teamTag}); + Expression* filter = TagPropertyExpression::make(&pool, "1", "name"); + filter = + RelationalExpression::makeEQ(&pool, filter, ConstantExpression::make(&pool, "Kobe Bryant")); + req.set_filter(filter->encode()); + auto* processor = ScanVertexProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + nebula::DataSet expect( + {"_vid", "1._vid", "1._tag", "1.name", "1.age", "1.avgScore", "2._tag", "2.name"}); + expect.emplace_back(List( + {"Kobe Bryant", "Kobe Bryant", 1, "Kobe Bryant", 41, 25, Value::kEmpty, Value::kEmpty})); + EXPECT_EQ(expect, *resp.vertex_data_ref()); + } + { + LOG(INFO) << "Scan one tag with some properties in one batch"; + // size_t totalRowCount = 0; + auto playerTag = + std::make_pair(player, std::vector{kVid, kTag, "name", "age", "avgScore"}); + auto teamTag = std::make_pair(team, std::vector{kTag, "name"}); + auto req = buildRequest({1}, {""}, {playerTag, teamTag}); + Expression* filter = TagPropertyExpression::make(&pool, "1", "name"); + filter = + RelationalExpression::makeEQ(&pool, filter, ConstantExpression::make(&pool, "Kobe Bryant")); + filter = LogicalExpression::makeAnd( + &pool, + filter, + UnaryExpression::makeIsEmpty(&pool, TagPropertyExpression::make(&pool, "2", "name"))); + req.set_filter(filter->encode()); + auto* processor = ScanVertexProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + nebula::DataSet expect( + {"_vid", "1._vid", "1._tag", "1.name", "1.age", "1.avgScore", "2._tag", "2.name"}); + expect.emplace_back(List( + {"Kobe Bryant", "Kobe Bryant", 1, "Kobe Bryant", 41, 25, Value::kEmpty, Value::kEmpty})); + EXPECT_EQ(expect, *resp.vertex_data_ref()); + } +} + } // namespace storage } // namespace nebula diff --git a/src/storage/test/UpdateEdgeTest.cpp b/src/storage/test/UpdateEdgeTest.cpp index 86d66825c69..0dc6b115229 100644 --- a/src/storage/test/UpdateEdgeTest.cpp +++ b/src/storage/test/UpdateEdgeTest.cpp @@ -291,7 +291,8 @@ TEST(UpdateEdgeTest, No_Filter_Test) { auto resp = std::move(f).get(); LOG(INFO) << "Check the results..."; - EXPECT_EQ(0, (*resp.result_ref()).failed_parts.size()); + EXPECT_EQ(0, (*resp.result_ref()).failed_parts.size()) + << apache::thrift::util::enumNameSafe((*resp.result_ref()).failed_parts.front().get_code()); EXPECT_EQ(9, (*resp.props_ref()).colNames.size()); EXPECT_EQ("_inserted", (*resp.props_ref()).colNames[0]); EXPECT_EQ("101.playerName", (*resp.props_ref()).colNames[1]); diff --git a/tests/maintain/test_zone.py b/tests/maintain/test_zone.py index bfa504d6bc6..9e74d876af9 100644 --- a/tests/maintain/test_zone.py +++ b/tests/maintain/test_zone.py @@ -36,39 +36,6 @@ def test_zone(self): resp = self.client.execute('SHOW ZONES') self.check_resp_succeeded(resp) - # Add Group - resp = self.client.execute('ADD GROUP group_0 zone_0') - self.check_resp_succeeded(resp) - - resp = self.client.execute('ADD GROUP default zone_0') - self.check_resp_failed(resp) - - # Get Group - resp = self.client.execute('DESC GROUP group_0') - self.check_resp_succeeded(resp) - - resp = self.client.execute('DESCRIBE GROUP group_0') - self.check_resp_succeeded(resp) - - # Get Group which is not exist - resp = self.client.execute('DESC GROUP group_not_exist') - self.check_resp_failed(resp) - - resp = self.client.execute('DESCRIBE GROUP group_not_exist') - self.check_resp_failed(resp) - - # SHOW Groups - resp = self.client.execute('SHOW GROUPS') - self.check_resp_succeeded(resp) - - # Drop Group - resp = self.client.execute('DROP GROUP group_0') - self.check_resp_succeeded(resp) - - # Drop Group which is not exist - resp = self.client.execute('DROP GROUP group_0') - self.check_resp_failed(resp) - # Drop Zone resp = self.client.execute('DROP ZONE zone_0') self.check_resp_succeeded(resp) diff --git a/tests/tck/features/bugfix/VariableExpression.feature b/tests/tck/features/bugfix/VariableExpression.feature new file mode 100644 index 00000000000..d95560d681f --- /dev/null +++ b/tests/tck/features/bugfix/VariableExpression.feature @@ -0,0 +1,26 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Variable usage + + Background: + Given a graph with space named "nba" + + Scenario: disable yield $var + When executing query: + """ + $var = yield 1;$var2 = yield 3;yield $var1 + $var2 + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `$var1 + $var2' + When executing query: + """ + $var=go from "Tim Duncan" over like yield like._dst as dst;yield $var + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `$var' + Then drop the used space + When executing query: + """ + $var=go from "Tim Duncan" over like yield like._dst as dst;yield $var[0][0] + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `$var[0][0]' diff --git a/tests/tck/features/go/GroupbyLimit.IntVid.feature b/tests/tck/features/go/GroupbyLimit.IntVid.feature index 86e457460f6..384c12928a8 100644 --- a/tests/tck/features/go/GroupbyLimit.IntVid.feature +++ b/tests/tck/features/go/GroupbyLimit.IntVid.feature @@ -15,7 +15,7 @@ Feature: Groupby & limit Sentence """ GO FROM hash("Marco Belinelli") OVER serve YIELD $$.team.name AS name | GROUP BY $-.start_year YIELD COUNT($var) """ - Then a SemanticError should be raised at runtime: + Then a SyntaxError should be raised at runtime: Scenario: Syntax test4 When executing query: diff --git a/tests/tck/features/go/GroupbyLimit.feature b/tests/tck/features/go/GroupbyLimit.feature index 2a282a2b7b0..7706c27c6b2 100644 --- a/tests/tck/features/go/GroupbyLimit.feature +++ b/tests/tck/features/go/GroupbyLimit.feature @@ -22,7 +22,7 @@ Feature: Groupby & limit Sentence """ GO FROM "Marco Belinelli" OVER serve YIELD $$.team.name AS name | GROUP BY $-.start_year YIELD COUNT($var) """ - Then a SemanticError should be raised at runtime: + Then a SyntaxError should be raised at runtime: Scenario: Syntax test4 When executing query: