Merge branch 'master' into s2-index-params

.github/pull_request_template.md

@@ -12,16 +12,16 @@
 #### Special notes for your reviewer, ex. impact of this fix, etc:
 
 
-#### Additional context:
+#### Additional context/ Design document:
 
 
-#### Checklist：
-- [ ] Documentation affected （Please add the label if documentation needs to be modified.)
-- [ ] Incompatible （If it is incompatible, please describe it and add corresponding label.）
-- [ ] Need to cherry-pick （If need to cherry-pick to some branches, please label the destination version(s).)
+#### Checklist:
+- [ ] Documentation affected (Please add the label if documentation needs to be modified.)
+- [ ] Incompatibility (If it breaks the compatibility, please describe it and add the corresponding label.）
+- [ ] If it's needed to cherry-pick (If cherry-pick to some branches is required, please label the destination version(s).)
 - [ ] Performance impacted: Consumes more CPU/Memory
 
+#### Release notes:
 
-#### Release notes：
-Please confirm whether to reflect in release notes and how to describe:
+Please confirm whether to be reflected in release notes and how to describe:
 >                                                                 `

.linters/cpp/checkKeyword.py

@@ -111,6 +111,10 @@
     'KW_EXPLAIN',
     'KW_UNWIND',
     'KW_CASE',
+    'KW_HOSTS',
+    'KW_ZONE',
+    'KW_ZONES',
+    'KW_RENAME',
 ]
 
 

CMakeLists.txt

@@ -26,6 +26,8 @@ project("Nebula Graph" C CXX)
 option(ENABLE_PACK_ONE "Whether to package into one" ON)
 option(ENABLE_VERBOSE_BISON "Enable Bison to report state" OFF)
 option(ENABLE_PACKAGE_TAR "Enable package artifacts to tar." OFF)
+option(ENABLE_CREATE_GIT_HOOKS "Enable create git hooks." ON)
+option(ENABLE_INCLUDE_WHAT_YOU_USE "Enable include-what-you-use find  nouse include files" OFF)
 
 add_definitions(-DNEBULA_HOME=${CMAKE_SOURCE_DIR})
 
@@ -44,6 +46,7 @@ include(SanitizerConfig)
 include(GitHooksConfig)
 include(GitInfoConfig)
 include(NebulaCustomTargets)
+include(IncludeWhatYouUse)
 
 add_custom_target(
   clang-format
@@ -79,6 +82,7 @@ macro(nebula_add_library name type)
         # hbase_thrift_generator
         parser_target
         wkt_parser_target
+        datetime_parser_target
     )
 endmacro()
 

cmake/nebula/GitHooksConfig.cmake

@@ -1,4 +1,4 @@
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git/")
+if(ENABLE_CREATE_GIT_HOOKS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git/")
     # Create the pre-commit hook every time we run cmake
     message(STATUS "Create the pre-commit hook")
     set(PRE_COMMIT_HOOK ${CMAKE_CURRENT_SOURCE_DIR}/.git/hooks/pre-commit)

cmake/nebula/IncludeWhatYouUse.cmake

@@ -0,0 +1,10 @@
+if(ENABLE_INCLUDE_WHAT_YOU_USE)
+  find_program(INCLUDE_WHAT_YOU_USE include-what-you-use)
+  if(INCLUDE_WHAT_YOU_USE)
+    message("use include-what-you-use")
+    set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "include-what-you-use;-Xiwyu;--transitive_includes_only")
+  else()
+    message(STATUS "iwyu requested but executable not found")
+  endif()
+endif()
+

src/clients/meta/MetaClient.cpp

@@ -780,8 +780,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("Part not existed!");
     case nebula::cpp2::ErrorCode::E_USER_NOT_FOUND:
       return Status::Error("User not existed!");
-    case nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND:
-      return Status::Error("Group not existed!");
+    case nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND:
+      return Status::Error("Machine not existed!");
     case nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND:
       return Status::Error("Zone not existed!");
     case nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND:
@@ -2447,7 +2447,8 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
         metaServerVersion_ = resp.get_meta_version();
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
-      std::move(promise));
+      std::move(promise),
+      true);
   return future;
 }
 
@@ -3032,17 +3033,65 @@ void MetaClient::loadLeader(const std::vector<cpp2::HostItem>& hostItems,
   }
 }
 
-folly::Future<StatusOr<bool>> MetaClient::addZone(std::string zoneName,
-                                                  std::vector<HostAddr> nodes) {
-  cpp2::AddZoneReq req;
-  req.set_zone_name(std::move(zoneName));
-  req.set_nodes(std::move(nodes));
+folly::Future<StatusOr<bool>> MetaClient::addHosts(std::vector<HostAddr> hosts) {
+  cpp2::AddHostsReq req;
+  req.set_hosts(std::move(hosts));
+
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_addHosts(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
+
+folly::Future<StatusOr<bool>> MetaClient::dropHosts(std::vector<HostAddr> hosts) {
+  cpp2::DropHostsReq req;
+  req.set_hosts(std::move(hosts));
 
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_addZone(request); },
+      [](auto client, auto request) { return client->future_dropHosts(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
+
+folly::Future<StatusOr<bool>> MetaClient::mergeZone(std::vector<std::string> zones,
+                                                    std::string zoneName) {
+  cpp2::MergeZoneReq req;
+  req.set_zone_name(zoneName);
+  req.set_zones(zones);
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_mergeZone(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
+
+folly::Future<StatusOr<bool>> MetaClient::renameZone(std::string originalZoneName,
+                                                     std::string zoneName) {
+  cpp2::RenameZoneReq req;
+  req.set_original_zone_name(originalZoneName);
+  req.set_zone_name(zoneName);
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_renameZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
@@ -3066,33 +3115,35 @@ folly::Future<StatusOr<bool>> MetaClient::dropZone(std::string zoneName) {
   return future;
 }
 
-folly::Future<StatusOr<bool>> MetaClient::addHostIntoZone(HostAddr node, std::string zoneName) {
-  cpp2::AddHostIntoZoneReq req;
-  req.set_node(node);
+folly::Future<StatusOr<bool>> MetaClient::splitZone(
+    std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>>) {
+  cpp2::SplitZoneReq req;
   req.set_zone_name(zoneName);
-
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_addHostIntoZone(request); },
+      [](auto client, auto request) { return client->future_splitZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
       std::move(promise));
   return future;
 }
 
-folly::Future<StatusOr<bool>> MetaClient::dropHostFromZone(HostAddr node, std::string zoneName) {
-  cpp2::DropHostFromZoneReq req;
-  req.set_node(node);
+folly::Future<StatusOr<bool>> MetaClient::addHostsIntoZone(std::vector<HostAddr> hosts,
+                                                           std::string zoneName,
+                                                           bool isNew) {
+  cpp2::AddHostsIntoZoneReq req;
+  req.set_hosts(hosts);
   req.set_zone_name(zoneName);
+  req.set_is_new(isNew);
 
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_dropHostFromZone(request); },
+      [](auto client, auto request) { return client->future_addHostsIntoZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
@@ -3500,6 +3551,7 @@ bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId)
 
 Status MetaClient::verifyVersion() {
   auto req = cpp2::VerifyClientVersionReq();
+  req.set_build_version(getOriginVersion());
   req.set_host(options_.localHost_);
   folly::Promise<StatusOr<cpp2::VerifyClientVersionResp>> promise;
   auto future = promise.getFuture();

src/clients/meta/MetaClient.h

@@ -599,13 +599,22 @@ class MetaClient {
 
   StatusOr<LeaderInfo> getLeaderInfo();
 
-  folly::Future<StatusOr<bool>> addZone(std::string zoneName, std::vector<HostAddr> nodes);
+  folly::Future<StatusOr<bool>> addHosts(std::vector<HostAddr> hosts);
+
+  folly::Future<StatusOr<bool>> dropHosts(std::vector<HostAddr> hosts);
+
+  folly::Future<StatusOr<bool>> mergeZone(std::vector<std::string> zones, std::string zoneName);
+
+  folly::Future<StatusOr<bool>> renameZone(std::string originalZoneName, std::string zoneName);
 
   folly::Future<StatusOr<bool>> dropZone(std::string zoneName);
 
-  folly::Future<StatusOr<bool>> addHostIntoZone(HostAddr node, std::string zoneName);
+  folly::Future<StatusOr<bool>> splitZone(
+      std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>> zones);
 
-  folly::Future<StatusOr<bool>> dropHostFromZone(HostAddr node, std::string zoneName);
+  folly::Future<StatusOr<bool>> addHostsIntoZone(std::vector<HostAddr> hosts,
+                                                 std::string zoneName,
+                                                 bool isNew);
 
   folly::Future<StatusOr<std::vector<HostAddr>>> getZone(std::string zoneName);
 

src/clients/storage/StorageClient.cpp

@@ -112,7 +112,8 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
     const CommonRequestParam& param,
     std::vector<cpp2::NewVertex> vertices,
     std::unordered_map<TagID, std::vector<std::string>> propNames,
-    bool ifNotExists) {
+    bool ifNotExists,
+    bool ignoreExistedIndex) {
   auto cbStatus = getIdFromNewVertex(param.space);
   if (!cbStatus.ok()) {
     return folly::makeFuture<StorageRpcResponse<cpp2::ExecResponse>>(
@@ -133,6 +134,7 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
     auto& req = requests[host];
     req.set_space_id(param.space);
     req.set_if_not_exists(ifNotExists);
+    req.set_ignore_existed_index(ignoreExistedIndex);
     req.set_parts(std::move(c.second));
     req.set_prop_names(propNames);
     req.set_common(common);
@@ -149,7 +151,8 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
 StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addEdges(const CommonRequestParam& param,
                                                                  std::vector<cpp2::NewEdge> edges,
                                                                  std::vector<std::string> propNames,
-                                                                 bool ifNotExists) {
+                                                                 bool ifNotExists,
+                                                                 bool ignoreExistedIndex) {
   auto cbStatus = getIdFromNewEdge(param.space);
   if (!cbStatus.ok()) {
     return folly::makeFuture<StorageRpcResponse<cpp2::ExecResponse>>(
@@ -170,6 +173,7 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addEdges(const CommonReq
     auto& req = requests[host];
     req.set_space_id(param.space);
     req.set_if_not_exists(ifNotExists);
+    req.set_ignore_existed_index(ignoreExistedIndex);
     req.set_parts(std::move(c.second));
     req.set_prop_names(propNames);
     req.set_common(common);
@@ -558,15 +562,15 @@ StorageRpcRespFuture<cpp2::GetNeighborsResponse> StorageClient::lookupAndTravers
       });
 }
 
-StorageRpcRespFuture<cpp2::ScanEdgeResponse> StorageClient::scanEdge(
+StorageRpcRespFuture<cpp2::ScanResponse> StorageClient::scanEdge(
     const CommonRequestParam& param,
-    const cpp2::EdgeProp& edgeProp,
+    const std::vector<cpp2::EdgeProp>& edgeProp,
     int64_t limit,
     const Expression* filter) {
   std::unordered_map<HostAddr, cpp2::ScanEdgeRequest> requests;
   auto status = getHostPartsWithCursor(param.space);
   if (!status.ok()) {
-    return folly::makeFuture<StorageRpcResponse<cpp2::ScanEdgeResponse>>(
+    return folly::makeFuture<StorageRpcResponse<cpp2::ScanResponse>>(
         std::runtime_error(status.status().toString()));
   }
   auto& clusters = status.value();
@@ -589,15 +593,15 @@ StorageRpcRespFuture<cpp2::ScanEdgeResponse> StorageClient::scanEdge(
                             const cpp2::ScanEdgeRequest& r) { return client->future_scanEdge(r); });
 }
 
-StorageRpcRespFuture<cpp2::ScanVertexResponse> StorageClient::scanVertex(
+StorageRpcRespFuture<cpp2::ScanResponse> StorageClient::scanVertex(
     const CommonRequestParam& param,
     const std::vector<cpp2::VertexProp>& vertexProp,
     int64_t limit,
     const Expression* filter) {
   std::unordered_map<HostAddr, cpp2::ScanVertexRequest> requests;
   auto status = getHostPartsWithCursor(param.space);
   if (!status.ok()) {
-    return folly::makeFuture<StorageRpcResponse<cpp2::ScanVertexResponse>>(
+    return folly::makeFuture<StorageRpcResponse<cpp2::ScanResponse>>(
         std::runtime_error(status.status().toString()));
   }
   auto& clusters = status.value();

src/clients/storage/StorageClient.h

@@ -81,12 +81,14 @@ class StorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsyncCli
       const CommonRequestParam& param,
       std::vector<cpp2::NewVertex> vertices,
       std::unordered_map<TagID, std::vector<std::string>> propNames,
-      bool ifNotExists);
+      bool ifNotExists,
+      bool ignoreExistedIndex);
 
   StorageRpcRespFuture<cpp2::ExecResponse> addEdges(const CommonRequestParam& param,
                                                     std::vector<cpp2::NewEdge> edges,
                                                     std::vector<std::string> propNames,
-                                                    bool ifNotExists);
+                                                    bool ifNotExists,
+                                                    bool ignoreExistedIndex);
 
   StorageRpcRespFuture<cpp2::ExecResponse> deleteEdges(const CommonRequestParam& param,
                                                        std::vector<storage::cpp2::EdgeKey> edges);
@@ -129,12 +131,12 @@ class StorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsyncCli
   StorageRpcRespFuture<cpp2::GetNeighborsResponse> lookupAndTraverse(
       const CommonRequestParam& param, cpp2::IndexSpec indexSpec, cpp2::TraverseSpec traverseSpec);
 
-  StorageRpcRespFuture<cpp2::ScanEdgeResponse> scanEdge(const CommonRequestParam& param,
-                                                        const cpp2::EdgeProp& vertexProp,
-                                                        int64_t limit,
-                                                        const Expression* filter);
+  StorageRpcRespFuture<cpp2::ScanResponse> scanEdge(const CommonRequestParam& param,
+                                                    const std::vector<cpp2::EdgeProp>& vertexProp,
+                                                    int64_t limit,
+                                                    const Expression* filter);
 
-  StorageRpcRespFuture<cpp2::ScanVertexResponse> scanVertex(
+  StorageRpcRespFuture<cpp2::ScanResponse> scanVertex(
       const CommonRequestParam& param,
       const std::vector<cpp2::VertexProp>& vertexProp,
       int64_t limit,

src/clients/storage/StorageClientBase-inl.h

@@ -242,7 +242,6 @@ void StorageClientBase<ClientType>::getResponseImpl(
         auto client = clientsMan_->client(host, evb, false, FLAGS_storage_client_timeout_ms);
         auto spaceId = request.second.get_space_id();
         auto partsId = getReqPartsId(request.second);
-        LOG(INFO) << "Send request to storage " << host;
         remoteFunc(client.get(), request.second)
             .via(evb)
             .thenValue([spaceId, pro, this](Response&& resp) mutable {
@@ -357,7 +356,6 @@ StorageClientBase<ClientType>::getHostPartsWithCursor(GraphSpaceID spaceId) cons
 
   // TODO support cursor
   cpp2::ScanCursor c;
-  c.set_has_next(false);
   auto parts = status.value();
   for (auto partId = 1; partId <= parts; partId++) {
     auto leader = getLeader(spaceId, partId);

src/codec/RowReaderV2.cpp

@@ -176,6 +176,17 @@ Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept {
       dt.microsec = microsec;
       return dt;
     }
+    case PropertyType::DURATION: {
+      Duration d;
+      memcpy(reinterpret_cast<void*>(&d.seconds), &data_[offset], sizeof(int64_t));
+      memcpy(reinterpret_cast<void*>(&d.microseconds),
+             &data_[offset + sizeof(int64_t)],
+             sizeof(int32_t));
+      memcpy(reinterpret_cast<void*>(&d.months),
+             &data_[offset + sizeof(int64_t) + sizeof(int32_t)],
+             sizeof(int32_t));
+      return d;
+    }
     case PropertyType::GEOGRAPHY: {
       int32_t strOffset;
       int32_t strLen;