Merge branch 'master' into fix

.github/actions/tagname-action/action.yml

@@ -13,7 +13,9 @@ runs:
     - id: tag
       run: |
         tag=$(echo ${{ github.ref }} | rev | cut -d/ -f1 | rev)
-        tagnum=$(echo $tag |sed 's/^v//')
+        tagnum=$(echo $tag | sed 's/^v//')
+        majorver=$(echo $tag | cut -d '.' -f 1)
         echo "::set-output name=tag::$tag"
         echo "::set-output name=tagnum::$tagnum"
+        echo "::set-output name=majorver::$majorver"
       shell: bash

.github/workflows/nightly.yml

@@ -39,7 +39,7 @@ jobs:
           echo "::set-output name=subdir::$subdir"
       - uses: actions/upload-artifact@v1
         with:
-          name: ${{ matrix.os }}-v2-nightly
+          name: ${{ matrix.os }}-nightly
           path: pkg-build/cpack_output
       - uses: ./.github/actions/upload-to-oss-action
         with:

.github/workflows/pull_request.yml

@@ -94,6 +94,7 @@ jobs:
                   -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/gcc \
                   -DCMAKE_BUILD_TYPE=Debug \
                   -DENABLE_TESTING=on \
+                  -DENABLE_COVERAGE=on \
                   -B build
               echo "::set-output name=j::10"
               echo "::set-output name=t::10"
@@ -126,7 +127,24 @@ jobs:
         timeout-minutes: 20
       - name: Setup cluster
         run: |
-          make up
+          case ${{ matrix.compiler }} in
+          gcc-*)
+            case ${{ matrix.os }} in
+            centos7)
+              # normal cluster
+              make up
+              ;;
+            ubuntu2004)
+              # ssl cluster
+              make ENABLE_SSL=true CA_SIGNED=true up
+              ;;
+            esac
+            ;;
+          clang-*)
+            # graph ssl only cluster
+            make ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
+            ;;
+          esac
         working-directory: tests/
         timeout-minutes: 2
       - name: Pytest
@@ -144,6 +162,11 @@ jobs:
           make RM_DIR=false down
         working-directory: tests/
         timeout-minutes: 2
+      - name: coverage
+        if: ${{ matrix.compiler == 'gcc-9.2' && matrix.os == 'ubuntu2004' }}
+        run: |
+            ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include --exclude=/opt/vesoft --exclude scanner.lex
+            bash <(curl -s https://codecov.io/bash) -Z -f fastcov.info
       - name: Sanitizer
         if: ${{ always() }}
         run: |

.github/workflows/release.yml

@@ -62,6 +62,14 @@ jobs:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/tagname-action
         id: tagname
+      - id: docker
+        run: |
+          majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.')
+          tag=""
+          if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then
+            tag="vesoft/nebula-${{ matrix.service }}:latest"
+          fi
+          echo "::set-output name=tag::$tag"
       - uses: docker/setup-buildx-action@v1
       - uses: docker/login-action@v1
         with:
@@ -71,7 +79,10 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.${{ matrix.service }}
-          tags: vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }},vesoft/nebula-${{ matrix.service }}:latest
+          tags: |
+            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
+            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
+            ${{ steps.docker.outputs.tag }}
           push: true
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}

.gitignore

@@ -52,6 +52,7 @@ cmake-build*/
 core.*
 workspace.*
 .metals/
+.cproject
 
 #py
 *.egg-info

README-CN.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/vesoft-inc/nebula/v1-head/docs/logo.png"/>
+  <img src="https://nebula-graph.io/img/nav-nebula-logo.png"/>
   <br>中文 | <a href="README.md">English</a>
   <br>世界上唯一能够容纳千亿个顶点和万亿条边，并提供毫秒级查询延时的图数据库解决方案<br>
 </p>
@@ -34,7 +34,7 @@
 
 ## 发布通告
 
-v1.x和v2.5.0之后的版本，Nebula Graph在这个repo管理。如需获取v2.0.0到v2.5.0之间的版本，请访问[Nebula Graph repo](https://github.com/vesoft-inc/nebula-graph)。
+v1.x和v2.5.1之后的版本，Nebula Graph在这个repo管理。如需获取v2.0.0到v2.5.1之间的版本，请访问[Nebula Graph repo](https://github.com/vesoft-inc/nebula-graph)。
 
 Nebula Graph 1.x 后续不再进行功能的更新，请升级到2.0版本中。Nebula Graph内核 1.x 与 2.x数据格式、通信协议、客户端等均双向不兼容，可参照[升级指导](https://docs.nebula-graph.com.cn/2.5.0/4.deployment-and-installation/3.upgrade-nebula-graph/upgrade-nebula-graph-to-250/)进行升级。
 

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/vesoft-inc/nebula/v1-head/docs/logo.png"/>
+  <img src="https://nebula-graph.io/img/nav-nebula-logo.png"/>
   <br> English | <a href="README-CN.md">中文</a>
   <br>A distributed, scalable, lightning-fast graph database<br>
 </p>
@@ -31,7 +31,7 @@ Compared with other graph database solutions, **Nebula Graph** has the following
 
 ## Notice of Release
 
-This repository hosts the source code of Nebula Graph versions before 2.0.0-alpha and after v2.5.0. If you are looking to use the versions between v2.0.0 and v2.5.0, please head to [Nebula Graph repo](https://github.com/vesoft-inc/nebula-graph). 
+This repository hosts the source code of Nebula Graph versions before 2.0.0-alpha and after v2.5.1. If you are looking to use the versions between v2.0.0 and v2.5.1, please head to [Nebula Graph repo](https://github.com/vesoft-inc/nebula-graph).
 
 Nebula Graph 1.x is not actively maintained. Please move to Nebula Graph 2.x. The data format, rpc protocols, clients, etc. are not compatible between Nebula Graph v1.x and v2.x, but we do offer [upgrade guide from 1.x to v2.5.0](https://docs.nebula-graph.io/2.5.0/4.deployment-and-installation/3.upgrade-nebula-graph/upgrade-nebula-graph-to-250/).
 

cmake/nebula/GeneralCompilerConfig.cmake

@@ -51,6 +51,8 @@ if(ENABLE_TESTING AND ENABLE_COVERAGE)
     add_compile_options(--coverage)
     add_compile_options(-g)
     add_compile_options(-O0)
+    nebula_add_exe_linker_flag(-coverage)
+    nebula_add_exe_linker_flag(-lgcov)
 endif()
 
 # TODO(doodle) Add option suggest-override for gnu

conf/nebula-storaged.conf.default

@@ -90,8 +90,10 @@
 #   * kAll, Collect all stats
 --rocksdb_stats_level=kExceptHistogramOrTimers
 
-# Whether or not to enable rocksdb's prefix bloom filter, disabled by default.
---enable_rocksdb_prefix_filtering=false
+# Whether or not to enable rocksdb's prefix bloom filter, enabled by default.
+--enable_rocksdb_prefix_filtering=true
+# Whether or not to enable rocksdb's whole key bloom filter, disabled by default.
+--enable_rocksdb_whole_key_filtering=false
 
 ############## rocksdb Options ##############
 # rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma

conf/nebula-storaged.conf.production

@@ -96,8 +96,10 @@
 #   * kAll, Collect all stats
 --rocksdb_stats_level=kExceptHistogramOrTimers
 
-# Whether or not to enable rocksdb's prefix bloom filter, disabled by default.
---enable_rocksdb_prefix_filtering=false
+# Whether or not to enable rocksdb's prefix bloom filter, enabled by default.
+--enable_rocksdb_prefix_filtering=true
+# Whether or not to enable rocksdb's whole key bloom filter, disabled by default.
+--enable_rocksdb_whole_key_filtering=false
 
 ############### misc ####################
 --snapshot_part_rate_limit=10485760

src/clients/meta/MetaClient.cpp

@@ -19,6 +19,7 @@
 #include "common/http/HttpClient.h"
 #include "common/meta/NebulaSchemaProvider.h"
 #include "common/network/NetworkUtils.h"
+#include "common/ssl/SSLConfig.h"
 #include "common/stats/StatsManager.h"
 #include "common/time/TimeUtils.h"
 #include "version/Version.h"
@@ -49,7 +50,8 @@ MetaClient::MetaClient(std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool
   CHECK(ioThreadPool_ != nullptr) << "IOThreadPool is required";
   CHECK(!addrs_.empty())
       << "No meta server address is specified or can be solved. Meta server is required";
-  clientsMan_ = std::make_shared<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>>();
+  clientsMan_ = std::make_shared<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>>(
+      FLAGS_enable_ssl || FLAGS_enable_meta_ssl);
   updateActive();
   updateLeader();
   bgThread_ = std::make_unique<thread::GenericWorker>();
@@ -798,6 +800,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("Failed to get meta dir!");
     case nebula::cpp2::ErrorCode::E_INVALID_JOB:
       return Status::Error("No valid job!");
+    case nebula::cpp2::ErrorCode::E_JOB_NOT_IN_SPACE:
+      return Status::Error("Job not in chosen space!");
     case nebula::cpp2::ErrorCode::E_BACKUP_EMPTY_TABLE:
       return Status::Error("Backup empty table!");
     case nebula::cpp2::ErrorCode::E_BACKUP_TABLE_FAILED:
@@ -1060,6 +1064,21 @@ folly::Future<StatusOr<GraphSpaceID>> MetaClient::createSpace(meta::cpp2::SpaceD
   return future;
 }
 
+folly::Future<StatusOr<GraphSpaceID>> MetaClient::createSpaceAs(const std::string& oldSpaceName,
+                                                                const std::string& newSpaceName) {
+  cpp2::CreateSpaceAsReq req;
+  req.set_old_space_name(oldSpaceName);
+  req.set_new_space_name(newSpaceName);
+  folly::Promise<StatusOr<GraphSpaceID>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_createSpaceAs(request); },
+      [](cpp2::ExecResp&& resp) -> GraphSpaceID { return resp.get_id().get_space_id(); },
+      std::move(promise));
+  return future;
+}
+
 folly::Future<StatusOr<std::vector<SpaceIdName>>> MetaClient::listSpaces() {
   cpp2::ListSpacesReq req;
   folly::Promise<StatusOr<std::vector<SpaceIdName>>> promise;

src/clients/meta/MetaClient.h

@@ -215,6 +215,9 @@ class MetaClient {
   folly::Future<StatusOr<GraphSpaceID>> createSpace(meta::cpp2::SpaceDesc spaceDesc,
                                                     bool ifNotExists = false);
 
+  folly::Future<StatusOr<GraphSpaceID>> createSpaceAs(const std::string& oldSpaceName,
+                                                      const std::string& newSpaceName);
+
   folly::Future<StatusOr<std::vector<SpaceIdName>>> listSpaces();
 
   folly::Future<StatusOr<cpp2::SpaceItem>> getSpace(std::string name);

src/clients/meta/test/CMakeLists.txt

@@ -10,5 +10,6 @@ nebula_add_test(
         $<TARGET_OBJECTS:file_based_cluster_id_man_obj>
         $<TARGET_OBJECTS:base_obj>
         $<TARGET_OBJECTS:fs_obj>
+        $<TARGET_OBJECTS:ssl_obj>
     LIBRARIES gtest
 )

src/clients/storage/StorageClientBase-inl.h

@@ -8,6 +8,7 @@
 
 #include <folly/Try.h>
 
+#include "common/ssl/SSLConfig.h"
 #include "common/time/WallClock.h"
 
 namespace nebula {
@@ -72,7 +73,7 @@ template <typename ClientType>
 StorageClientBase<ClientType>::StorageClientBase(
     std::shared_ptr<folly::IOThreadPoolExecutor> threadPool, meta::MetaClient* metaClient)
     : metaClient_(metaClient), ioThreadPool_(threadPool) {
-  clientsMan_ = std::make_unique<thrift::ThriftClientManager<ClientType>>();
+  clientsMan_ = std::make_unique<thrift::ThriftClientManager<ClientType>>(FLAGS_enable_ssl);
 }
 
 template <typename ClientType>

src/codec/test/CMakeLists.txt

@@ -31,6 +31,7 @@ set(CODEC_TEST_LIBS
     $<TARGET_OBJECTS:agg_function_manager_obj>
     $<TARGET_OBJECTS:time_utils_obj>
     $<TARGET_OBJECTS:version_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 

src/common/CMakeLists.txt

@@ -25,3 +25,4 @@ nebula_add_subdirectory(function)
 nebula_add_subdirectory(graph)
 nebula_add_subdirectory(plugin)
 nebula_add_subdirectory(utils)
+nebula_add_subdirectory(ssl)

src/common/algorithm/ReservoirSampling.h

@@ -37,7 +37,13 @@ class ReservoirSampling final {
     return false;
   }
 
-  std::vector<T>&& samples() && { return std::move(samples_); }
+  std::vector<T> samples() {
+    auto result = std::move(samples_);
+    samples_.clear();
+    samples_.reserve(num_);
+    cnt_ = 0;
+    return result;
+  }
 
  private:
   std::vector<T> samples_;

src/common/algorithm/test/ReservoirSamplingTest.cpp

@@ -18,7 +18,7 @@ TEST(ReservoirSamplingTest, Sample) {
       sampler.sampling(std::move(i));
     }
 
-    auto result = std::move(sampler).samples();
+    auto result = sampler.samples();
     EXPECT_EQ(5, result.size());
     for (auto i : result) {
       EXPECT_LE(0, i);
@@ -27,16 +27,18 @@ TEST(ReservoirSamplingTest, Sample) {
   }
   {
     ReservoirSampling<int64_t> sampler(5);
-    std::vector<int64_t> sampleSpace = {0, 1, 2};
-    for (auto i : sampleSpace) {
-      sampler.sampling(std::move(i));
-    }
+    for (size_t count = 0; count < 10; count++) {
+      std::vector<int64_t> sampleSpace = {0, 1, 2};
+      for (auto i : sampleSpace) {
+        sampler.sampling(std::move(i));
+      }
 
-    auto result = std::move(sampler).samples();
-    EXPECT_EQ(3, result.size());
-    EXPECT_EQ(0, result[0]);
-    EXPECT_EQ(1, result[1]);
-    EXPECT_EQ(2, result[2]);
+      auto result = sampler.samples();
+      EXPECT_EQ(3, result.size());
+      EXPECT_EQ(0, result[0]);
+      EXPECT_EQ(1, result[1]);
+      EXPECT_EQ(2, result[2]);
+    }
   }
 }
 }  // namespace algorithm

src/common/datatypes/DataSet.h

@@ -7,6 +7,8 @@
 #ifndef COMMON_DATATYPES_DATASET_H_
 #define COMMON_DATATYPES_DATASET_H_
 
+#include <folly/dynamic.h>
+
 #include <iostream>
 #include <iterator>
 #include <sstream>
@@ -153,6 +155,44 @@ struct DataSet {
     return os.str();
   }
 
+  // format:
+  // [
+  //   {
+  //     "row": [ row-data ],
+  //     "meta": [ metadata ]
+  //   },
+  // ]
+  folly::dynamic toJson() const {
+    // parse rows to json
+    auto dataBody = folly::dynamic::array();
+    for (auto& row : rows) {
+      dataBody.push_back(rowToJson(row));
+    }
+
+    return dataBody;
+  }
+
+  // parse Nebula::Row to json
+  // format:
+  // {
+  //   "row": [ row-data ],
+  //   "meta": [ metadata ]
+  // }
+  folly::dynamic rowToJson(const Row& row) const {
+    folly::dynamic rowJsonObj = folly::dynamic::object();
+    auto rowDataList = folly::dynamic::array();
+    auto metaDataList = folly::dynamic::array();
+
+    for (const auto& ele : row.values) {
+      rowDataList.push_back(ele.toJson());
+      metaDataList.push_back(ele.getMetaData());
+    }
+
+    rowJsonObj.insert("row", rowDataList);
+    rowJsonObj.insert("meta", metaDataList);
+    return rowJsonObj;
+  }
+
   bool operator==(const DataSet& rhs) const { return colNames == rhs.colNames && rows == rhs.rows; }
 };