Merge branch 'keep' of github.com:heroicNeZha/nebula into keep

.clang-tidy

@@ -67,7 +67,7 @@ Checks: '-*,clang-diagnostic-*,clang-analyzer-*,-misc-unused-parameters,
   google-upgrade-googletest-case,
 
   modernize-avoid-bind,
-  modernize-concat-nested-namespaces,
+  -modernize-concat-nested-namespaces,
   modernize-deprecated-headers,
   modernize-deprecated-ios-base-aliases,
   modernize-loop-convert,

.github/workflows/nightly.yml

@@ -54,7 +54,7 @@ jobs:
           endpoint: ${{ secrets.OSS_ENDPOINT }}
           bucket: nebula-graph
           asset-path: pkg-build/cpack_output
-          target-path: package/v2-nightly/${{ steps.vars.outputs.subdir }}
+          target-path: package/nightly/${{ steps.vars.outputs.subdir }}
 
   docker:
     name: build docker image

.github/workflows/rc.yml

@@ -0,0 +1,109 @@
+name: rc
+
+on:
+  push:
+    branches:
+      - 'v[0-9]+.*'
+
+concurrency:
+  group: rc
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  package:
+    name: build package
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu1604
+          - ubuntu1804
+          - ubuntu2004
+          - centos7
+          - centos8
+    container:
+      image: vesoft/nebula-dev:${{ matrix.os }}
+    env:
+      BUILD_DIR: ./pkg-build
+      CPACK_DIR: ./pkg-build/cpack_output
+      SYMS_DIR: ./pkg-build/symbols
+    steps:
+      - uses: webiny/[email protected]
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tagname-action
+        id: tag
+      - name: package
+        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE
+      - name: output some vars
+        run: |
+          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
+          find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
+      - uses: ./.github/actions/upload-to-oss-action
+        with:
+          key-id: ${{ secrets.OSS_ID }}
+          key-secret: ${{ secrets.OSS_SECRET }}
+          endpoint: ${{ secrets.OSS_ENDPOINT }}
+          bucket: nebula-graph
+          asset-path: ${{ env.CPACK_DIR }}
+          target-path: rc/${{ steps.tag.outputs.tagnum }}
+      - uses: ./.github/actions/upload-to-oss-action
+        with:
+          key-id: ${{ secrets.OSS_ID }}
+          key-secret: ${{ secrets.OSS_SECRET }}
+          endpoint: ${{ secrets.OSS_ENDPOINT }}
+          bucket: nebula-graph
+          asset-path: ${{ env.SYMS_DIR }}
+          target-path: rc/${{ steps.tag.outputs.tagnum }}/symbols
+
+  docker_build:
+    name: docker-build
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          - graphd
+          - metad
+          - storaged
+          - tools
+    steps:
+      - uses: webiny/[email protected]
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tagname-action
+        id: tagname
+      - id: docker
+        run: |
+          majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.')
+          tag=""
+          if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then
+            tag="${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:latest"
+          fi
+          echo "::set-output name=tag::$tag"
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.HARBOR_REGISTRY }}
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/Dockerfile.${{ matrix.service }}
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
+            ${{ steps.docker.outputs.tag }}
+          push: true
+          build-args: |
+            BRANCH=${{ steps.tagname.outputs.tag }}

.github/workflows/release.yml

@@ -15,54 +15,21 @@ defaults:
 
 jobs:
   package:
-    name: build package
-    runs-on: [self-hosted, nebula]
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu1604
-          - ubuntu1804
-          - ubuntu2004
-          - centos7
-          - centos8
+    name: package
+    runs-on: ubuntu-latest
     container:
-      image: vesoft/nebula-dev:${{ matrix.os }}
-    env:
-      BUILD_DIR: ./pkg-build
-      CPACK_DIR: ./pkg-build/cpack_output
-      SYMS_DIR: ./pkg-build/symbols
+      image: vesoft/nebula-dev:centos7
     steps:
-      - uses: webiny/[email protected]
-        with:
-          run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
-      - name: Check License Header
-        uses: apache/skywalking-eyes@main
       - uses: ./.github/actions/tagname-action
         id: tag
-      - name: package
-        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE
-      - name: output some vars
-        run: |
-          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
-          find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
-      - uses: ./.github/actions/upload-to-oss-action
-        with:
-          key-id: ${{ secrets.OSS_ID }}
-          key-secret: ${{ secrets.OSS_SECRET }}
-          endpoint: ${{ secrets.OSS_ENDPOINT }}
-          bucket: nebula-graph
-          asset-path: ${{ env.CPACK_DIR }}
-          target-path: package/${{ steps.tag.outputs.tagnum }}
-      - uses: ./.github/actions/upload-to-oss-action
-        with:
-          key-id: ${{ secrets.OSS_ID }}
-          key-secret: ${{ secrets.OSS_SECRET }}
-          endpoint: ${{ secrets.OSS_ENDPOINT }}
-          bucket: nebula-graph
-          asset-path: ${{ env.SYMS_DIR }}
-          target-path: package/${{ steps.tag.outputs.tagnum }}/symbols
+      - run: |
+          ossutil64 cp -rf \
+            -i ${{ secrets.OSS_ID }} \
+            -k ${{ secrets.OSS_SECRET }} \
+            -e ${{ secrets.OSS_ENDPOINT }} \
+            oss://nebula-graph/rc/${{ steps.tag.outputs.tagnum }} \
+            oss://nebula-graph/package/${{ steps.tag.outputs.tagnum }}
 
   docker_build:
     name: docker-build
@@ -80,33 +47,24 @@ jobs:
         with:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
-      - name: Check License Header
-        uses: apache/skywalking-eyes@main
       - uses: ./.github/actions/tagname-action
         id: tagname
       - id: docker
         run: |
           majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.')
           tag=""
           if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then
-            tag="vesoft/nebula-${{ matrix.service }}:latest"
+            tag="latest"
           fi
           echo "::set-output name=tag::$tag"
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile.${{ matrix.service }}
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
-            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
-            ${{ steps.docker.outputs.tag }}
-          push: true
-          build-args: |
-            BRANCH=${{ steps.tagname.outputs.tag }}
+      - name: Sync docker images
+        env:
+          FROM_IMAGE: docker://${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}
+          TO_IMAGE: docker://docker.io/vesoft/nebula-${{ matrix.service }}
+          CMD: docker run --rm -ti quay.io/containers/skopeo:v1.4.1 copy -a --src-creds ${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }} --dest-creds ${{ secrets.DOCKER_USERNAME }}:${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.tag }}
+          ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.majorver }}
+          if [[ ! -z "${{ steps.docker.outputs.tag }}" ]]; then
+            ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.docker.outputs.tag }}
+          fi

.linters/cpp/checkKeyword.py

@@ -8,7 +8,7 @@
 import re
 import sys
 
-PASER_FILE_PATH = 'src/parser/parser.yy'
+PARSER_FILE_PATH = 'src/parser/parser.yy'
 SCANNER_FILE_PATH = 'src/parser/scanner.lex'
 
 reserved_key_words = [
@@ -139,7 +139,7 @@ def get_unreserved_keyword(file_path):
 
     if len(keywords) == 0:
         exit(0)
-    unreserved_key_words = get_unreserved_keyword(PASER_FILE_PATH)
+    unreserved_key_words = get_unreserved_keyword(PARSER_FILE_PATH)
     new_key_words = [word for word in keywords if word not in reserved_key_words]
     if len(new_key_words) == 0:
         exit(0)

.linters/cpp/cpplint.py

@@ -641,7 +641,7 @@
 # Files to exclude from linting. This is set by the --exclude flag.
 _excludes = None
 
-# Whether to supress PrintInfo messages
+# Whether to suppress PrintInfo messages
 _quiet = False
 
 # The allowed line length of files.
@@ -752,7 +752,7 @@ def ParseNolintSuppressions(filename, raw_line, linenum, error):
                 'Unknown NOLINT error category: %s' % category)
 
 
-def ProcessGlobalSuppresions(lines):
+def ProcessGlobalSuppressions(lines):
   """Updates the list of global error suppressions.
 
   Parses any lint directives in the file that have global effect.
@@ -780,7 +780,7 @@ def IsErrorSuppressedByNolint(category, linenum):
   """Returns true if the specified error category is suppressed on this line.
 
   Consults the global error_suppressions map populated by
-  ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions.
+  ParseNolintSuppressions/ProcessGlobalSuppressions/ResetNolintSuppressions.
 
   Args:
     category: str, the category of the error.
@@ -1013,7 +1013,7 @@ def __init__(self):
     self._filters_backup = self.filters[:]
     self.counting = 'total'  # In what way are we counting errors?
     self.errors_by_category = {}  # string to int dict storing error counts
-    self.quiet = False  # Suppress non-error messagess?
+    self.quiet = False  # Suppress non-error messages?
 
     # output format:
     # "emacs" - format that emacs can parse (default)
@@ -6202,7 +6202,7 @@ def ProcessFileData(filename, file_extension, lines, error,
   ResetNolintSuppressions()
 
   CheckForCopyright(filename, lines, error)
-  ProcessGlobalSuppresions(lines)
+  ProcessGlobalSuppressions(lines)
   RemoveMultiLineComments(filename, lines, error)
   clean_lines = CleansedLines(lines)
 

cmake/ThriftGenerate.cmake

@@ -32,7 +32,7 @@ endmacro()
 #   @output_path - The directory where the thrift file lives
 #
 # Output:
-#  file-cpp2-target     - A custom target to add a dependenct
+#  file-cpp2-target     - A custom target to add a dependency
 #  ${file-cpp2-HEADERS} - The generated Header Files.
 #  ${file-cpp2-SOURCES} - The generated Source Files.
 #

conf/nebula-graphd.conf.default

@@ -7,7 +7,7 @@
 --enable_optimizer=true
 # The default charset when a space is created
 --default_charset=utf8
-# The defaule collate when a space is created
+# The default collate when a space is created
 --default_collate=utf8_bin
 # Whether to use the configuration obtained from the configuration file
 --local_config=true

conf/nebula-storaged.conf.production

@@ -102,7 +102,7 @@
 --enable_rocksdb_whole_key_filtering=false
 
 ############### misc ####################
---snapshot_part_rate_limit=8388608
+--snapshot_part_rate_limit=10485760
 --snapshot_batch_size=1048576
 --rebuild_index_part_rate_limit=4194304
 --rebuild_index_batch_size=1048576

package/package.sh

@@ -183,7 +183,7 @@ function _find_dump_syms_tool {
 function _strip_unnecessary_binaries {
     for bin in $(ls -1 -F ${build_dir}/bin/ | grep -v [/$] | sed -e '/nebula-metad/d;/nebula-graphd/d;/nebula-storaged/d'); do
         if ! (strip ${build_dir}/bin/${bin}); then
-            echo ">>> strip ${bin} faild: $?. <<<"
+            echo ">>> strip ${bin} failed: $?. <<<"
             exit 1
         fi
     done
@@ -203,7 +203,7 @@ function dump_syms {
 
     for bin in nebula-graphd nebula-storaged nebula-metad; do
         if ! (${dump_syms} ${build_dir}/bin/${bin} > ${syms_dir}/${bin}${ver}.sym); then
-            echo ">>> dump ${bin} symbols faild: $?. <<<"
+            echo ">>> dump ${bin} symbols failed: $?. <<<"
             exit 1
         fi
     done

resources/gflags.json

@@ -4,7 +4,7 @@
         "v",
         "heartbeat_interval_secs",
         "meta_client_retry_times",
-        "slow_op_threshhold_ms",
+        "slow_op_threshold_ms",
         "clean_wal_interval_secs",
         "wal_ttl",
         "clean_wal_interval_secs",

scripts/meta-transfer-tools.sh

@@ -16,7 +16,7 @@ Usage="this tool is a simple wrapper to scp to copy local folder of metad to ano
         1. -f (from) local metad dir \n \
         2. -t (to) remote destination \n \
         3. -u (update) any configs need to be changed \n \
-            different configs should be seperated by ':' \n \
+            different configs should be separated by ':' \n \
             each config has to be the form of "local_ip=172.0.0.1" \n \
         \n \
     for example \n \

scripts/nebula.service

@@ -18,7 +18,7 @@ SCRIPT_PATH=$(readlink -f $0)
 # Directory of this script
 SCRIPT_DIR=$(dirname ${SCRIPT_PATH})
 # Installation directory, i.e. parent of SCRIPT_DIR
-# For now we assume that the directory hierachy of the nebula installation is:
+# For now we assume that the directory hierarchy of the nebula installation is:
 # root/bin, root/etc/, root/scripts, etc.
 INSTALL_ROOT=$(cd ${SCRIPT_DIR}/.. &>/dev/null; pwd)
 UTILS_PATH=${SCRIPT_DIR}/utils.sh

src/clients/meta/FileBasedClusterIdMan.cpp

@@ -31,7 +31,7 @@ bool FileBasedClusterIdMan::persistInFile(ClusterID clusterId, const std::string
     ::close(fd);
     return false;
   }
-  LOG(INFO) << "Persiste clusterId " << clusterId << " succeeded!";
+  LOG(INFO) << "Persist clusterId " << clusterId << " succeeded!";
   ::close(fd);
   return true;
 }