Merge branch 'master' into stop

vesoft-inc · Mar 27, 2022 · b36fd5f · b36fd5f
2 parents 4ccbb1c + 5d96c60
commit b36fd5f
Show file tree

Hide file tree

Showing 27 changed files with 442 additions and 388 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -62,6 +62,14 @@ jobs:
   docker:
     name: build docker image
     runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          - graphd
+          - metad
+          - storaged
+          - tools
     steps:
       - uses: webiny/[email protected]
         with:
@@ -76,38 +84,10 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            vesoft/nebula-graphd:nightly
-          target: graphd
-          push: true
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            vesoft/nebula-storaged:nightly
-          target: storaged
-          push: true
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            vesoft/nebula-metad:nightly
-          target: metad
-          push: true
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
+          file: ./docker/Dockerfile.${{ matrix.service }}
           platforms: linux/amd64,linux/arm64
           tags: |
-            vesoft/nebula-tools:nightly
-          target: tools
+            vesoft/nebula-${{ matrix.service }}:nightly
           push: true
 
   coverage:

diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
@@ -68,6 +68,14 @@ jobs:
   docker_build:
     name: docker-build
     runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          - graphd
+          - metad
+          - storaged
+          - tools
     steps:
       - uses: webiny/[email protected]
         with:
@@ -93,62 +101,16 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-graphd:${{ steps.tagname.outputs.tag }}
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-graphd:${{ steps.tagname.outputs.majorver }}
-            ${{ steps.docker.outputs.tag }}
-          push: true
-          target: graphd
-          build-args: |
-            BRANCH=${{ steps.tagname.outputs.tag }}
-            VERSION=${{ steps.tagname.outputs.tagnum }}
-
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.tag }}
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.majorver }}
-            ${{ steps.docker.outputs.tag }}
-          push: true
-          target: storaged
-          build-args: |
-            BRANCH=${{ steps.tagname.outputs.tag }}
-            VERSION=${{ steps.tagname.outputs.tagnum }}
-
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.tag }}
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.majorver }}
-            ${{ steps.docker.outputs.tag }}
-          push: true
-          target: metad
-          build-args: |
-            BRANCH=${{ steps.tagname.outputs.tag }}
-            VERSION=${{ steps.tagname.outputs.tagnum }}
-      
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile
+          file: ./docker/Dockerfile.${{ matrix.service }}
           platforms: linux/amd64,linux/arm64
           tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.tag }}
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.majorver }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.tag }}
           push: true
-          target: tools
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
-            VERSION=${{ steps.tagname.outputs.tagnum }}
+            VERSON=${{ steps.tagname.outputs.tagnum }}
 
   test:
     name: test

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -2,6 +2,7 @@ FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
 ARG VERSION=
+
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \

diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd
@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-graph.rpm /usr/local/nebula/nebula-graphd.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+  && mkdir -p ./{logs,data,pids} \
+  && rm -rf *.rpm
+
+EXPOSE 9669 19669 19670
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"]
diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad
@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-meta.rpm /usr/local/nebula/nebula-metad.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+  && mkdir -p ./{logs,data,pids} \
+  && rm -rf *.rpm
+
+EXPOSE 9559 9560 19559 19560
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"]
diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged
@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-storage.rpm /usr/local/nebula/nebula-storaged.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+  && mkdir -p ./{logs,data,pids} \
+  && rm -rf *.rpm
+
+EXPOSE 9777 9778 9779 9780 19779 19780
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"]
diff --git a/docker/Dockerfile.tools b/docker/Dockerfile.tools
@@ -0,0 +1,23 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-tool.rpm /usr/local/nebula/nebula-tool.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+  && rm -rf *.rpm
+
+# default entrypoint
+ENTRYPOINT ["/usr/local/nebula/bin/db_dump"]
diff --git a/docker/README.md b/docker/README.md
@@ -2,7 +2,7 @@
 
 Following docker images will be ready in production.
 
-- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile` and target `graphd`
-- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile` and target `metad`
-- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile` and target `storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile` and target `tools`, including db_dump, meta_dump and db_upgrader
+- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
+- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
+- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader
diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -2484,11 +2484,6 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
     }
   }
 
-  // TTL for clientAddrMap
-  // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
-  // growing. This is to clear the clientAddrMap_ regularly.
-  clearClientAddrMap();
-
   // info used in the agent, only set once
   // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
   // reported every time it changes
@@ -3658,20 +3653,5 @@ Status MetaClient::verifyVersion() {
   return Status::OK();
 }
 
-void MetaClient::clearClientAddrMap() {
-  if (clientAddrMap_.size() == 0) {
-    return;
-  }
-
-  auto curTimestamp = time::WallClock::fastNowInSec();
-  for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
-    // The clientAddr is expired
-    if (it->second < curTimestamp) {
-      it = clientAddrMap_.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
 }  // namespace meta
 }  // namespace nebula
diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h
@@ -155,7 +155,6 @@ using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;
 
 using SessionMap = std::unordered_map<SessionID, cpp2::Session>;
 
-using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
 class MetaChangedListener {
  public:
   virtual ~MetaChangedListener() = default;
@@ -657,10 +656,6 @@ class MetaClient : public BaseMetaClient {
     return options_.localHost_.toString();
   }
 
-  clientAddrMap& getClientAddrMap() {
-    return clientAddrMap_;
-  }
-
  protected:
   // Return true if load succeeded.
   bool loadData();
@@ -747,9 +742,6 @@ class MetaClient : public BaseMetaClient {
 
   Status verifyVersion();
 
-  // Removes expired keys in the clientAddrMap_
-  void clearClientAddrMap();
-
  private:
   std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
   std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
@@ -830,18 +822,6 @@ class MetaClient : public BaseMetaClient {
   NameIndexMap tagNameIndexMap_;
   NameIndexMap edgeNameIndexMap_;
 
-  // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
-  // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
-  //
-  // The key here is the host of the client that sends the request, and the value indicates the
-  // expiration of the key because we don't want to keep the key forever.
-  //
-  // The assumption here is that there is ONLY ONE VERSION of the client in the host.
-  //
-  // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
-  // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
-  clientAddrMap clientAddrMap_;
-
   // Global service client
   ServiceClientsList serviceClientList_;
   FTIndexMap fulltextIndexMap_;

diff --git a/src/common/thread/test/GenericWorkerTest.cpp b/src/common/thread/test/GenericWorkerTest.cpp
@@ -120,7 +120,7 @@ TEST(GenericWorker, addRepeatTask) {
   }
 }
 
-TEST(GenericWorker, purgeRepeatTask) {
+TEST(GenericWorker, DISABLE_purgeRepeatTask) {
   GenericWorker worker;
   ASSERT_TRUE(worker.start());
   {