From 987d4d3c11b17171e80e87b00ef40b3acf3781e6 Mon Sep 17 00:00:00 2001 From: Yee <2520865+yixinglu@users.noreply.github.com> Date: Thu, 24 Feb 2022 17:57:37 +0800 Subject: [PATCH 1/3] Support the ARM platform of docker image (#3939) --- .github/workflows/nightly.yml | 3 +-- .github/workflows/rc.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 23e5d29fe19..19004ffab4c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -82,8 +82,7 @@ jobs: with: context: . file: ./docker/Dockerfile.${{ matrix.service }} - # platforms: linux/amd64,linux/arm64 - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 tags: | vesoft/nebula-${{ matrix.service }}:nightly push: true diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 97abcd91cbb..14798fe7ec0 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -99,8 +99,7 @@ jobs: with: context: . file: ./docker/Dockerfile.${{ matrix.service }} - # platforms: linux/amd64,linux/arm64 - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 tags: | ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }} ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }} From 247bb55c1febb73a29a5886cc892b01a55b105ff Mon Sep 17 00:00:00 2001 From: "Harris.Chu" <1726587+HarrisChu@users.noreply.github.com> Date: Fri, 25 Mar 2022 17:34:33 +0800 Subject: [PATCH 2/3] fix version in docker image and Revert 4068 (#4080) * Revert "enhance docker build, using cache" This reverts commit 9e2bec3ce4867a81f881e38aba9b63513847967c. * fix docker version Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .github/workflows/rc.yml | 1 + docker/Dockerfile | 1 + docker/Dockerfile.graphd | 9 ++++++--- docker/Dockerfile.metad | 5 ++++- docker/Dockerfile.storaged | 5 ++++- docker/Dockerfile.tools | 5 ++++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 14798fe7ec0..99025ab48b1 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -107,6 +107,7 @@ jobs: push: true build-args: | BRANCH=${{ steps.tagname.outputs.tag }} + VERSON=${{ steps.tagname.outputs.tagnum }} test: name: test diff --git a/docker/Dockerfile b/docker/Dockerfile index ba9588ab4ee..ec6d2305eb4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,7 @@ FROM vesoft/nebula-dev:centos7 as builder ARG BRANCH=master +ARG VERSION= COPY . /home/nebula/BUILD diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd index a896791c7ba..75bf3e3984e 100644 --- a/docker/Dockerfile.graphd +++ b/docker/Dockerfile.graphd @@ -1,11 +1,14 @@ FROM vesoft/nebula-dev:centos7 as builder -COPY . /home/nebula/BUILD - ARG BRANCH=master +ARG VERSION= + +COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE + && [[ -z "${VERSION}" ]] \ + && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ + || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad index 6b0a246007c..b4ab34f8409 100644 --- a/docker/Dockerfile.metad +++ b/docker/Dockerfile.metad @@ -1,11 +1,14 @@ FROM vesoft/nebula-dev:centos7 as builder ARG BRANCH=master +ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE + && [[ -z "${VERSION}" ]] \ + && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ + || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged index 4d22252dbd2..6ca29687e37 100644 --- a/docker/Dockerfile.storaged +++ b/docker/Dockerfile.storaged @@ -1,11 +1,14 @@ FROM vesoft/nebula-dev:centos7 as builder ARG BRANCH=master +ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE + && [[ -z "${VERSION}" ]] \ + && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ + || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.tools b/docker/Dockerfile.tools index ebec734e3c3..fd696809109 100644 --- a/docker/Dockerfile.tools +++ b/docker/Dockerfile.tools @@ -1,11 +1,14 @@ FROM vesoft/nebula-dev:centos7 as builder ARG BRANCH=master +ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE + && [[ -z "${VERSION}" ]] \ + && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ + || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 From 7f08c8ecc2d84354abd3fde43564c3cd4c43a874 Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Fri, 25 Mar 2022 15:29:14 +0800 Subject: [PATCH 3/3] Revert pr 3942 (#4075) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/clients/meta/MetaClient.cpp | 20 ----------------- src/clients/meta/MetaClient.h | 20 ----------------- src/graph/service/GraphService.cpp | 35 ++++-------------------------- src/graph/service/GraphService.h | 2 +- 4 files changed, 5 insertions(+), 72 deletions(-) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index c8de564aef8..75655009fed 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -2601,11 +2601,6 @@ folly::Future> MetaClient::heartbeat() { } } - // TTL for clientAddrMap - // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep - // growing. This is to clear the clientAddrMap_ regularly. - clearClientAddrMap(); - // info used in the agent, only set once // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be // reported every time it changes @@ -3802,20 +3797,5 @@ Status MetaClient::verifyVersion() { return Status::OK(); } -void MetaClient::clearClientAddrMap() { - if (clientAddrMap_.size() == 0) { - return; - } - - auto curTimestamp = time::WallClock::fastNowInSec(); - for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) { - // The clientAddr is expired - if (it->second < curTimestamp) { - it = clientAddrMap_.erase(it); - } else { - ++it; - } - } -} } // namespace meta } // namespace nebula diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index cb4d33a84ff..0a3eb1579e4 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -156,7 +156,6 @@ using FTIndexMap = std::unordered_map; using SessionMap = std::unordered_map; -using clientAddrMap = folly::ConcurrentHashMap; class MetaChangedListener { public: virtual ~MetaChangedListener() = default; @@ -671,10 +670,6 @@ class MetaClient { return options_.localHost_.toString(); } - clientAddrMap& getClientAddrMap() { - return clientAddrMap_; - } - protected: // Return true if load succeeded. bool loadData(); @@ -761,9 +756,6 @@ class MetaClient { Status verifyVersion(); - // Removes expired keys in the clientAddrMap_ - void clearClientAddrMap(); - private: std::shared_ptr ioThreadPool_; std::shared_ptr> clientsMan_; @@ -844,18 +836,6 @@ class MetaClient { NameIndexMap tagNameIndexMap_; NameIndexMap edgeNameIndexMap_; - // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0) - // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash. - // - // The key here is the host of the client that sends the request, and the value indicates the - // expiration of the key because we don't want to keep the key forever. - // - // The assumption here is that there is ONLY ONE VERSION of the client in the host. - // - // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will - // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0 - clientAddrMap clientAddrMap_; - // Global service client ServiceClientsList serviceClientList_; FTIndexMap fulltextIndexMap_; diff --git a/src/graph/service/GraphService.cpp b/src/graph/service/GraphService.cpp index 8e310bdb5b8..4327e2d0c8e 100644 --- a/src/graph/service/GraphService.cpp +++ b/src/graph/service/GraphService.cpp @@ -24,9 +24,6 @@ namespace nebula { namespace graph { -// The default value is 28800 seconds -const int64_t clientAddrTimeout = FLAGS_client_idle_timeout_secs; - Status GraphService::init(std::shared_ptr ioExecutor, const HostAddr& hostAddr) { auto addrs = network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); @@ -72,10 +69,9 @@ folly::Future GraphService::future_authenticate(const std::string& auto ctx = std::make_unique>(); auto future = ctx->future(); - // Check username and password failed - // Check whether the client has called verifyClientVersion() - auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort()); - auto authResult = auth(username, password, clientAddr); + + // check username and password failed + auto authResult = auth(username, password); if (!authResult.ok()) { ctx->resp().errorCode = ErrorCode::E_BAD_USERNAME_PASSWORD; ctx->resp().errorMsg.reset(new std::string(authResult.toString())); @@ -207,24 +203,8 @@ folly::Future GraphService::future_executeJsonWithParameter( }); } -Status GraphService::auth(const std::string& username, - const std::string& password, - const HostAddr& clientIp) { +Status GraphService::auth(const std::string& username, const std::string& password) { auto metaClient = queryEngine_->metaClient(); - - // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0) - // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash. - // - // Only the clients since v2.6.0 will call verifyVersion(), thus we could determine whether the - // client version is lower than v2.6.0 - auto clientAddrIt = metaClient->getClientAddrMap().find(clientIp); - if (clientAddrIt == metaClient->getClientAddrMap().end()) { - return Status::Error( - folly::sformat("The version of the client sending request from {} is lower than v2.6.0, " - "please update the client.", - clientIp.toString())); - } - // Skip authentication if FLAGS_enable_authorize is false if (!FLAGS_enable_authorize) { return Status::OK(); @@ -266,13 +246,6 @@ folly::Future GraphService::future_verifyClientVe resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; } - // The client sent request has a version >= v2.6.0, mark the address as valid - auto* peer = getRequestContext()->getPeerAddress(); - auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort()); - - auto ttlTimestamp = time::WallClock::fastNowInSec() + clientAddrTimeout; - auto clientAddrMap = &metaClient_->getClientAddrMap(); - clientAddrMap->insert_or_assign(clientAddr, ttlTimestamp); return folly::makeFuture(std::move(resp)); } } // namespace graph diff --git a/src/graph/service/GraphService.h b/src/graph/service/GraphService.h index d96d6385004..85ed21a6ce4 100644 --- a/src/graph/service/GraphService.h +++ b/src/graph/service/GraphService.h @@ -54,7 +54,7 @@ class GraphService final : public cpp2::GraphServiceSvIf { std::unique_ptr metaClient_; private: - Status auth(const std::string& username, const std::string& password, const HostAddr& clientIp); + Status auth(const std::string& username, const std::string& password); std::unique_ptr sessionManager_; std::unique_ptr queryEngine_;