Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cherry pick: 3.0.2 #4082

Merged
merged 3 commits into from
Mar 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ jobs:
with:
context: .
file: ./docker/Dockerfile.${{ matrix.service }}
# platforms: linux/amd64,linux/arm64
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
tags: |
vesoft/nebula-${{ matrix.service }}:nightly
push: true
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/rc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,15 @@ jobs:
with:
context: .
file: ./docker/Dockerfile.${{ matrix.service }}
# platforms: linux/amd64,linux/arm64
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
tags: |
${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
${{ steps.docker.outputs.tag }}
push: true
build-args: |
BRANCH=${{ steps.tagname.outputs.tag }}
VERSON=${{ steps.tagname.outputs.tagnum }}

test:
name: test
Expand Down
1 change: 1 addition & 0 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
FROM vesoft/nebula-dev:centos7 as builder

ARG BRANCH=master
ARG VERSION=

COPY . /home/nebula/BUILD

Expand Down
9 changes: 6 additions & 3 deletions docker/Dockerfile.graphd
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
FROM vesoft/nebula-dev:centos7 as builder

COPY . /home/nebula/BUILD

ARG BRANCH=master
ARG VERSION=

COPY . /home/nebula/BUILD

RUN cd /home/nebula/BUILD/package \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
&& [[ -z "${VERSION}" ]] \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
|| ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON

FROM centos:7

Expand Down
5 changes: 4 additions & 1 deletion docker/Dockerfile.metad
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
FROM vesoft/nebula-dev:centos7 as builder

ARG BRANCH=master
ARG VERSION=

COPY . /home/nebula/BUILD

RUN cd /home/nebula/BUILD/package \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
&& [[ -z "${VERSION}" ]] \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
|| ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON

FROM centos:7

Expand Down
5 changes: 4 additions & 1 deletion docker/Dockerfile.storaged
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
FROM vesoft/nebula-dev:centos7 as builder

ARG BRANCH=master
ARG VERSION=

COPY . /home/nebula/BUILD

RUN cd /home/nebula/BUILD/package \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
&& [[ -z "${VERSION}" ]] \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
|| ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON

FROM centos:7

Expand Down
5 changes: 4 additions & 1 deletion docker/Dockerfile.tools
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
FROM vesoft/nebula-dev:centos7 as builder

ARG BRANCH=master
ARG VERSION=

COPY . /home/nebula/BUILD

RUN cd /home/nebula/BUILD/package \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
&& [[ -z "${VERSION}" ]] \
&& ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
|| ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON

FROM centos:7

Expand Down
20 changes: 0 additions & 20 deletions src/clients/meta/MetaClient.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2601,11 +2601,6 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
}
}

// TTL for clientAddrMap
// If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
// growing. This is to clear the clientAddrMap_ regularly.
clearClientAddrMap();

// info used in the agent, only set once
// TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
// reported every time it changes
Expand Down Expand Up @@ -3802,20 +3797,5 @@ Status MetaClient::verifyVersion() {
return Status::OK();
}

void MetaClient::clearClientAddrMap() {
if (clientAddrMap_.size() == 0) {
return;
}

auto curTimestamp = time::WallClock::fastNowInSec();
for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
// The clientAddr is expired
if (it->second < curTimestamp) {
it = clientAddrMap_.erase(it);
} else {
++it;
}
}
}
} // namespace meta
} // namespace nebula
20 changes: 0 additions & 20 deletions src/clients/meta/MetaClient.h
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,6 @@ using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;

using SessionMap = std::unordered_map<SessionID, cpp2::Session>;

using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
class MetaChangedListener {
public:
virtual ~MetaChangedListener() = default;
Expand Down Expand Up @@ -671,10 +670,6 @@ class MetaClient {
return options_.localHost_.toString();
}

clientAddrMap& getClientAddrMap() {
return clientAddrMap_;
}

protected:
// Return true if load succeeded.
bool loadData();
Expand Down Expand Up @@ -761,9 +756,6 @@ class MetaClient {

Status verifyVersion();

// Removes expired keys in the clientAddrMap_
void clearClientAddrMap();

private:
std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
Expand Down Expand Up @@ -844,18 +836,6 @@ class MetaClient {
NameIndexMap tagNameIndexMap_;
NameIndexMap edgeNameIndexMap_;

// TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
// client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
//
// The key here is the host of the client that sends the request, and the value indicates the
// expiration of the key because we don't want to keep the key forever.
//
// The assumption here is that there is ONLY ONE VERSION of the client in the host.
//
// This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
// call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
clientAddrMap clientAddrMap_;

// Global service client
ServiceClientsList serviceClientList_;
FTIndexMap fulltextIndexMap_;
Expand Down
35 changes: 4 additions & 31 deletions src/graph/service/GraphService.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@
namespace nebula {
namespace graph {

// The default value is 28800 seconds
const int64_t clientAddrTimeout = FLAGS_client_idle_timeout_secs;

Status GraphService::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExecutor,
const HostAddr& hostAddr) {
auto addrs = network::NetworkUtils::toHosts(FLAGS_meta_server_addrs);
Expand Down Expand Up @@ -72,10 +69,9 @@ folly::Future<AuthResponse> GraphService::future_authenticate(const std::string&

auto ctx = std::make_unique<RequestContext<AuthResponse>>();
auto future = ctx->future();
// Check username and password failed
// Check whether the client has called verifyClientVersion()
auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
auto authResult = auth(username, password, clientAddr);

// check username and password failed
auto authResult = auth(username, password);
if (!authResult.ok()) {
ctx->resp().errorCode = ErrorCode::E_BAD_USERNAME_PASSWORD;
ctx->resp().errorMsg.reset(new std::string(authResult.toString()));
Expand Down Expand Up @@ -207,24 +203,8 @@ folly::Future<std::string> GraphService::future_executeJsonWithParameter(
});
}

Status GraphService::auth(const std::string& username,
const std::string& password,
const HostAddr& clientIp) {
Status GraphService::auth(const std::string& username, const std::string& password) {
auto metaClient = queryEngine_->metaClient();

// TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
// client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
//
// Only the clients since v2.6.0 will call verifyVersion(), thus we could determine whether the
// client version is lower than v2.6.0
auto clientAddrIt = metaClient->getClientAddrMap().find(clientIp);
if (clientAddrIt == metaClient->getClientAddrMap().end()) {
return Status::Error(
folly::sformat("The version of the client sending request from {} is lower than v2.6.0, "
"please update the client.",
clientIp.toString()));
}

// Skip authentication if FLAGS_enable_authorize is false
if (!FLAGS_enable_authorize) {
return Status::OK();
Expand Down Expand Up @@ -266,13 +246,6 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
}

// The client sent request has a version >= v2.6.0, mark the address as valid
auto* peer = getRequestContext()->getPeerAddress();
auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());

auto ttlTimestamp = time::WallClock::fastNowInSec() + clientAddrTimeout;
auto clientAddrMap = &metaClient_->getClientAddrMap();
clientAddrMap->insert_or_assign(clientAddr, ttlTimestamp);
return folly::makeFuture<cpp2::VerifyClientVersionResp>(std::move(resp));
}
} // namespace graph
Expand Down
2 changes: 1 addition & 1 deletion src/graph/service/GraphService.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class GraphService final : public cpp2::GraphServiceSvIf {
std::unique_ptr<meta::MetaClient> metaClient_;

private:
Status auth(const std::string& username, const std::string& password, const HostAddr& clientIp);
Status auth(const std::string& username, const std::string& password);

std::unique_ptr<GraphSessionManager> sessionManager_;
std::unique_ptr<QueryEngine> queryEngine_;
Expand Down