From 27e9f0d9c62e5bf0c177eaad0b3fb7092c1dec50 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Thu, 26 Aug 2021 10:00:45 +0800 Subject: [PATCH] cherry-pick #549 #550 #551 --- src/meta/CMakeLists.txt | 8 +++++++ src/meta/MetaServiceUtils.cpp | 25 ++++++++++++++++----- src/meta/processors/admin/HBProcessor.cpp | 25 +++++++++------------ src/meta/processors/admin/HBProcessor.h | 1 + src/meta/test/HBProcessorTest.cpp | 2 -- src/meta/test/RestoreProcessorTest.cpp | 19 ++++++++++++++++ src/storage/test/CMakeLists.txt | 27 +++-------------------- src/tools/db-dump/CMakeLists.txt | 3 +++ src/tools/db-upgrade/CMakeLists.txt | 3 +++ src/tools/meta-dump/CMakeLists.txt | 3 +++ src/tools/simple-kv-verify/CMakeLists.txt | 3 +++ src/tools/storage-perf/CMakeLists.txt | 3 +++ 12 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index a52603be180..63e2d75dde1 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -96,6 +96,11 @@ nebula_add_library( processors/session/SessionManagerProcessor.cpp ) +add_dependencies( + meta_service_handler + meta_version_man_obj +) + nebula_add_library( meta_version_man_obj OBJECT MetaVersionMan.cpp @@ -109,6 +114,9 @@ add_dependencies( set(meta_test_deps $ $ + $ + $ + $ $ $ $ diff --git a/src/meta/MetaServiceUtils.cpp b/src/meta/MetaServiceUtils.cpp index b699e3d0979..735abbbd912 100644 --- a/src/meta/MetaServiceUtils.cpp +++ b/src/meta/MetaServiceUtils.cpp @@ -28,8 +28,11 @@ static const std::unordered_map> syste {"groups", {"__groups__", true}}, {"zones", {"__zones__", true}}, {"ft_service", {"__ft_service__", false}}, - {"sessions", {"__sessions__", true}}, - {"id", {"__id__", true}}}; + {"sessions", {"__sessions__", true}}}; + +// SystemInfo will always be backuped +static const std::unordered_map> systemInfoMaps{ + {"autoIncrementId", {"__id__", true}}, {"lastUpdateTime", {"__last_update_time__", true}}}; // name => {prefix, parseSpaceid}, nullptr means that the backup should be skipped. static const std::unordered_map< @@ -45,7 +48,6 @@ static const std::unordered_map< {"index", {"__index__", nullptr}}, {"index_status", {"__index_status__", MetaServiceUtils::parseIndexStatusKeySpaceID}}, {"roles", {"__roles__", MetaServiceUtils::parseRoleSpace}}, - {"last_update_time", {"__last_update_time__", nullptr}}, {"leaders", {"__leaders__", nullptr}}, {"leader_terms", {"__leader_terms__", nullptr}}, {"listener", {"__listener__", nullptr}}, @@ -68,7 +70,6 @@ static const std::string kUsersTable = systemTableMaps.at("users").firs static const std::string kRolesTable = tableMaps.at("roles").first; // NOLINT static const std::string kConfigsTable = systemTableMaps.at("configs").first; // NOLINT static const std::string kSnapshotsTable = systemTableMaps.at("snapshots").first; // NOLINT -static const std::string kLastUpdateTimeTable = tableMaps.at("last_update_time").first; // NOLINT static const std::string kLeadersTable = tableMaps.at("leaders").first; // NOLINT static const std::string kLeaderTermsTable = tableMaps.at("leader_terms").first; // NOLINT static const std::string kGroupsTable = systemTableMaps.at("groups").first; // NOLINT @@ -87,7 +88,9 @@ const std::string kFTIndexTable = tableMaps.at("ft_index").first; const std::string kFTServiceTable = systemTableMaps.at("ft_service").first; // NOLINT const std::string kSessionsTable = systemTableMaps.at("sessions").first; // NOLINT -const std::string kIdKey = systemTableMaps.at("id").first; // NOLINT +const std::string kIdKey = systemInfoMaps.at("autoIncrementId").first; // NOLINT +const std::string kLastUpdateTimeTable = systemInfoMaps.at("lastUpdateTime").first; // NOLINT + // clang-format on const int kMaxIpAddrLen = 15; // '255.255.255.255' @@ -1095,6 +1098,18 @@ ErrorOr> MetaServiceUtils::bac } } + for (const auto& table : systemInfoMaps) { + if (!table.second.second) { + LOG(INFO) << table.first << " table skipped"; + continue; + } + auto result = backupTable(kvstore, backupName, table.second.first, files, nullptr); + if (result != nebula::cpp2::ErrorCode::SUCCEEDED) { + return result; + } + LOG(INFO) << table.first << " table backup successed"; + } + // The mapping of space name and space id needs to be handled separately. auto ret = backupIndex(kvstore, spaces, backupName, spaceNames); if (!ok(ret)) { diff --git a/src/meta/processors/admin/HBProcessor.cpp b/src/meta/processors/admin/HBProcessor.cpp index 4806a6a08e7..a957337c923 100644 --- a/src/meta/processors/admin/HBProcessor.cpp +++ b/src/meta/processors/admin/HBProcessor.cpp @@ -9,14 +9,15 @@ #include "common/time/WallClock.h" #include "meta/ActiveHostsMan.h" #include "meta/KVBasedClusterIdMan.h" - -DEFINE_bool(hosts_whitelist_enabled, false, "Check host whether in whitelist when received hb"); +#include "meta/MetaVersionMan.h" namespace nebula { namespace meta { HBCounters kHBCounters; +std::atomic HBProcessor::metaVersion_ = -1; + void HBProcessor::onFinished() { if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); @@ -29,18 +30,6 @@ void HBProcessor::onFinished() { void HBProcessor::process(const cpp2::HBReq& req) { HostAddr host((*req.host_ref()).host, (*req.host_ref()).port); nebula::cpp2::ErrorCode ret; - if (FLAGS_hosts_whitelist_enabled) { - ret = hostExist(MetaServiceUtils::hostKey(host.host, host.port)); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Reject unregistered host " << host << "!"; - if (ret != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { - ret = nebula::cpp2::ErrorCode::E_INVALID_HOST; - } - handleErrorCode(ret); - onFinished(); - return; - } - } LOG(INFO) << "Receive heartbeat from " << host << ", role = " << apache::thrift::util::enumNameSafe(req.get_role()); @@ -79,6 +68,14 @@ void HBProcessor::process(const cpp2::HBReq& req) { } else if (nebula::error(lastUpdateTimeRet) == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { resp_.set_last_update_time_in_ms(0); } + + auto version = metaVersion_.load(); + if (version == -1) { + metaVersion_.store(static_cast(MetaVersionMan::getMetaVersionFromKV(kvstore_))); + } + + resp_.set_meta_version(metaVersion_.load()); + handleErrorCode(ret); onFinished(); } diff --git a/src/meta/processors/admin/HBProcessor.h b/src/meta/processors/admin/HBProcessor.h index ca77754ebfa..cd286654542 100644 --- a/src/meta/processors/admin/HBProcessor.h +++ b/src/meta/processors/admin/HBProcessor.h @@ -56,6 +56,7 @@ class HBProcessor : public BaseProcessor { ClusterID clusterId_{0}; const HBCounters* counters_{nullptr}; + static std::atomic metaVersion_; }; } // namespace meta diff --git a/src/meta/test/HBProcessorTest.cpp b/src/meta/test/HBProcessorTest.cpp index a8479825227..6fcc7bb003c 100644 --- a/src/meta/test/HBProcessorTest.cpp +++ b/src/meta/test/HBProcessorTest.cpp @@ -11,8 +11,6 @@ #include "meta/processors/admin/HBProcessor.h" #include "meta/test/TestUtils.h" -DECLARE_bool(hosts_whitelist_enabled); - namespace nebula { namespace meta { diff --git a/src/meta/test/RestoreProcessorTest.cpp b/src/meta/test/RestoreProcessorTest.cpp index 1c4f9657258..91c418f7382 100644 --- a/src/meta/test/RestoreProcessorTest.cpp +++ b/src/meta/test/RestoreProcessorTest.cpp @@ -89,6 +89,14 @@ TEST(RestoreProcessorTest, RestoreTest) { std::string(reinterpret_cast(&zoneId), sizeof(ZoneID))); data.emplace_back(MetaServiceUtils::zoneKey(zoneName), MetaServiceUtils::zoneVal(hosts)); + int32_t autoId = 666; + data.emplace_back(MetaServiceUtils::idKey(), + std::string(reinterpret_cast(&autoId), sizeof(autoId))); + + auto lastUpdateTime = time::WallClock::fastNowInMilliSec(); + data.emplace_back(MetaServiceUtils::lastUpdateTimeKey(), + MetaServiceUtils::lastUpdateTimeVal(lastUpdateTime)); + folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { ret = (code == nebula::cpp2::ErrorCode::SUCCEEDED); @@ -218,6 +226,17 @@ TEST(RestoreProcessorTest, RestoreTest) { result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); ASSERT_FALSE(iter->valid()); + + std::string key = "__id__"; + std::string value; + result = kvRestore->get(kDefaultSpaceId, kDefaultPartId, key, &value); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); + ASSERT_EQ(autoId, *reinterpret_cast(value.data())); + + key = "__last_update_time__"; + result = kvRestore->get(kDefaultSpaceId, kDefaultPartId, key, &value); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); + ASSERT_EQ(lastUpdateTime, *reinterpret_cast(value.data())); } } diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index fb756f7ab84..95a8950078e 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -1,6 +1,9 @@ set(storage_test_deps $ $ + $ + $ + $ $ $ $ @@ -575,13 +578,6 @@ nebula_add_test( SOURCES KVTest.cpp OBJECTS - $ - $ - $ - $ - $ - $ - $ ${storage_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} @@ -597,15 +593,7 @@ nebula_add_test( SOURCES KVClientTest.cpp OBJECTS - $ - $ - $ - $ $ - $ - $ - $ - $ ${storage_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} @@ -636,15 +624,6 @@ nebula_add_executable( SOURCES ElasticSearchBulkInsertTest.cpp OBJECTS - $ - $ - $ - $ - $ - $ - $ - $ - $ ${storage_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index d5f419ffad2..ae25df40ea2 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -1,5 +1,8 @@ set(tools_test_deps $ + $ + $ + $ $ $ $ diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index bf7a64c065f..0259465280e 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -8,6 +8,9 @@ nebula_add_executable( DbUpgrader.cpp OBJECTS $ + $ + $ + $ $ $ $ diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 64758a8b4f6..b715015e61f 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -5,6 +5,9 @@ nebula_add_executable( MetaDumpTool.cpp OBJECTS $ + $ + $ + $ $ $ $ diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 1067e495db9..9d988f9cd18 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -5,6 +5,9 @@ nebula_add_executable( SimpleKVVerifyTool.cpp OBJECTS $ + $ + $ + $ $ $ $ diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index d9b5c80370c..ee298ac6737 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -1,5 +1,8 @@ set(perf_test_deps $ + $ + $ + $ $ $ $