diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 7103c88e6c7..bb97c1dad12 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -773,7 +773,7 @@ class MetaClient { std::atomic metadLastUpdateTime_{0}; int64_t metaServerVersion_{-1}; - static constexpr int64_t EXPECT_META_VERSION = 2; + static constexpr int64_t EXPECT_META_VERSION = 3; // leadersLock_ is used to protect leadersInfo folly::RWSpinLock leadersLock_; diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index 2735b9b1d41..b03d253c6b9 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -1757,7 +1757,7 @@ FunctionManager::FunctionManager() { attr.maxArity_ = 1; attr.isPure_ = false; attr.body_ = [](const auto &args) -> Value { - if (args.size() == 0) { + if (args.empty()) { return time::WallClock::fastNowInSec(); } auto status = time::TimeUtils::toTimestamp(args[0].get()); diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 1d476baf543..026ca7512b9 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -99,6 +99,7 @@ nebula_add_executable( $ $ $ + $ $ ${common_deps} ${storage_meta_deps} diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index c41b1e02c06..09640d5dc58 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -138,26 +138,22 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - if (leader == localhost) { - LOG(INFO) << "I am leader, begin upgrade meta data"; - // need to upgrade the v1.0 meta data format to v2.0 meta data format - auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); - if (!ret.ok()) { - LOG(ERROR) << ret; - return nullptr; - } - } else { - LOG(INFO) << "I am follower, wait for leader to sync upgrade"; - while (version != nebula::meta::MetaVersion::V2) { - VLOG(1) << "Waiting for leader to upgrade"; - sleep(1); - version = nebula::meta::MetaVersionMan::getMetaVersionFromKV(kvstore.get()); - } + auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; + } + + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V2); + } else if (version == nebula::meta::MetaVersion::V2) { + LOG(INFO) << "version 3"; + auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; } - } - if (leader == localhost) { - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get()); + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V3); } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/graph/executor/query/LeftJoinExecutor.cpp b/src/graph/executor/query/LeftJoinExecutor.cpp index 9dddd2ee0d2..6a8725ad076 100644 --- a/src/graph/executor/query/LeftJoinExecutor.cpp +++ b/src/graph/executor/query/LeftJoinExecutor.cpp @@ -33,14 +33,14 @@ folly::Future LeftJoinExecutor::join(const std::vector& has DataSet result; if (hashKeys.size() == 1 && probeKeys.size() == 1) { std::unordered_map> hashTable; - hashTable.reserve(rhsIter_->size() == 0 ? 1 : rhsIter_->size()); + hashTable.reserve(rhsIter_->empty() ? 1 : rhsIter_->size()); if (!lhsIter_->empty()) { buildSingleKeyHashTable(probeKeys.front(), rhsIter_.get(), hashTable); result = singleKeyProbe(hashKeys.front(), lhsIter_.get(), hashTable); } } else { std::unordered_map> hashTable; - hashTable.reserve(rhsIter_->size() == 0 ? 1 : rhsIter_->size()); + hashTable.reserve(rhsIter_->empty() ? 1 : rhsIter_->size()); if (!lhsIter_->empty()) { buildHashTable(probeKeys, rhsIter_.get(), hashTable); result = probe(hashKeys, lhsIter_.get(), hashTable); diff --git a/src/graph/planner/match/ScanSeek.cpp b/src/graph/planner/match/ScanSeek.cpp index 8b08a157858..659efeb6be1 100644 --- a/src/graph/planner/match/ScanSeek.cpp +++ b/src/graph/planner/match/ScanSeek.cpp @@ -28,7 +28,7 @@ StatusOr ScanSeek::transformEdge(EdgeContext *edgeCtx) { bool ScanSeek::matchNode(NodeContext *nodeCtx) { auto &node = *nodeCtx->info; // only require the tag - if (node.tids.size() == 0) { + if (node.tids.empty()) { // empty labels means all labels const auto *qctx = nodeCtx->matchClauseCtx->qctx; auto allLabels = qctx->schemaMng()->getAllTags(nodeCtx->matchClauseCtx->space.id); diff --git a/src/kvstore/plugins/hbase/HBaseClient.cpp b/src/kvstore/plugins/hbase/HBaseClient.cpp index bbdd1178842..6e870bead63 100644 --- a/src/kvstore/plugins/hbase/HBaseClient.cpp +++ b/src/kvstore/plugins/hbase/HBaseClient.cpp @@ -188,7 +188,7 @@ ResultCode HBaseClient::range(const std::string& tableName, while (true) { std::vector tResultList; client_->sync_getScannerRows(tResultList, scannerId, kScanRowNum); - if (tResultList.size() == 0) break; + if (tResultList.empty()) break; for (auto& tResult : tResultList) { std::vector tColumnValueList = tResult.columnValues; if (tColumnValueList.size() > 0) { diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index ee0025d35aa..4d3447d4477 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -114,12 +114,14 @@ nebula_add_library( add_dependencies( meta_version_man_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) set(meta_test_deps $ $ $ + $ $ $ $ diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index dd61af74c2e..68f139a2a31 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -5,10 +5,12 @@ #include "meta/MetaVersionMan.h" +#include "meta/ActiveHostsMan.h" #include "meta/processors/job/JobDescription.h" #include "meta/processors/job/JobUtils.h" #include "meta/upgrade/MetaDataUpgrade.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DEFINE_bool(null_type, true, "set schema to support null type"); DEFINE_bool(print_info, false, "enable to print the rewrite data"); @@ -26,7 +28,7 @@ MetaVersion MetaVersionMan::getMetaVersionFromKV(kvstore::KVStore* kv) { auto code = kv->get(kDefaultSpaceId, kDefaultPartId, kMetaVersionKey, &value, true); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { auto version = *reinterpret_cast(value.data()); - return (version == MetaVersion::V2) ? MetaVersion::V2 : MetaVersion::UNKNOWN; + return version; } else { return getVersionByHost(kv); } @@ -42,19 +44,18 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { } if (iter->valid()) { auto v1KeySize = hostPrefix.size() + sizeof(int64_t); - return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V2; + return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3; } - // No hosts exists, regard as regard as version 2 - return MetaVersion::V2; + // No hosts exists, regard as version 3 + return MetaVersion::V3; } // static -bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { +bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version) { CHECK_NOTNULL(kv); - auto v2 = MetaVersion::V2; std::vector data; data.emplace_back(kMetaVersionKey, - std::string(reinterpret_cast(&v2), sizeof(MetaVersion))); + std::string(reinterpret_cast(&version), sizeof(MetaVersion))); bool ret = true; folly::Baton baton; kv->asyncMultiPut( @@ -63,7 +64,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { LOG(ERROR) << "Put failed, error: " << static_cast(code); ret = false; } else { - LOG(INFO) << "Write meta version 2 succeeds"; + LOG(INFO) << "Write meta version 3 succeeds"; } baton.post(); }); @@ -80,7 +81,28 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgrade(kv); + auto status = doUpgradeV1ToV2(kv); + if (!status.ok()) { + // rollback by snapshot + return status; + } + // delete snapshot file + auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); + if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; + } + return Status::OK(); +} + +Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVStore* kv) { + CHECK_NOTNULL(kv); + auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); + auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); + if (meteRet.isLeftType()) { + LOG(ERROR) << "Create snapshot failed: " << snapshot; + return Status::Error("Create snapshot failed"); + } + auto status = doUpgradeV2ToV3(kv); if (!status.ok()) { // rollback by snapshot return status; @@ -94,7 +116,7 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { } // static -Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { +Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { MetaDataUpgrade upgrader(kv); { // kSpacesTable @@ -105,7 +127,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { Status status = Status::OK(); while (iter->valid()) { if (FLAGS_print_info) { - upgrader.printSpaces(iter->val()); + upgrader.printSpacesV1(iter->val()); } status = upgrader.rewriteSpaces(iter->key(), iter->val()); if (!status.ok()) { @@ -158,6 +180,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kLeadersTable auto prefix = nebula::meta::v1::kLeadersTable; @@ -178,6 +201,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kTagsTable auto prefix = nebula::meta::v1::kTagsTable; @@ -327,7 +351,95 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } - if (!setMetaVersionToKV(kv)) { + if (!setMetaVersionToKV(kv, MetaVersion::V2)) { + return Status::Error("Persist meta version failed"); + } else { + return Status::OK(); + } +} + +Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { + MetaDataUpgrade upgrader(kv); + // Step 1: Upgrade HeartBeat into machine list + { + // collect all hosts association with zone + std::vector zoneHosts; + const auto& zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr zoneIter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &zoneIter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + while (zoneIter->valid()) { + auto hosts = MetaKeyUtils::parseZoneHosts(zoneIter->val()); + if (!hosts.empty()) { + zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end()); + } + zoneIter->next(); + } + + const auto& prefix = MetaKeyUtils::hostPrefix(); + std::unique_ptr iter; + code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + std::vector data; + while (iter->valid()) { + auto info = HostInfo::decode(iter->val()); + + if (info.role_ == meta::cpp2::HostRole::STORAGE) { + // Save the machine information + auto host = MetaKeyUtils::parseHostKey(iter->key()); + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + data.emplace_back(std::move(machineKey), ""); + + auto hostIt = std::find(zoneHosts.begin(), zoneHosts.end(), host); + if (hostIt == zoneHosts.end()) { + // Save the zone information + auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); + auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName)); + auto zoneVal = MetaKeyUtils::zoneVal({host}); + data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + } + } + iter->next(); + } + auto status = upgrader.saveMachineAndZone(std::move(data)); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + } + + // Step 2: Update Create space properties about Group + { + const auto& prefix = MetaKeyUtils::spacePrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get spaces failed"; + return Status::Error("Get spaces failed"); + } + + while (iter->valid()) { + if (FLAGS_print_info) { + upgrader.printSpacesV2(iter->val()); + } + auto spaceProperties = meta::v2::MetaServiceUtilsV2::parseSpace(iter->val()); + auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + iter->next(); + } + } + if (!setMetaVersionToKV(kv, MetaVersion::V3)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 59737175430..c6e31b8d01d 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -17,6 +17,7 @@ enum class MetaVersion { UNKNOWN = 0, V1 = 1, V2 = 2, + V3 = 3, }; /** @@ -28,17 +29,21 @@ class MetaVersionMan final { static MetaVersion getMetaVersionFromKV(kvstore::KVStore* kv); - static bool setMetaVersionToKV(kvstore::KVStore* kv); + static bool setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version); static Status updateMetaV1ToV2(kvstore::KVStore* kv); + static Status updateMetaV2ToV3(kvstore::KVStore* kv); + private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgrade(kvstore::KVStore* kv); + static Status doUpgradeV1ToV2(kvstore::KVStore* kv); + + static Status doUpgradeV2ToV3(kvstore::KVStore* kv); }; } // namespace meta } // namespace nebula -#endif // META_ROOTUSERMAN_H_ +#endif // META_METAVERSIONMAN_H_ diff --git a/src/meta/processors/admin/ListClusterInfoProcessor.cpp b/src/meta/processors/admin/ListClusterInfoProcessor.cpp index 3ebd993d35f..3c291fa9ca8 100644 --- a/src/meta/processors/admin/ListClusterInfoProcessor.cpp +++ b/src/meta/processors/admin/ListClusterInfoProcessor.cpp @@ -37,8 +37,8 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { } auto iter = nebula::value(iterRet).get(); for (; iter->valid(); iter->next()) { - HostAddr addr = MetaKeyUtils::parseHostKey(iter->key()); - HostInfo info = HostInfo::decode(iter->val()); + auto addr = MetaKeyUtils::parseHostKey(iter->key()); + auto info = HostInfo::decode(iter->val()); cpp2::ServiceInfo service; service.role_ref() = info.role_; diff --git a/src/meta/processors/job/DataBalanceJobExecutor.cpp b/src/meta/processors/job/DataBalanceJobExecutor.cpp index d125bb2ca25..e2abaca40fc 100644 --- a/src/meta/processors/job/DataBalanceJobExecutor.cpp +++ b/src/meta/processors/job/DataBalanceJobExecutor.cpp @@ -99,7 +99,7 @@ Status DataBalanceJobExecutor::buildBalancePlan() { for (Host* h : hostVec) { totalPartNum += h->parts_.size(); } - if (hostVec.size() == 0) { + if (hostVec.empty()) { LOG(ERROR) << "rebalance error: zone has no host"; return {}; } diff --git a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp index 0d1833247ce..a152bd02d75 100644 --- a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp +++ b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp @@ -121,7 +121,7 @@ nebula::cpp2::ErrorCode ZoneBalanceJobExecutor::rebalanceActiveZones( for (auto& z : sortedActiveZonesRef) { totalPartNum += z->partNum_; } - if (sortedActiveZonesRef.size() == 0) { + if (sortedActiveZonesRef.empty()) { LOG(ERROR) << "rebalance error: no active zones"; return nebula::cpp2::ErrorCode::E_NO_HOSTS; } diff --git a/src/meta/processors/zone/ListZonesProcessor.cpp b/src/meta/processors/zone/ListZonesProcessor.cpp index 61a1b07d2ff..282468ecf13 100644 --- a/src/meta/processors/zone/ListZonesProcessor.cpp +++ b/src/meta/processors/zone/ListZonesProcessor.cpp @@ -27,7 +27,7 @@ void ListZonesProcessor::process(const cpp2::ListZonesReq&) { auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); cpp2::Zone zone; zone.zone_name_ref() = std::move(zoneName); - if (hosts.size() != 0) { + if (!hosts.empty()) { zone.nodes_ref() = std::move(hosts); } else { zone.nodes_ref() = {HostAddr("", 0)}; diff --git a/src/meta/processors/zone/MergeZoneProcessor.cpp b/src/meta/processors/zone/MergeZoneProcessor.cpp index c390211f50e..fe8f1f227bd 100644 --- a/src/meta/processors/zone/MergeZoneProcessor.cpp +++ b/src/meta/processors/zone/MergeZoneProcessor.cpp @@ -147,7 +147,6 @@ void MergeZoneProcessor::process(const cpp2::MergeZoneReq& req) { auto id = MetaKeyUtils::spaceId(iter->key()); auto properties = MetaKeyUtils::parseSpace(iter->val()); auto spaceZones = properties.get_zone_names(); - bool replacement = false; for (auto& zone : zones) { auto it = std::find(spaceZones.begin(), spaceZones.end(), zone); diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index 8f66fad305b..b5675e2d110 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -1208,6 +1208,7 @@ TEST(MetaClientTest, DiffTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { TestUtils::registerHB(kv, {{"0", 0}}); } meta::MetaClientOptions options; options.localHost_ = {"0", 0}; @@ -1370,19 +1371,14 @@ TEST(MetaClientTest, ListenerDiffTest) { TEST(MetaClientTest, HeartbeatTest) { FLAGS_heartbeat_interval_secs = 1; - // const nebula::ClusterID kClusterId = 10; fs::TempDir rootPath("/tmp/HeartbeatTest.XXXXXX"); mock::MockCluster cluster; cluster.startMeta(rootPath.path()); auto* kv = cluster.metaKV_.get(); - // meta::MetaClientOptions options; TestUtils::createSomeHosts(kv, {{"0", 0}}); HostAddr localHost("0", 0); - // options.localHost_ = localHost; - // options.clusterId_ = kClusterId; - // options.role_ = meta::cpp2::HostRole::STORAGE; cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index ca88c3bb999..7e7f5c3edad 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -3947,7 +3947,6 @@ TEST(ProcessorTest, MergeZoneTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } { - LOG(INFO) << "========================"; cpp2::MergeZoneReq req; req.zones_ref() = {"default_zone_127.0.0.1_8978", "z_1"}; req.zone_name_ref() = "z_1"; diff --git a/src/meta/upgrade/CMakeLists.txt b/src/meta/upgrade/CMakeLists.txt index 7b895c31e36..b0700952b86 100644 --- a/src/meta/upgrade/CMakeLists.txt +++ b/src/meta/upgrade/CMakeLists.txt @@ -6,11 +6,14 @@ nebula_add_library( meta_data_upgrade_obj OBJECT MetaDataUpgrade.cpp v1/MetaServiceUtilsV1.cpp + v2/MetaServiceUtilsV2.cpp ) add_dependencies( meta_data_upgrade_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) nebula_add_subdirectory(v1) +nebula_add_subdirectory(v2) diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index 8bd89806738..e057e8d6f85 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -19,6 +19,7 @@ #include "meta/ActiveHostsMan.h" #include "meta/MetaServiceUtils.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DECLARE_bool(null_type); DECLARE_uint32(string_index_limit); @@ -62,6 +63,68 @@ Status MetaDataUpgrade::rewriteSpaces(const folly::StringPiece &key, return Status::OK(); } +Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, + const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + cpp2::SpaceDesc spaceDesc; + spaceDesc.space_name_ref() = oldProps.get_space_name(); + spaceDesc.partition_num_ref() = oldProps.get_partition_num(); + spaceDesc.replica_factor_ref() = oldProps.get_replica_factor(); + spaceDesc.charset_name_ref() = oldProps.get_charset_name(); + spaceDesc.collate_name_ref() = oldProps.get_collate_name(); + cpp2::ColumnTypeDef def; + auto &type = oldProps.get_vid_type(); + def.type_length_ref() = *type.get_type_length(); + def.type_ref() = convertToPropertyType(type.get_type()); + + if (type.geo_shape_ref().has_value()) { + def.geo_shape_ref() = convertToGeoShape(*type.get_geo_shape()); + } + spaceDesc.vid_type_ref() = std::move(def); + if (oldProps.isolation_level_ref().has_value()) { + if (*oldProps.isolation_level_ref() == nebula::meta::v2::cpp2::IsolationLevel::DEFAULT) { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::DEFAULT; + } else { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::TOSS; + } + } + + if (oldProps.comment_ref().has_value()) { + spaceDesc.comment_ref() = *oldProps.comment_ref(); + } + + if (oldProps.group_name_ref().has_value()) { + auto groupName = *oldProps.group_name_ref(); + auto groupKey = meta::v2::MetaServiceUtilsV2::groupKey(groupName); + std::string zoneValue; + auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, std::move(groupKey), &zoneValue); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Group Failed"); + } + + auto zones = meta::v2::MetaServiceUtilsV2::parseZoneNames(std::move(zoneValue)); + spaceDesc.zone_names_ref() = std::move(zones); + } else { + const auto &zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr iter; + auto code = kv_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Zones Failed"); + } + + std::vector<::std::string> zones; + while (iter->valid()) { + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + zones.emplace_back(std::move(zoneName)); + iter->next(); + } + spaceDesc.zone_names_ref() = std::move(zones); + } + + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::spaceVal(spaceDesc))); + return Status::OK(); +} + Status MetaDataUpgrade::rewriteParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto oldHosts = meta::v1::MetaServiceUtilsV1::parsePartVal(val); std::vector newHosts; @@ -278,6 +341,60 @@ Status MetaDataUpgrade::convertToNewIndexColumns( return Status::OK(); } +nebula::cpp2::PropertyType MetaDataUpgrade::convertToPropertyType( + nebula::meta::v2::cpp2::PropertyType type) { + switch (type) { + case nebula::meta::v2::cpp2::PropertyType::BOOL: + return nebula::cpp2::PropertyType::BOOL; + case nebula::meta::v2::cpp2::PropertyType::INT64: + return nebula::cpp2::PropertyType::INT64; + case nebula::meta::v2::cpp2::PropertyType::VID: + return nebula::cpp2::PropertyType::VID; + case nebula::meta::v2::cpp2::PropertyType::FLOAT: + return nebula::cpp2::PropertyType::FLOAT; + case nebula::meta::v2::cpp2::PropertyType::DOUBLE: + return nebula::cpp2::PropertyType::DOUBLE; + case nebula::meta::v2::cpp2::PropertyType::STRING: + return nebula::cpp2::PropertyType::STRING; + case nebula::meta::v2::cpp2::PropertyType::FIXED_STRING: + return nebula::cpp2::PropertyType::FIXED_STRING; + case nebula::meta::v2::cpp2::PropertyType::INT8: + return nebula::cpp2::PropertyType::INT8; + case nebula::meta::v2::cpp2::PropertyType::INT16: + return nebula::cpp2::PropertyType::INT16; + case nebula::meta::v2::cpp2::PropertyType::INT32: + return nebula::cpp2::PropertyType::INT32; + case nebula::meta::v2::cpp2::PropertyType::TIMESTAMP: + return nebula::cpp2::PropertyType::TIMESTAMP; + case nebula::meta::v2::cpp2::PropertyType::DATE: + return nebula::cpp2::PropertyType::DATE; + case nebula::meta::v2::cpp2::PropertyType::DATETIME: + return nebula::cpp2::PropertyType::DATETIME; + case nebula::meta::v2::cpp2::PropertyType::TIME: + return nebula::cpp2::PropertyType::TIME; + case nebula::meta::v2::cpp2::PropertyType::GEOGRAPHY: + return nebula::cpp2::PropertyType::GEOGRAPHY; + default: + return nebula::cpp2::PropertyType::UNKNOWN; + } +} + +nebula::meta::cpp2::GeoShape MetaDataUpgrade::convertToGeoShape( + nebula::meta::v2::cpp2::GeoShape shape) { + switch (shape) { + case nebula::meta::v2::cpp2::GeoShape::ANY: + return nebula::meta::cpp2::GeoShape::ANY; + case nebula::meta::v2::cpp2::GeoShape::POINT: + return nebula::meta::cpp2::GeoShape::POINT; + case nebula::meta::v2::cpp2::GeoShape::LINESTRING: + return nebula::meta::cpp2::GeoShape::LINESTRING; + case nebula::meta::v2::cpp2::GeoShape::POLYGON: + return nebula::meta::cpp2::GeoShape::POLYGON; + default: + LOG(FATAL) << "Unimplemented"; + } +} + void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::StringPiece &val) { auto host = meta::v1::MetaServiceUtilsV1::parseHostKey(key); auto info = HostInfo::decodeV1(val); @@ -288,7 +405,7 @@ void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::Stri LOG(INFO) << "Host info: gitInfoSha_: " << info.gitInfoSha_; } -void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { +void MetaDataUpgrade::printSpacesV1(const folly::StringPiece &val) { auto oldProps = meta::v1::MetaServiceUtilsV1::parseSpace(val); LOG(INFO) << "Space name: " << oldProps.get_space_name(); LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); @@ -297,6 +414,18 @@ void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); } +void MetaDataUpgrade::printSpacesV2(const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + LOG(INFO) << "Space name: " << oldProps.get_space_name(); + LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); + LOG(INFO) << "Replica factor: " << oldProps.get_replica_factor(); + LOG(INFO) << "Charset name: " << oldProps.get_charset_name(); + LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); + if (oldProps.group_name_ref().has_value()) { + LOG(INFO) << "Group name: " << *oldProps.group_name_ref(); + } +} + void MetaDataUpgrade::printParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto spaceId = meta::v1::MetaServiceUtilsV1::parsePartKeySpaceId(key); auto partId = meta::v1::MetaServiceUtilsV1::parsePartKeyPartId(key); @@ -434,5 +563,10 @@ void MetaDataUpgrade::printJobDesc(const folly::StringPiece &key, const folly::S LOG(INFO) << "JobDesc stopTime: " << stopTime; } +Status MetaDataUpgrade::saveMachineAndZone(std::vector data) { + NG_LOG_AND_RETURN_IF_ERROR(put(data)); + return Status::OK(); +} + } // namespace meta } // namespace nebula diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index dbea1afcbfc..bd4ffea08a1 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -14,6 +14,7 @@ #include "kvstore/KVStore.h" #include "meta/processors/Common.h" #include "meta/upgrade/v1/gen-cpp2/meta_types.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" namespace nebula { namespace meta { @@ -33,10 +34,15 @@ class MetaDataUpgrade final { Status rewriteConfigs(const folly::StringPiece &key, const folly::StringPiece &val); Status rewriteJobDesc(const folly::StringPiece &key, const folly::StringPiece &val); + Status rewriteSpacesV2ToV3(const folly::StringPiece &key, const folly::StringPiece &val); + Status deleteKeyVal(const folly::StringPiece &key); + Status saveMachineAndZone(std::vector data); + void printHost(const folly::StringPiece &key, const folly::StringPiece &val); - void printSpaces(const folly::StringPiece &val); + void printSpacesV1(const folly::StringPiece &val); + void printSpacesV2(const folly::StringPiece &val); void printParts(const folly::StringPiece &key, const folly::StringPiece &val); void printLeaders(const folly::StringPiece &key); void printSchemas(const folly::StringPiece &val); @@ -67,6 +73,26 @@ class MetaDataUpgrade final { return Status::OK(); } + Status put(std::vector data) { + folly::Baton baton; + auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; + kv_->asyncMultiPut(kDefaultSpaceId, + kDefaultPartId, + std::move(data), + [&ret, &baton](nebula::cpp2::ErrorCode code) { + if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { + ret = code; + LOG(INFO) << "Put data error on meta server"; + } + baton.post(); + }); + baton.wait(); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Put data failed"); + } + return Status::OK(); + } + Status remove(const folly::StringPiece &key) { std::vector keys{key.str()}; folly::Baton baton; @@ -94,6 +120,10 @@ class MetaDataUpgrade final { Status convertToNewIndexColumns(const std::vector &oldCols, std::vector &newCols); + nebula::cpp2::PropertyType convertToPropertyType(nebula::meta::v2::cpp2::PropertyType type); + + nebula::meta::cpp2::GeoShape convertToGeoShape(nebula::meta::v2::cpp2::GeoShape shape); + private: kvstore::KVStore *kv_ = nullptr; }; diff --git a/src/meta/upgrade/v2/CMakeLists.txt b/src/meta/upgrade/v2/CMakeLists.txt new file mode 100644 index 00000000000..df5d9d2549f --- /dev/null +++ b/src/meta/upgrade/v2/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(THRIFT1 ${Fbthrift_BIN}) + +set(meta_v2_sources + gen-cpp2/meta_constants.cpp + gen-cpp2/meta_data.cpp + gen-cpp2/meta_metadata.cpp + gen-cpp2/meta_types.cpp + ) + +add_custom_command( + OUTPUT ${meta_v2_sources} + COMMAND "${THRIFT1}" "--strict" "--allow-neg-enum-vals" "--gen" "mstch_cpp2:include_prefix=\"meta/upgrade/v2\",process_in_event_base,stack_arguments" "-o" "." "${CMAKE_CURRENT_SOURCE_DIR}/meta.thrift" + DEPENDS meta.thrift + ) + +nebula_add_library(meta_v2_thrift_obj OBJECT ${meta_v2_sources}) + +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-pedantic") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-extra") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-deprecated-declarations") diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp new file mode 100644 index 00000000000..bb23cd29c74 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" + +#include +#include + +namespace nebula::meta::v2 { + +std::string MetaServiceUtilsV2::groupKey(const std::string& group) { + std::string key; + key.reserve(kGroupsTable.size() + group.size()); + key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); + return key; +} + +std::vector MetaServiceUtilsV2::parseZoneNames(folly::StringPiece rawData) { + std::vector zones; + folly::split(',', rawData.str(), zones); + return zones; +} + +cpp2::SpaceDesc MetaServiceUtilsV2::parseSpace(folly::StringPiece rawData) { + cpp2::SpaceDesc spaceDesc; + apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); + return spaceDesc; +} + +} // namespace nebula::meta::v2 diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.h b/src/meta/upgrade/v2/MetaServiceUtilsV2.h new file mode 100644 index 00000000000..47bf010ce98 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ +#define TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" +#include "common/thrift/ThriftTypes.h" +#include "interface/gen-cpp2/meta_types.h" +#include "kvstore/Common.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" + +namespace nebula::meta::v2 { + +const std::string kGroupsTable = "__groups__"; // NOLINT + +class MetaServiceUtilsV2 final { + public: + MetaServiceUtilsV2() = delete; + + static std::string groupKey(const std::string& group); + + static std::vector parseZoneNames(folly::StringPiece rawData); + + static cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); +}; + +} // namespace nebula::meta::v2 + +#endif // TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ diff --git a/src/meta/upgrade/v2/meta.thrift b/src/meta/upgrade/v2/meta.thrift new file mode 100644 index 00000000000..d46922478b8 --- /dev/null +++ b/src/meta/upgrade/v2/meta.thrift @@ -0,0 +1,81 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +namespace cpp nebula.meta.v2 + +cpp_include "common/thrift/ThriftTypes.h" + +typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID +typedef i32 (cpp.type = "nebula::PartitionID") PartitionID +typedef i32 (cpp.type = "nebula::TagID") TagID +typedef i32 (cpp.type = "nebula::EdgeType") EdgeType +typedef i64 (cpp.type = "nebula::EdgeRanking") EdgeRanking +typedef i64 (cpp.type = "nebula::VertexID") VertexID +typedef i32 (cpp.type = "nebula::IndexID") IndexID + +typedef i32 IPv4 +typedef i32 (cpp.type = "nebula::Port") Port + +typedef i64 (cpp.type = "nebula::SchemaVer") SchemaVer + +struct SpaceDesc { + 1: binary space_name, + 2: i32 partition_num = 0, + 3: i32 replica_factor = 0, + 4: binary charset_name, + 5: binary collate_name, + 6: ColumnTypeDef vid_type = {"type": PropertyType.FIXED_STRING, "type_length": 8}, + 7: optional binary group_name, + 8: optional IsolationLevel isolation_level, + 9: optional binary comment, +} + +enum PropertyType { + UNKNOWN = 0, + + // Simple types + BOOL = 1, + INT64 = 2, // This is the same as INT in v1 + VID = 3, // Deprecated, only supported by v1 + FLOAT = 4, + DOUBLE = 5, + STRING = 6, + // String with fixed length. If the string content is shorteri + // than the given length, '\0' will be padded to the end + FIXED_STRING = 7, // New in v2 + INT8 = 8, // New in v2 + INT16 = 9, // New in v2 + INT32 = 10, // New in v2 + + // Date time + TIMESTAMP = 21, + DATE = 24, + DATETIME = 25, + TIME = 26, + + // Geo spatial + GEOGRAPHY = 31, +} (cpp.enum_strict) + +// Geo shape type +enum GeoShape { + ANY = 0, + POINT = 1, + LINESTRING = 2, + POLYGON = 3, +} (cpp.enum_strict) + +struct ColumnTypeDef { + 1: required PropertyType type, + // type_length is valid for fixed_string type + 2: optional i16 type_length = 0, + // geo_shape is valid for geography type + 3: optional GeoShape geo_shape, +} + +enum IsolationLevel { + DEFAULT = 0x00, // allow add half edge(either in or out edge succeeded) + TOSS = 0x01, // add in and out edge atomic +} (cpp.enum_strict) \ No newline at end of file diff --git a/src/storage/http/StorageHttpDownloadHandler.cpp b/src/storage/http/StorageHttpDownloadHandler.cpp index 30b2cdfa169..ed15bc77f50 100644 --- a/src/storage/http/StorageHttpDownloadHandler.cpp +++ b/src/storage/http/StorageHttpDownloadHandler.cpp @@ -94,7 +94,7 @@ void StorageHttpDownloadHandler::onEOM() noexcept { if (helper_->checkHadoopPath()) { std::vector parts; folly::split(",", partitions_, parts, true); - if (parts.size() == 0) { + if (parts.empty()) { ResponseBuilder(downstream_) .status(400, "SSTFile download failed") .body("Partitions should be not empty") diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 44604cfbac2..52e9c352dc8 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -2,6 +2,7 @@ set(storage_test_deps $ $ $ + $ $ $ $ diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 80919a420a6..c489c751a6f 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -1,6 +1,7 @@ set(tools_test_deps $ $ + $ $ $ $ diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 22c951d3ad9..674b8973910 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -10,6 +10,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 466b91ef62d..1e827f5dfce 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 81de926c583..01e01040542 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 41f9b0a18a5..bdde2f7bcdb 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -1,6 +1,7 @@ set(perf_test_deps $ $ + $ $ $ $