From 349b685042d8415dc0fb7abb4052393e4a31f886 Mon Sep 17 00:00:00 2001 From: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:25:11 +0800 Subject: [PATCH 1/4] Separate Metakey Utils from meta service utils (#3176) Co-authored-by: nebula-bots <88429921+nebula-bots@users.noreply.github.com> Co-authored-by: CBS <56461666+bright-starry-sky@users.noreply.github.com> --- src/common/utils/CMakeLists.txt | 6 + src/common/utils/MetaKeyUtils.cpp | 1185 +++++++++++++++++ src/common/utils/MetaKeyUtils.h | 393 ++++++ src/common/utils/test/CMakeLists.txt | 28 + .../utils/test/MetaKeyUtilsTest.cpp} | 104 +- src/daemons/CMakeLists.txt | 1 + src/daemons/MetaDaemon.cpp | 8 +- src/kvstore/test/CMakeLists.txt | 1 + src/meta/ActiveHostsMan.cpp | 33 +- src/meta/ActiveHostsMan.h | 2 +- src/meta/CMakeLists.txt | 1 + src/meta/MetaServiceHandler.cpp | 2 +- src/meta/MetaServiceUtils.cpp | 1184 +--------------- src/meta/MetaServiceUtils.h | 318 ----- src/meta/MetaVersionMan.cpp | 9 +- src/meta/MetaVersionMan.h | 2 +- src/meta/RootUserMan.h | 12 +- src/meta/common/MetaCommon.h | 1 + src/meta/http/MetaHttpDownloadHandler.cpp | 6 +- src/meta/http/MetaHttpIngestHandler.cpp | 6 +- src/meta/http/MetaHttpReplaceHostHandler.cpp | 12 +- .../http/test/MetaHttpReplaceHandlerTest.cpp | 8 +- src/meta/processors/BaseProcessor-inl.h | 44 +- src/meta/processors/BaseProcessor.h | 1 + src/meta/processors/Common.h | 38 - src/meta/processors/admin/AdminClient.cpp | 8 +- src/meta/processors/admin/BalancePlan.cpp | 13 +- src/meta/processors/admin/BalanceTask.cpp | 4 +- src/meta/processors/admin/Balancer.cpp | 52 +- .../admin/CreateBackupProcessor.cpp | 24 +- .../admin/CreateSnapshotProcessor.cpp | 12 +- .../admin/DropSnapshotProcessor.cpp | 14 +- .../admin/ListClusterInfoProcessor.cpp | 6 +- .../admin/ListSnapshotsProcessor.cpp | 8 +- .../processors/admin/RestoreProcessor.cpp | 18 +- src/meta/processors/admin/SnapShot.cpp | 8 +- .../processors/config/GetConfigProcessor.cpp | 4 +- .../config/ListConfigsProcessor.cpp | 6 +- .../processors/config/RegConfigProcessor.cpp | 4 +- .../processors/config/SetConfigProcessor.cpp | 6 +- .../index/CreateEdgeIndexProcessor.cpp | 8 +- .../index/CreateTagIndexProcessor.cpp | 8 +- .../index/DropEdgeIndexProcessor.cpp | 4 +- .../index/DropTagIndexProcessor.cpp | 4 +- .../processors/index/FTIndexProcessor.cpp | 23 +- .../processors/index/FTServiceProcessor.cpp | 10 +- .../index/GetEdgeIndexProcessor.cpp | 4 +- .../processors/index/GetTagIndexProcessor.cpp | 4 +- .../index/ListEdgeIndexesProcessor.cpp | 4 +- .../index/ListTagIndexesProcessor.cpp | 4 +- .../processors/job/BalanceJobExecutor.cpp | 2 +- src/meta/processors/job/GetStatsProcessor.cpp | 4 +- src/meta/processors/job/JobManager.cpp | 4 +- src/meta/processors/job/MetaJobExecutor.cpp | 22 +- .../processors/job/RebuildJobExecutor.cpp | 4 +- .../job/SimpleConcurrentJobExecutor.cpp | 2 +- src/meta/processors/job/StatsJobExecutor.cpp | 20 +- src/meta/processors/job/TaskDescription.cpp | 6 +- src/meta/processors/kv/GetProcessor.cpp | 2 +- src/meta/processors/kv/MultiGetProcessor.cpp | 2 +- src/meta/processors/kv/MultiPutProcessor.cpp | 3 +- src/meta/processors/kv/RemoveProcessor.cpp | 2 +- .../processors/kv/RemoveRangeProcessor.cpp | 4 +- src/meta/processors/kv/ScanProcessor.cpp | 4 +- .../processors/listener/ListenerProcessor.cpp | 18 +- .../parts/CreateSpaceAsProcessor.cpp | 45 +- .../processors/parts/CreateSpaceProcessor.cpp | 22 +- .../processors/parts/DropSpaceProcessor.cpp | 22 +- .../parts/GetPartsAllocProcessor.cpp | 12 +- .../processors/parts/GetSpaceProcessor.cpp | 4 +- .../processors/parts/ListHostsProcessor.cpp | 22 +- .../processors/parts/ListPartsProcessor.cpp | 12 +- .../processors/parts/ListSpacesProcessor.cpp | 6 +- .../processors/schema/AlterEdgeProcessor.cpp | 10 +- .../processors/schema/AlterTagProcessor.cpp | 10 +- .../processors/schema/CreateEdgeProcessor.cpp | 6 +- .../processors/schema/CreateTagProcessor.cpp | 6 +- .../processors/schema/DropEdgeProcessor.cpp | 4 +- .../processors/schema/DropTagProcessor.cpp | 4 +- .../processors/schema/GetEdgeProcessor.cpp | 6 +- .../processors/schema/GetTagProcessor.cpp | 6 +- .../processors/schema/ListEdgesProcessor.cpp | 6 +- .../processors/schema/ListTagsProcessor.cpp | 6 +- .../session/SessionManagerProcessor.cpp | 28 +- .../user/AuthenticationProcessor.cpp | 36 +- .../processors/zone/AddGroupProcessor.cpp | 14 +- src/meta/processors/zone/AddZoneProcessor.cpp | 10 +- .../processors/zone/DropGroupProcessor.cpp | 8 +- .../processors/zone/DropZoneProcessor.cpp | 10 +- .../processors/zone/GetGroupProcessor.cpp | 4 +- src/meta/processors/zone/GetZoneProcessor.cpp | 4 +- .../processors/zone/ListGroupsProcessor.cpp | 6 +- .../processors/zone/ListZonesProcessor.cpp | 6 +- .../processors/zone/UpdateGroupProcessor.cpp | 20 +- .../processors/zone/UpdateZoneProcessor.cpp | 30 +- src/meta/test/ActiveHostsManTest.cpp | 12 +- src/meta/test/AdminClientTest.cpp | 2 +- src/meta/test/AuthProcessorTest.cpp | 6 +- src/meta/test/BalancerTest.cpp | 26 +- src/meta/test/CMakeLists.txt | 25 - src/meta/test/CreateBackupProcessorTest.cpp | 20 +- src/meta/test/GetStatsTest.cpp | 24 +- src/meta/test/IndexProcessorTest.cpp | 18 +- src/meta/test/MetaClientTest.cpp | 2 +- src/meta/test/ProcessorTest.cpp | 12 +- src/meta/test/RestoreProcessorTest.cpp | 118 +- src/meta/test/TestUtils.h | 50 +- src/meta/upgrade/MetaDataUpgrade.cpp | 15 +- src/meta/upgrade/MetaDataUpgrade.h | 1 + src/storage/test/CMakeLists.txt | 1 + src/tools/db-dump/CMakeLists.txt | 1 + src/tools/db-upgrade/CMakeLists.txt | 1 + src/tools/meta-dump/CMakeLists.txt | 1 + src/tools/meta-dump/MetaDumpTool.cpp | 42 +- src/tools/simple-kv-verify/CMakeLists.txt | 1 + src/tools/storage-perf/CMakeLists.txt | 1 + 116 files changed, 2301 insertions(+), 2227 deletions(-) create mode 100644 src/common/utils/MetaKeyUtils.cpp create mode 100644 src/common/utils/MetaKeyUtils.h rename src/{meta/test/MetaServiceUtilsTest.cpp => common/utils/test/MetaKeyUtilsTest.cpp} (62%) diff --git a/src/common/utils/CMakeLists.txt b/src/common/utils/CMakeLists.txt index ce8ff1ff48a..f240cb25f5b 100644 --- a/src/common/utils/CMakeLists.txt +++ b/src/common/utils/CMakeLists.txt @@ -7,4 +7,10 @@ nebula_add_library( OperationKeyUtils.cpp ) +nebula_add_library( + meta_keyutils_obj OBJECT + MetaKeyUtils.cpp +) + + nebula_add_subdirectory(test) diff --git a/src/common/utils/MetaKeyUtils.cpp b/src/common/utils/MetaKeyUtils.cpp new file mode 100644 index 00000000000..07445e07146 --- /dev/null +++ b/src/common/utils/MetaKeyUtils.cpp @@ -0,0 +1,1185 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "common/utils/MetaKeyUtils.h" + +#include +#include +#include + +#include + +#include "common/network/NetworkUtils.h" + +namespace nebula { + +// Systemtable means that it does not contain any space information(id).false +// means that the backup should be skipped. +static const std::unordered_map> systemTableMaps = { + {"users", {"__users__", true}}, + {"hosts", {"__hosts__", false}}, + {"snapshots", {"__snapshots__", false}}, + {"configs", {"__configs__", true}}, + {"groups", {"__groups__", true}}, + {"zones", {"__zones__", true}}, + {"ft_service", {"__ft_service__", false}}, + {"sessions", {"__sessions__", true}}}; + +// SystemInfo will always be backuped +static const std::unordered_map> systemInfoMaps{ + {"autoIncrementId", {"__id__", true}}, {"lastUpdateTime", {"__last_update_time__", true}}}; + +// name => {prefix, parseSpaceid}, nullptr means that the backup should be skipped. +static const std::unordered_map< + std::string, + std::pair>> + tableMaps = {{"spaces", {"__spaces__", MetaKeyUtils::spaceId}}, + {"parts", {"__parts__", MetaKeyUtils::parsePartKeySpaceId}}, + {"tags", {"__tags__", MetaKeyUtils::parseTagsKeySpaceID}}, + {"edges", {"__edges__", MetaKeyUtils::parseEdgesKeySpaceID}}, + {"indexes", {"__indexes__", MetaKeyUtils::parseIndexesKeySpaceID}}, + // Index tables are handled separately. + {"index", {"__index__", nullptr}}, + {"index_status", {"__index_status__", MetaKeyUtils::parseIndexStatusKeySpaceID}}, + {"roles", {"__roles__", MetaKeyUtils::parseRoleSpace}}, + {"leaders", {"__leaders__", nullptr}}, + {"leader_terms", {"__leader_terms__", nullptr}}, + {"listener", {"__listener__", nullptr}}, + {"stats", {"__stats__", MetaKeyUtils::parseStatsSpace}}, + {"balance_task", {"__balance_task__", nullptr}}, + {"balance_plan", {"__balance_plan__", nullptr}}, + {"ft_index", {"__ft_index__", nullptr}}, + {"local_id", {"__local_id__", MetaKeyUtils::parseLocalIdSpace}}}; + +// clang-format off +static const std::string kSpacesTable = tableMaps.at("spaces").first; // NOLINT +static const std::string kPartsTable = tableMaps.at("parts").first; // NOLINT +static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT +static const std::string kTagsTable = tableMaps.at("tags").first; // NOLINT +static const std::string kEdgesTable = tableMaps.at("edges").first; // NOLINT +static const std::string kIndexesTable = tableMaps.at("indexes").first; // NOLINT +static const std::string kIndexTable = tableMaps.at("index").first; // NOLINT +static const std::string kIndexStatusTable = tableMaps.at("index_status").first; // NOLINT +static const std::string kUsersTable = systemTableMaps.at("users").first; // NOLINT +static const std::string kRolesTable = tableMaps.at("roles").first; // NOLINT +static const std::string kConfigsTable = systemTableMaps.at("configs").first; // NOLINT +static const std::string kSnapshotsTable = systemTableMaps.at("snapshots").first; // NOLINT +static const std::string kLeadersTable = tableMaps.at("leaders").first; // NOLINT +static const std::string kLeaderTermsTable = tableMaps.at("leader_terms").first; // NOLINT +static const std::string kGroupsTable = systemTableMaps.at("groups").first; // NOLINT +static const std::string kZonesTable = systemTableMaps.at("zones").first; // NOLINT +static const std::string kListenerTable = tableMaps.at("listener").first; // NOLINT + +// Used to record the number of vertices and edges in the space +// The number of vertices of each tag in the space +// The number of edges of each edgetype in the space +static const std::string kStatsTable = tableMaps.at("stats").first; // NOLINT +static const std::string kBalanceTaskTable = tableMaps.at("balance_task").first; // NOLINT +static const std::string kBalancePlanTable = tableMaps.at("balance_plan").first; // NOLINT +static const std::string kLocalIdTable = tableMaps.at("local_id").first; // NOLINT + +const std::string kFTIndexTable = tableMaps.at("ft_index").first; // NOLINT +const std::string kFTServiceTable = systemTableMaps.at("ft_service").first; // NOLINT +const std::string kSessionsTable = systemTableMaps.at("sessions").first; // NOLINT + +const std::string kIdKey = systemInfoMaps.at("autoIncrementId").first; // NOLINT +const std::string kLastUpdateTimeTable = systemInfoMaps.at("lastUpdateTime").first; // NOLINT + +// clang-format on + +const int kMaxIpAddrLen = 15; // '255.255.255.255' + +std::string MetaKeyUtils::getIndexTable() { return tableMaps.at("index").first; } + +std::unordered_map>> +MetaKeyUtils::getTableMaps() { + return tableMaps; +} + +std::unordered_map> MetaKeyUtils::getSystemInfoMaps() { + return systemInfoMaps; +} + +std::unordered_map> MetaKeyUtils::getSystemTableMaps() { + return systemTableMaps; +} + +std::string MetaKeyUtils::lastUpdateTimeKey() { + std::string key; + key.reserve(kLastUpdateTimeTable.size()); + key.append(kLastUpdateTimeTable.data(), kLastUpdateTimeTable.size()); + return key; +} + +std::string MetaKeyUtils::lastUpdateTimeVal(const int64_t timeInMilliSec) { + std::string val; + val.reserve(sizeof(int64_t)); + val.append(reinterpret_cast(&timeInMilliSec), sizeof(int64_t)); + return val; +} + +std::string MetaKeyUtils::spaceKey(GraphSpaceID spaceId) { + std::string key; + key.reserve(kSpacesTable.size() + sizeof(GraphSpaceID)); + key.append(kSpacesTable.data(), kSpacesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +std::string MetaKeyUtils::spaceVal(const meta::cpp2::SpaceDesc& spaceDesc) { + std::string val; + apache::thrift::CompactSerializer::serialize(spaceDesc, &val); + return val; +} + +meta::cpp2::SpaceDesc MetaKeyUtils::parseSpace(folly::StringPiece rawData) { + meta::cpp2::SpaceDesc spaceDesc; + apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); + return spaceDesc; +} + +const std::string& MetaKeyUtils::spacePrefix() { return kSpacesTable; } + +GraphSpaceID MetaKeyUtils::spaceId(folly::StringPiece rawKey) { + return *reinterpret_cast(rawKey.data() + kSpacesTable.size()); +} + +std::string MetaKeyUtils::spaceName(folly::StringPiece rawVal) { + return parseSpace(rawVal).get_space_name(); +} + +std::string MetaKeyUtils::partKey(GraphSpaceID spaceId, PartitionID partId) { + std::string key; + key.reserve(kPartsTable.size() + sizeof(GraphSpaceID) + sizeof(PartitionID)); + key.append(kPartsTable.data(), kPartsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&partId), sizeof(PartitionID)); + return key; +} + +GraphSpaceID MetaKeyUtils::parsePartKeySpaceId(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kPartsTable.size()); +} + +PartitionID MetaKeyUtils::parsePartKeyPartId(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kPartsTable.size() + + sizeof(GraphSpaceID)); +} + +std::string MetaKeyUtils::partVal(const std::vector& hosts) { return partValV2(hosts); } + +// dataVer(int) + vectorSize(size_t) + vector of (strIp(string) + port(int)) +std::string MetaKeyUtils::partValV2(const std::vector& hosts) { + std::string encodedVal; + int dataVersion = 2; + encodedVal.append(reinterpret_cast(&dataVersion), sizeof(int)) + .append(network::NetworkUtils::toHostsStr(hosts)); + return encodedVal; +} + +std::string MetaKeyUtils::partPrefix() { + std::string prefix; + prefix.reserve(kPartsTable.size() + sizeof(GraphSpaceID)); + prefix.append(kPartsTable.data(), kPartsTable.size()); + return prefix; +} + +std::string MetaKeyUtils::partPrefix(GraphSpaceID spaceId) { + std::string prefix; + prefix.reserve(kPartsTable.size() + sizeof(GraphSpaceID)); + prefix.append(kPartsTable.data(), kPartsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return prefix; +} + +std::vector MetaKeyUtils::parsePartVal(folly::StringPiece val, int partNum) { + static const size_t unitSizeV1 = sizeof(int64_t); + if (unitSizeV1 * partNum == val.size()) { + return parsePartValV1(val); + } + int dataVer = *reinterpret_cast(val.data()); + UNUSED(dataVer); // currently if not ver1, it must be v2 + val.advance(sizeof(int)); + return parsePartValV2(val); +} + +// partion val is ip(int) + port(int) +std::vector MetaKeyUtils::parsePartValV1(folly::StringPiece val) { + std::vector hosts; + static const size_t unitSize = sizeof(int32_t) * 2; + auto hostsNum = val.size() / unitSize; + hosts.reserve(hostsNum); + VLOG(3) << "Total size:" << val.size() << ", host size:" << unitSize << ", host num:" << hostsNum; + for (decltype(hostsNum) i = 0; i < hostsNum; i++) { + HostAddr h; + uint32_t ip = *reinterpret_cast(val.data() + i * unitSize); + h.host = network::NetworkUtils::intToIPv4(ip); + h.port = *reinterpret_cast(val.data() + i * unitSize + sizeof(int32_t)); + hosts.emplace_back(std::move(h)); + } + return hosts; +} + +// dataVer(int) + vectorSize(size_t) + vector of (strIpV4(string) + port(int)) +std::vector MetaKeyUtils::parsePartValV2(folly::StringPiece val) { + std::vector ret; + auto hostsOrErr = network::NetworkUtils::toHosts(val.str()); + if (hostsOrErr.ok()) { + ret = std::move(hostsOrErr.value()); + } else { + LOG(ERROR) << "invalid input for parsePartValV2()"; + } + return ret; +} + +std::string MetaKeyUtils::hostKey(std::string addr, Port port) { return hostKeyV2(addr, port); } + +std::string MetaKeyUtils::hostKeyV2(std::string addr, Port port) { + std::string key; + HostAddr h(addr, port); + key.append(kHostsTable.data(), kHostsTable.size()).append(MetaKeyUtils::serializeHostAddr(h)); + return key; +} + +const std::string& MetaKeyUtils::hostPrefix() { return kHostsTable; } + +HostAddr MetaKeyUtils::parseHostKey(folly::StringPiece key) { + if (key.size() == kHostsTable.size() + sizeof(int64_t)) { + return parseHostKeyV1(key); + } + return parseHostKeyV2(key); +} + +HostAddr MetaKeyUtils::parseHostKeyV1(folly::StringPiece key) { + HostAddr host; + key.advance(kHostsTable.size()); + uint32_t ip = *reinterpret_cast(key.begin()); + host.host = network::NetworkUtils::intToIPv4(ip); + host.port = *reinterpret_cast(key.begin() + sizeof(uint32_t)); + return host; +} + +HostAddr MetaKeyUtils::parseHostKeyV2(folly::StringPiece key) { + key.advance(kHostsTable.size()); + return MetaKeyUtils::deserializeHostAddr(key); +} + +std::string MetaKeyUtils::leaderKey(std::string addr, Port port) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + return leaderKeyV2(addr, port); +} + +std::string MetaKeyUtils::leaderKeyV2(std::string addr, Port port) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + std::string key; + HostAddr h(addr, port); + + key.reserve(kLeadersTable.size() + kMaxIpAddrLen + sizeof(Port)); + key.append(kLeadersTable.data(), kLeadersTable.size()).append(MetaKeyUtils::serializeHostAddr(h)); + return key; +} + +std::string MetaKeyUtils::leaderKey(GraphSpaceID spaceId, PartitionID partId) { + std::string key; + key.reserve(kLeaderTermsTable.size() + sizeof(GraphSpaceID) + sizeof(PartitionID)); + key.append(kLeaderTermsTable.data(), kLeaderTermsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&partId), sizeof(PartitionID)); + return key; +} + +std::string MetaKeyUtils::leaderVal(const LeaderParts& leaderParts) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + std::string value; + value.reserve(512); + for (const auto& spaceEntry : leaderParts) { + GraphSpaceID spaceId = spaceEntry.first; + value.append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + size_t leaderCount = spaceEntry.second.size(); + value.append(reinterpret_cast(&leaderCount), sizeof(size_t)); + for (const auto& partId : spaceEntry.second) { + value.append(reinterpret_cast(&partId), sizeof(PartitionID)); + } + } + return value; +} + +// v3: dataVer(int) + lenOfHost(8) + HostAddr(varchar) + term(int64_t) +std::string MetaKeyUtils::leaderValV3(const HostAddr& h, int64_t term) { + std::string leaderVal; + leaderVal.reserve(256); + + int dataVersion = 3; + + auto sHost = serializeHostAddr(h); + auto lenOfHost = sHost.size(); + + leaderVal.append(reinterpret_cast(&dataVersion), sizeof(dataVersion)) + .append(reinterpret_cast(&lenOfHost), sizeof(lenOfHost)) + .append(sHost) + .append(reinterpret_cast(&term), sizeof(term)); + return leaderVal; +} + +// v3: dataVer(int) + lenOfHost(8) + HostAddr(varchar) + term(int64_t) +std::tuple MetaKeyUtils::parseLeaderValV3( + folly::StringPiece val) { + std::tuple ret; + std::get<2>(ret) = nebula::cpp2::ErrorCode::SUCCEEDED; + int dataVer = *reinterpret_cast(val.data()); + if (dataVer != 3) { + std::get<2>(ret) = nebula::cpp2::ErrorCode::E_INVALID_PARM; + return ret; + } + + CHECK_GE(val.size(), sizeof(int)); + val.advance(sizeof(int)); + auto lenOfHost = *reinterpret_cast(val.data()); + + val.advance(sizeof(size_t)); + CHECK_GE(val.size(), lenOfHost); + std::get<0>(ret) = MetaKeyUtils::deserializeHostAddr(val.subpiece(0, lenOfHost)); + + val.advance(lenOfHost); + std::get<1>(ret) = *reinterpret_cast(val.data()); + return ret; +} + +const std::string& MetaKeyUtils::leaderPrefix() { return kLeaderTermsTable; } + +std::string MetaKeyUtils::leaderPrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kLeaderTermsTable.size() + sizeof(GraphSpaceID)); + key.append(kLeaderTermsTable.data(), kLeaderTermsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +HostAddr MetaKeyUtils::parseLeaderKey(folly::StringPiece key) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + if (key.size() == kLeadersTable.size() + sizeof(int64_t)) { + return parseLeaderKeyV1(key); + } + return parseLeaderKeyV2(key); +} + +// input should be a pair of int32_t +HostAddr MetaKeyUtils::parseLeaderKeyV1(folly::StringPiece key) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + HostAddr host; + CHECK_EQ(key.size(), kLeadersTable.size() + sizeof(int64_t)); + key.advance(kLeadersTable.size()); + auto ip = *reinterpret_cast(key.begin()); + host.host = network::NetworkUtils::intToIPv4(ip); + host.port = *reinterpret_cast(key.begin() + sizeof(ip)); + return host; +} + +HostAddr MetaKeyUtils::parseLeaderKeyV2(folly::StringPiece key) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + key.advance(kLeadersTable.size()); + return MetaKeyUtils::deserializeHostAddr(key); +} + +std::pair MetaKeyUtils::parseLeaderKeyV3(folly::StringPiece key) { + std::pair ret; + ret.first = *reinterpret_cast(key.data() + kLeaderTermsTable.size()); + ret.second = *reinterpret_cast(key.data() + kLeaderTermsTable.size() + + sizeof(GraphSpaceID)); + return ret; +} + +LeaderParts MetaKeyUtils::parseLeaderValV1(folly::StringPiece val) { + LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); + LeaderParts leaderParts; + size_t size = val.size(); + // decode leader info + size_t offset = 0; + while (offset + sizeof(GraphSpaceID) + sizeof(size_t) < size) { + GraphSpaceID spaceId = *reinterpret_cast(val.data() + offset); + offset += sizeof(GraphSpaceID); + size_t leaderCount = *reinterpret_cast(val.data() + offset); + offset += sizeof(size_t); + std::vector partIds; + for (size_t i = 0; i < leaderCount && offset < size; i++) { + partIds.emplace_back(*reinterpret_cast(val.data() + offset)); + offset += sizeof(PartitionID); + } + leaderParts.emplace(spaceId, std::move(partIds)); + } + return leaderParts; +} + +std::string MetaKeyUtils::schemaEdgePrefix(GraphSpaceID spaceId, EdgeType edgeType) { + std::string key; + key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(edgeType)); + key.append(kEdgesTable.data(), kEdgesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&edgeType), sizeof(edgeType)); + return key; +} + +std::string MetaKeyUtils::schemaEdgesPrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID)); + key.append(kEdgesTable.data(), kEdgesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +std::string MetaKeyUtils::schemaEdgeKey(GraphSpaceID spaceId, + EdgeType edgeType, + SchemaVer version) { + auto storageVer = std::numeric_limits::max() - version; + std::string key; + key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(EdgeType) + sizeof(SchemaVer)); + key.append(kEdgesTable.data(), kEdgesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&edgeType), sizeof(EdgeType)) + .append(reinterpret_cast(&storageVer), sizeof(SchemaVer)); + return key; +} + +std::string MetaKeyUtils::schemaVal(const std::string& name, const meta::cpp2::Schema& schema) { + auto len = name.size(); + std::string val, sval; + apache::thrift::CompactSerializer::serialize(schema, &sval); + val.reserve(sizeof(int32_t) + name.size() + sval.size()); + val.append(reinterpret_cast(&len), sizeof(int32_t)).append(name).append(sval); + return val; +} + +EdgeType MetaKeyUtils::parseEdgeType(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kEdgesTable.size() + sizeof(GraphSpaceID)); +} + +SchemaVer MetaKeyUtils::parseEdgeVersion(folly::StringPiece key) { + auto offset = kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(EdgeType); + return std::numeric_limits::max() - + *reinterpret_cast(key.begin() + offset); +} + +GraphSpaceID MetaKeyUtils::parseEdgesKeySpaceID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kEdgesTable.size()); +} + +std::string MetaKeyUtils::schemaTagKey(GraphSpaceID spaceId, TagID tagId, SchemaVer version) { + auto storageVer = std::numeric_limits::max() - version; + std::string key; + key.reserve(kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID) + sizeof(SchemaVer)); + key.append(kTagsTable.data(), kTagsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&tagId), sizeof(TagID)) + .append(reinterpret_cast(&storageVer), sizeof(SchemaVer)); + return key; +} + +TagID MetaKeyUtils::parseTagId(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kTagsTable.size() + sizeof(GraphSpaceID)); +} + +SchemaVer MetaKeyUtils::parseTagVersion(folly::StringPiece key) { + auto offset = kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID); + return std::numeric_limits::max() - + *reinterpret_cast(key.begin() + offset); +} + +std::string MetaKeyUtils::schemaTagPrefix(GraphSpaceID spaceId, TagID tagId) { + std::string key; + key.reserve(kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID)); + key.append(kTagsTable.data(), kTagsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&tagId), sizeof(TagID)); + return key; +} + +std::string MetaKeyUtils::schemaTagsPrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kTagsTable.size() + sizeof(GraphSpaceID)); + key.append(kTagsTable.data(), kTagsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +GraphSpaceID MetaKeyUtils::parseTagsKeySpaceID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kTagsTable.size()); +} + +meta::cpp2::Schema MetaKeyUtils::parseSchema(folly::StringPiece rawData) { + meta::cpp2::Schema schema; + int32_t offset = sizeof(int32_t) + *reinterpret_cast(rawData.begin()); + auto schval = rawData.subpiece(offset, rawData.size() - offset); + apache::thrift::CompactSerializer::deserialize(schval, schema); + return schema; +} + +std::string MetaKeyUtils::indexKey(GraphSpaceID spaceID, IndexID indexID) { + std::string key; + key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID) + sizeof(IndexID)); + key.append(kIndexesTable.data(), kIndexesTable.size()) + .append(reinterpret_cast(&spaceID), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&indexID), sizeof(IndexID)); + return key; +} + +std::string MetaKeyUtils::indexVal(const nebula::meta::cpp2::IndexItem& item) { + std::string value; + apache::thrift::CompactSerializer::serialize(item, &value); + return value; +} + +std::string MetaKeyUtils::indexPrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID)); + key.append(kIndexesTable.data(), kIndexesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +GraphSpaceID MetaKeyUtils::parseIndexesKeySpaceID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kIndexesTable.size()); +} + +IndexID MetaKeyUtils::parseIndexesKeyIndexID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kIndexesTable.size() + + sizeof(GraphSpaceID)); +} + +nebula::meta::cpp2::IndexItem MetaKeyUtils::parseIndex(const folly::StringPiece& rawData) { + nebula::meta::cpp2::IndexItem item; + apache::thrift::CompactSerializer::deserialize(rawData, item); + return item; +} + +// This method should replace with JobManager when it ready. +std::string MetaKeyUtils::rebuildIndexStatus(GraphSpaceID space, + char type, + const std::string& indexName) { + std::string key; + key.reserve(64); + key.append(kIndexStatusTable.data(), kIndexStatusTable.size()) + .append(reinterpret_cast(&space), sizeof(GraphSpaceID)) + .append(1, type) + .append(indexName); + return key; +} + +// This method should replace with JobManager when it ready. +std::string MetaKeyUtils::rebuildIndexStatusPrefix(GraphSpaceID space, char type) { + std::string key; + key.reserve(kIndexStatusTable.size() + sizeof(GraphSpaceID) + sizeof(char)); + key.append(kIndexStatusTable.data(), kIndexStatusTable.size()) + .append(reinterpret_cast(&space), sizeof(GraphSpaceID)) + .append(1, type); + return key; +} + +std::string MetaKeyUtils::rebuildIndexStatusPrefix() { + std::string key; + key.reserve(kIndexStatusTable.size()); + key.append(kIndexStatusTable.data(), kIndexStatusTable.size()); + return key; +} + +GraphSpaceID MetaKeyUtils::parseIndexStatusKeySpaceID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kIndexStatusTable.size()); +} + +std::string MetaKeyUtils::indexSpaceKey(const std::string& name) { + EntryType type = EntryType::SPACE; + std::string key; + key.reserve(64); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(type)) + .append(name); + return key; +} + +std::string MetaKeyUtils::indexTagKey(GraphSpaceID spaceId, const std::string& name) { + EntryType type = EntryType::TAG; + std::string key; + key.reserve(128); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(type)) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(name); + return key; +} + +std::string MetaKeyUtils::indexEdgeKey(GraphSpaceID spaceId, const std::string& name) { + EntryType type = EntryType::EDGE; + std::string key; + key.reserve(128); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(type)) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(name); + return key; +} + +GraphSpaceID MetaKeyUtils::parseIndexKeySpaceID(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kIndexTable.size() + + sizeof(EntryType)); +} + +std::string MetaKeyUtils::indexIndexKey(GraphSpaceID spaceID, const std::string& indexName) { + EntryType type = EntryType::INDEX; + std::string key; + key.reserve(128); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(type)) + .append(reinterpret_cast(&spaceID), sizeof(GraphSpaceID)) + .append(indexName); + return key; +} + +std::string MetaKeyUtils::indexGroupKey(const std::string& name) { + EntryType type = EntryType::GROUP; + std::string key; + key.reserve(128); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(EntryType)) + .append(name); + return key; +} + +std::string MetaKeyUtils::indexZoneKey(const std::string& name) { + EntryType type = EntryType::ZONE; + std::string key; + key.reserve(128); + key.append(kIndexTable.data(), kIndexTable.size()) + .append(reinterpret_cast(&type), sizeof(type)) + .append(name); + return key; +} + +std::string MetaKeyUtils::assembleSegmentKey(const std::string& segment, const std::string& key) { + std::string segmentKey; + segmentKey.reserve(64); + segmentKey.append(segment).append(key.data(), key.size()); + return segmentKey; +} + +std::string MetaKeyUtils::userPrefix() { return kUsersTable; } + +std::string MetaKeyUtils::userKey(const std::string& account) { + std::string key; + key.reserve(kUsersTable.size() + account.size()); + key.append(kUsersTable.data(), kUsersTable.size()).append(account); + return key; +} + +std::string MetaKeyUtils::userVal(const std::string& val) { + std::string key; + auto pwdLen = val.size(); + key.reserve(sizeof(int64_t) + pwdLen); + key.append(reinterpret_cast(&pwdLen), sizeof(size_t)).append(val); + return key; +} + +std::string MetaKeyUtils::parseUser(folly::StringPiece key) { + return key.subpiece(kUsersTable.size(), key.size() - kUsersTable.size()).str(); +} + +std::string MetaKeyUtils::parseUserPwd(folly::StringPiece val) { + auto len = *reinterpret_cast(val.data()); + return val.subpiece(sizeof(size_t), len).str(); +} + +std::string MetaKeyUtils::roleKey(GraphSpaceID spaceId, const std::string& account) { + std::string key; + key.reserve(kRolesTable.size() + sizeof(GraphSpaceID) + account.size()); + key.append(kRolesTable.data(), kRolesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(account); + return key; +} + +std::string MetaKeyUtils::roleVal(meta::cpp2::RoleType roleType) { + std::string val; + val.reserve(sizeof(meta::cpp2::RoleType)); + val.append(reinterpret_cast(&roleType), sizeof(meta::cpp2::RoleType)); + return val; +} + +std::string MetaKeyUtils::parseRoleUser(folly::StringPiece key) { + auto offset = kRolesTable.size() + sizeof(GraphSpaceID); + return key.subpiece(offset, key.size() - offset).str(); +} + +GraphSpaceID MetaKeyUtils::parseRoleSpace(folly::StringPiece key) { + return *reinterpret_cast(key.data() + kRolesTable.size()); +} + +std::string MetaKeyUtils::rolesPrefix() { return kRolesTable; } + +std::string MetaKeyUtils::roleSpacePrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kRolesTable.size() + sizeof(GraphSpaceID)); + key.append(kRolesTable.data(), kRolesTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +std::string MetaKeyUtils::parseRoleStr(folly::StringPiece key) { + auto* c = reinterpret_cast(&key); + auto type = *reinterpret_cast(c); + std::string role; + switch (type) { + case meta::cpp2::RoleType::GOD: { + role = "GOD"; + break; + } + case meta::cpp2::RoleType::ADMIN: { + role = "ADMIN"; + break; + } + case meta::cpp2::RoleType::DBA: { + role = "DBA"; + break; + } + case meta::cpp2::RoleType::USER: { + role = "USER"; + break; + } + case meta::cpp2::RoleType::GUEST: { + role = "GUEST"; + break; + } + } + return role; +} + +std::string MetaKeyUtils::configKey(const meta::cpp2::ConfigModule& module, + const std::string& name) { + int32_t nSize = name.size(); + std::string key; + key.reserve(128); + key.append(kConfigsTable.data(), kConfigsTable.size()) + .append(reinterpret_cast(&module), sizeof(meta::cpp2::ConfigModule)) + .append(reinterpret_cast(&nSize), sizeof(int32_t)) + .append(name); + return key; +} + +std::string MetaKeyUtils::configKeyPrefix(const meta::cpp2::ConfigModule& module) { + std::string key; + key.reserve(128); + key.append(kConfigsTable.data(), kConfigsTable.size()); + if (module != meta::cpp2::ConfigModule::ALL) { + key.append(reinterpret_cast(&module), sizeof(meta::cpp2::ConfigModule)); + } + return key; +} + +std::string MetaKeyUtils::configValue(const meta::cpp2::ConfigMode& valueMode, const Value& value) { + std::string val, cVal; + apache::thrift::CompactSerializer::serialize(value, &cVal); + val.reserve(sizeof(meta::cpp2::ConfigMode) + cVal.size()); + val.append(reinterpret_cast(&valueMode), sizeof(meta::cpp2::ConfigMode)) + .append(cVal); + return val; +} + +ConfigName MetaKeyUtils::parseConfigKey(folly::StringPiece rawKey) { + std::string key; + auto offset = kConfigsTable.size(); + auto module = *reinterpret_cast(rawKey.data() + offset); + offset += sizeof(meta::cpp2::ConfigModule); + int32_t nSize = *reinterpret_cast(rawKey.data() + offset); + offset += sizeof(int32_t); + auto name = rawKey.subpiece(offset, nSize); + return {module, name.str()}; +} + +meta::cpp2::ConfigItem MetaKeyUtils::parseConfigValue(folly::StringPiece rawData) { + int32_t offset = 0; + meta::cpp2::ConfigMode mode = + *reinterpret_cast(rawData.data() + offset); + offset += sizeof(meta::cpp2::ConfigMode); + Value value; + apache::thrift::CompactSerializer::deserialize(rawData.subpiece(offset, rawData.size() - offset), + value); + + meta::cpp2::ConfigItem item; + item.set_mode(mode); + item.set_value(value); + return item; +} + +std::string MetaKeyUtils::snapshotKey(const std::string& name) { + std::string key; + key.reserve(kSnapshotsTable.size() + name.size()); + key.append(kSnapshotsTable.data(), kSnapshotsTable.size()).append(name); + return key; +} + +std::string MetaKeyUtils::snapshotVal(const meta::cpp2::SnapshotStatus& status, + const std::string& hosts) { + std::string val; + val.reserve(sizeof(meta::cpp2::SnapshotStatus) + sizeof(hosts)); + val.append(reinterpret_cast(&status), sizeof(meta::cpp2::SnapshotStatus)) + .append(hosts); + return val; +} + +meta::cpp2::SnapshotStatus MetaKeyUtils::parseSnapshotStatus(folly::StringPiece rawData) { + return *reinterpret_cast(rawData.data()); +} + +std::string MetaKeyUtils::parseSnapshotHosts(folly::StringPiece rawData) { + return rawData + .subpiece(sizeof(meta::cpp2::SnapshotStatus), + rawData.size() - sizeof(meta::cpp2::SnapshotStatus)) + .str(); +} + +std::string MetaKeyUtils::parseSnapshotName(folly::StringPiece rawData) { + int32_t offset = kSnapshotsTable.size(); + return rawData.subpiece(offset, rawData.size() - offset).str(); +} + +const std::string& MetaKeyUtils::snapshotPrefix() { return kSnapshotsTable; } + +std::string MetaKeyUtils::serializeHostAddr(const HostAddr& host) { + std::string ret; + ret.reserve(sizeof(size_t) + 15 + sizeof(Port)); // 255.255.255.255 + size_t len = host.host.size(); + ret.append(reinterpret_cast(&len), sizeof(size_t)) + .append(host.host.data(), len) + .append(reinterpret_cast(&host.port), sizeof(Port)); + return ret; +} + +HostAddr MetaKeyUtils::deserializeHostAddr(folly::StringPiece raw) { + HostAddr addr; + CHECK_GE(raw.size(), sizeof(size_t) + sizeof(Port)); // host may be "" + size_t offset = 0; + size_t len = *reinterpret_cast(raw.begin() + offset); + offset += sizeof(size_t); + addr.host = std::string(raw.begin() + offset, len); + offset += len; + addr.port = *reinterpret_cast(raw.begin() + offset); + return addr; +} + +std::string MetaKeyUtils::genTimestampStr() { + char ch[60]; + std::time_t t = std::time(nullptr); + std::strftime(ch, sizeof(ch), "%Y_%m_%d_%H_%M_%S", localtime(&t)); + return ch; +} + +std::string MetaKeyUtils::idKey() { return kIdKey; } + +std::string MetaKeyUtils::balanceTaskKey( + BalanceID balanceId, GraphSpaceID spaceId, PartitionID partId, HostAddr src, HostAddr dst) { + std::string str; + str.reserve(64); + str.append(reinterpret_cast(kBalanceTaskTable.data()), kBalanceTaskTable.size()) + .append(reinterpret_cast(&balanceId), sizeof(BalanceID)) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&partId), sizeof(PartitionID)) + .append(serializeHostAddr(src)) + .append(serializeHostAddr(dst)); + return str; +} + +std::string MetaKeyUtils::balanceTaskVal(BalanceTaskStatus status, + BalanceTaskResult retult, + int64_t startTime, + int64_t endTime) { + std::string val; + val.reserve(32); + val.append(reinterpret_cast(&status), sizeof(BalanceTaskStatus)) + .append(reinterpret_cast(&retult), sizeof(BalanceTaskResult)) + .append(reinterpret_cast(&startTime), sizeof(int64_t)) + .append(reinterpret_cast(&endTime), sizeof(int64_t)); + return val; +} + +std::string MetaKeyUtils::balanceTaskPrefix(BalanceID balanceId) { + std::string prefix; + prefix.reserve(32); + prefix.append(reinterpret_cast(kBalanceTaskTable.data()), kBalanceTaskTable.size()) + .append(reinterpret_cast(&balanceId), sizeof(BalanceID)); + return prefix; +} + +std::string MetaKeyUtils::balancePlanKey(BalanceID id) { + CHECK_GE(id, 0); + // make the balance id is stored in decend order + auto encode = folly::Endian::big(std::numeric_limits::max() - id); + std::string key; + key.reserve(sizeof(BalanceID) + kBalancePlanTable.size()); + key.append(reinterpret_cast(kBalancePlanTable.data()), kBalancePlanTable.size()) + .append(reinterpret_cast(&encode), sizeof(BalanceID)); + return key; +} + +std::string MetaKeyUtils::balancePlanVal(BalanceStatus status) { + std::string val; + val.reserve(sizeof(BalanceStatus)); + val.append(reinterpret_cast(&status), sizeof(BalanceStatus)); + return val; +} + +std::string MetaKeyUtils::balancePlanPrefix() { return kBalancePlanTable; } + +std::tuple +MetaKeyUtils::parseBalanceTaskKey(const folly::StringPiece& rawKey) { + uint32_t offset = kBalanceTaskTable.size(); + auto balanceId = *reinterpret_cast(rawKey.begin() + offset); + offset += sizeof(balanceId); + auto spaceId = *reinterpret_cast(rawKey.begin() + offset); + offset += sizeof(GraphSpaceID); + auto partId = *reinterpret_cast(rawKey.begin() + offset); + offset += sizeof(PartitionID); + auto src = MetaKeyUtils::deserializeHostAddr({rawKey, offset}); + offset += src.host.size() + sizeof(size_t) + sizeof(uint32_t); + auto dst = MetaKeyUtils::deserializeHostAddr({rawKey, offset}); + return std::make_tuple(balanceId, spaceId, partId, src, dst); +} + +std::tuple +MetaKeyUtils::parseBalanceTaskVal(const folly::StringPiece& rawVal) { + int32_t offset = 0; + auto status = *reinterpret_cast(rawVal.begin() + offset); + offset += sizeof(BalanceTaskStatus); + auto ret = *reinterpret_cast(rawVal.begin() + offset); + offset += sizeof(BalanceTaskResult); + auto start = *reinterpret_cast(rawVal.begin() + offset); + offset += sizeof(int64_t); + auto end = *reinterpret_cast(rawVal.begin() + offset); + return std::make_tuple(status, ret, start, end); +} + +std::string MetaKeyUtils::groupKey(const std::string& group) { + std::string key; + key.reserve(kGroupsTable.size() + group.size()); + key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); + return key; +} + +BalanceID MetaKeyUtils::parseBalanceID(const folly::StringPiece& rawKey) { + auto decode = *reinterpret_cast(rawKey.begin() + kBalancePlanTable.size()); + auto id = std::numeric_limits::max() - folly::Endian::big(decode); + CHECK_GE(id, 0); + return id; +} + +BalanceStatus MetaKeyUtils::parseBalanceStatus(const folly::StringPiece& rawVal) { + return static_cast(*rawVal.begin()); +} + +std::string MetaKeyUtils::groupVal(const std::vector& zones) { + return folly::join(",", zones); +} + +const std::string& MetaKeyUtils::groupPrefix() { return kGroupsTable; } + +std::string MetaKeyUtils::parseGroupName(folly::StringPiece rawData) { + return rawData.subpiece(kGroupsTable.size(), rawData.size()).toString(); +} + +std::vector MetaKeyUtils::parseZoneNames(folly::StringPiece rawData) { + std::vector zones; + folly::split(',', rawData.str(), zones); + return zones; +} + +std::string MetaKeyUtils::zoneKey(const std::string& zone) { + std::string key; + key.reserve(kZonesTable.size() + zone.size()); + key.append(kZonesTable.data(), kZonesTable.size()).append(zone); + return key; +} + +std::string MetaKeyUtils::zoneVal(const std::vector& hosts) { + std::string value; + value.append(network::NetworkUtils::toHostsStr(hosts)); + return value; +} + +const std::string& MetaKeyUtils::zonePrefix() { return kZonesTable; } + +std::string MetaKeyUtils::parseZoneName(folly::StringPiece rawData) { + return rawData.subpiece(kZonesTable.size(), rawData.size()).toString(); +} + +std::vector MetaKeyUtils::parseZoneHosts(folly::StringPiece rawData) { + std::vector addresses; + auto hostsOrErr = network::NetworkUtils::toHosts(rawData.str()); + if (hostsOrErr.ok()) { + addresses = std::move(hostsOrErr.value()); + } else { + LOG(ERROR) << "invalid input for parseZoneHosts()"; + } + return addresses; +} + +std::string MetaKeyUtils::listenerKey(GraphSpaceID spaceId, + PartitionID partId, + meta::cpp2::ListenerType type) { + std::string key; + key.reserve(kListenerTable.size() + sizeof(GraphSpaceID) + sizeof(meta::cpp2::ListenerType) + + sizeof(PartitionID)); + key.append(kListenerTable.data(), kListenerTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&type), sizeof(meta::cpp2::ListenerType)) + .append(reinterpret_cast(&partId), sizeof(PartitionID)); + return key; +} + +std::string MetaKeyUtils::listenerPrefix(GraphSpaceID spaceId) { + std::string key; + key.reserve(kListenerTable.size() + sizeof(GraphSpaceID)); + key.append(kListenerTable.data(), kListenerTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +std::string MetaKeyUtils::listenerPrefix(GraphSpaceID spaceId, meta::cpp2::ListenerType type) { + std::string key; + key.reserve(kListenerTable.size() + sizeof(GraphSpaceID) + sizeof(meta::cpp2::ListenerType)); + key.append(kListenerTable.data(), kListenerTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) + .append(reinterpret_cast(&type), sizeof(meta::cpp2::ListenerType)); + return key; +} + +meta::cpp2::ListenerType MetaKeyUtils::parseListenerType(folly::StringPiece rawData) { + auto offset = kListenerTable.size() + sizeof(GraphSpaceID); + return *reinterpret_cast(rawData.data() + offset); +} + +GraphSpaceID MetaKeyUtils::parseListenerSpace(folly::StringPiece rawData) { + auto offset = kListenerTable.size(); + return *reinterpret_cast(rawData.data() + offset); +} + +PartitionID MetaKeyUtils::parseListenerPart(folly::StringPiece rawData) { + auto offset = kListenerTable.size() + sizeof(meta::cpp2::ListenerType) + sizeof(GraphSpaceID); + return *reinterpret_cast(rawData.data() + offset); +} + +std::string MetaKeyUtils::statsKey(GraphSpaceID spaceId) { + std::string key; + key.reserve(kStatsTable.size() + sizeof(GraphSpaceID)); + key.append(kStatsTable.data(), kStatsTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +std::string MetaKeyUtils::statsVal(const meta::cpp2::StatsItem& statsItem) { + std::string val; + apache::thrift::CompactSerializer::serialize(statsItem, &val); + return val; +} + +meta::cpp2::StatsItem MetaKeyUtils::parseStatsVal(folly::StringPiece rawData) { + meta::cpp2::StatsItem statsItem; + apache::thrift::CompactSerializer::deserialize(rawData, statsItem); + return statsItem; +} + +GraphSpaceID MetaKeyUtils::parseStatsSpace(folly::StringPiece rawData) { + auto offset = kStatsTable.size(); + return *reinterpret_cast(rawData.data() + offset); +} + +const std::string& MetaKeyUtils::statsKeyPrefix() { return kStatsTable; } + +std::string MetaKeyUtils::fulltextServiceKey() { + std::string key; + key.reserve(kFTServiceTable.size()); + key.append(kFTServiceTable.data(), kFTServiceTable.size()); + return key; +} + +std::string MetaKeyUtils::fulltextServiceVal(meta::cpp2::FTServiceType type, + const std::vector& clients) { + std::string val, cval; + apache::thrift::CompactSerializer::serialize(clients, &cval); + val.reserve(sizeof(meta::cpp2::FTServiceType) + cval.size()); + val.append(reinterpret_cast(&type), sizeof(meta::cpp2::FTServiceType)).append(cval); + return val; +} + +std::vector MetaKeyUtils::parseFTClients(folly::StringPiece rawData) { + std::vector clients; + int32_t offset = sizeof(meta::cpp2::FTServiceType); + auto clientsRaw = rawData.subpiece(offset, rawData.size() - offset); + apache::thrift::CompactSerializer::deserialize(clientsRaw, clients); + return clients; +} + +const std::string& MetaKeyUtils::sessionPrefix() { return kSessionsTable; } + +std::string MetaKeyUtils::sessionKey(SessionID sessionId) { + std::string key; + key.reserve(kSessionsTable.size() + sizeof(sessionId)); + key.append(kSessionsTable.data(), kSessionsTable.size()) + .append(reinterpret_cast(&sessionId), sizeof(SessionID)); + return key; +} + +std::string MetaKeyUtils::sessionVal(const meta::cpp2::Session& session) { + std::string val; + apache::thrift::CompactSerializer::serialize(session, &val); + return val; +} + +SessionID MetaKeyUtils::getSessionId(const folly::StringPiece& key) { + return *reinterpret_cast(key.data() + kSessionsTable.size()); +} + +meta::cpp2::Session MetaKeyUtils::parseSessionVal(const folly::StringPiece& val) { + meta::cpp2::Session session; + apache::thrift::CompactSerializer::deserialize(val, session); + return session; +} + +std::string MetaKeyUtils::fulltextIndexKey(const std::string& indexName) { + std::string key; + key.reserve(kFTIndexTable.size() + indexName.size()); + key.append(kFTIndexTable.data(), kFTIndexTable.size()).append(indexName); + return key; +} + +std::string MetaKeyUtils::fulltextIndexVal(const meta::cpp2::FTIndex& index) { + std::string val; + apache::thrift::CompactSerializer::serialize(index, &val); + return val; +} + +std::string MetaKeyUtils::parsefulltextIndexName(folly::StringPiece key) { + return key.subpiece(kFTIndexTable.size(), key.size()).toString(); +} + +meta::cpp2::FTIndex MetaKeyUtils::parsefulltextIndex(folly::StringPiece val) { + meta::cpp2::FTIndex ftIndex; + apache::thrift::CompactSerializer::deserialize(val, ftIndex); + return ftIndex; +} + +std::string MetaKeyUtils::fulltextIndexPrefix() { return kFTIndexTable; } + +std::string MetaKeyUtils::localIdKey(GraphSpaceID spaceId) { + std::string key; + key.reserve(kLocalIdTable.size() + sizeof(GraphSpaceID)); + key.append(kLocalIdTable.data(), kLocalIdTable.size()) + .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); + return key; +} + +GraphSpaceID MetaKeyUtils::parseLocalIdSpace(folly::StringPiece rawData) { + auto offset = kLocalIdTable.size(); + return *reinterpret_cast(rawData.data() + offset); +} + +} // namespace nebula diff --git a/src/common/utils/MetaKeyUtils.h b/src/common/utils/MetaKeyUtils.h new file mode 100644 index 00000000000..47cc1f1f69e --- /dev/null +++ b/src/common/utils/MetaKeyUtils.h @@ -0,0 +1,393 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef META_KEY_UTILS_H_ +#define META_KEY_UTILS_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" +#include "common/datatypes/HostAddr.h" +#include "interface/gen-cpp2/meta_types.h" + +namespace nebula { + +static const PartitionID kDefaultPartId = 0; +static const GraphSpaceID kDefaultSpaceId = 0; + +using BalanceID = int64_t; + +enum class BalanceTaskStatus : uint8_t { + START = 0x01, + CHANGE_LEADER = 0x02, + ADD_PART_ON_DST = 0x03, + ADD_LEARNER = 0x04, + CATCH_UP_DATA = 0x05, + MEMBER_CHANGE_ADD = 0x06, + MEMBER_CHANGE_REMOVE = 0x07, + UPDATE_PART_META = 0x08, // After this state, we can't rollback anymore. + REMOVE_PART_ON_SRC = 0x09, + CHECK = 0x0A, + END = 0xFF, +}; + +enum class BalanceTaskResult : uint8_t { + SUCCEEDED = 0x01, + FAILED = 0x02, + IN_PROGRESS = 0x03, + INVALID = 0x04, +}; + +enum class BalanceStatus : uint8_t { + NOT_START = 0x01, + IN_PROGRESS = 0x02, + SUCCEEDED = 0x03, + /** + * TODO(heng): Currently, after the plan failed, we will try to resume it + * when running "balance" again. But in many cases, the plan will be failed + * forever, it this cases, we should rollback the plan. + * */ + FAILED = 0x04, +}; + +enum class EntryType : int8_t { + SPACE = 0x01, + TAG = 0x02, + EDGE = 0x03, + INDEX = 0x04, + CONFIG = 0x05, + GROUP = 0x06, + ZONE = 0x07, +}; + +using ConfigName = std::pair; +using LeaderParts = std::unordered_map>; + +class MetaKeyUtils final { + public: + MetaKeyUtils() = delete; + + static std::string lastUpdateTimeKey(); + + static std::string lastUpdateTimeVal(const int64_t timeInMilliSec); + + static std::string spaceKey(GraphSpaceID spaceId); + + static std::string spaceVal(const meta::cpp2::SpaceDesc& spaceDesc); + + static meta::cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); + + static const std::string& spacePrefix(); + + static GraphSpaceID spaceId(folly::StringPiece rawKey); + + static std::string spaceName(folly::StringPiece rawVal); + + static std::string partKey(GraphSpaceID spaceId, PartitionID partId); + + static GraphSpaceID parsePartKeySpaceId(folly::StringPiece key); + + static PartitionID parsePartKeyPartId(folly::StringPiece key); + + static std::string partVal(const std::vector& hosts); + + static std::string partValV1(const std::vector& hosts); + + static std::string partValV2(const std::vector& hosts); + + static std::string partPrefix(); + + static std::string partPrefix(GraphSpaceID spaceId); + + static std::string encodeHostAddrV2(int ip, int port); + + static HostAddr decodeHostAddrV2(folly::StringPiece val, int& offset); + + static std::vector parsePartVal(folly::StringPiece val, int parNum = 0); + + static std::vector parsePartValV1(folly::StringPiece val); + + static std::vector parsePartValV2(folly::StringPiece val); + + static std::string hostKey(std::string ip, Port port); + + static std::string hostKeyV2(std::string addr, Port port); + + static const std::string& hostPrefix(); + + static HostAddr parseHostKey(folly::StringPiece key); + + static HostAddr parseHostKeyV1(folly::StringPiece key); + + static HostAddr parseHostKeyV2(folly::StringPiece key); + + static std::string leaderKey(std::string ip, Port port); + + static std::string leaderKeyV2(std::string addr, Port port); + + static std::string leaderKey(GraphSpaceID spaceId, PartitionID partId); + + static std::string leaderVal(const LeaderParts& leaderParts); + + static std::string leaderValV3(const HostAddr& h, int64_t term); + + static std::string leaderPrefix(GraphSpaceID spaceId); + + static const std::string& leaderPrefix(); + + static HostAddr parseLeaderKey(folly::StringPiece key); + + static HostAddr parseLeaderKeyV1(folly::StringPiece key); + + static std::pair parseLeaderKeyV3(folly::StringPiece key); + + static HostAddr parseLeaderKeyV2(folly::StringPiece key); + + static LeaderParts parseLeaderValV1(folly::StringPiece val); + + static std::tuple parseLeaderValV3( + folly::StringPiece val); + + static std::string schemaVal(const std::string& name, const meta::cpp2::Schema& schema); + + static std::string schemaEdgePrefix(GraphSpaceID spaceId, EdgeType edgeType); + + static std::string schemaEdgesPrefix(GraphSpaceID spaceId); + + static std::string schemaEdgeKey(GraphSpaceID spaceId, EdgeType edgeType, SchemaVer version); + + static EdgeType parseEdgeType(folly::StringPiece key); + + static SchemaVer parseEdgeVersion(folly::StringPiece key); + + static std::string schemaTagKey(GraphSpaceID spaceId, TagID tagId, SchemaVer version); + + static TagID parseTagId(folly::StringPiece key); + + static SchemaVer parseTagVersion(folly::StringPiece key); + + static std::string schemaTagPrefix(GraphSpaceID spaceId, TagID tagId); + + static std::string schemaTagsPrefix(GraphSpaceID spaceId); + + static meta::cpp2::Schema parseSchema(folly::StringPiece rawData); + + static std::string indexKey(GraphSpaceID spaceId, IndexID indexID); + + static std::string indexVal(const meta::cpp2::IndexItem& item); + + static std::string indexPrefix(GraphSpaceID spaceId); + + static IndexID parseIndexesKeyIndexID(folly::StringPiece key); + + static meta::cpp2::IndexItem parseIndex(const folly::StringPiece& rawData); + + static std::string rebuildIndexStatus(GraphSpaceID space, + char type, + const std::string& indexName); + + static std::string rebuildIndexStatusPrefix(GraphSpaceID spaceId, char type); + + static std::string rebuildIndexStatusPrefix(); + + static std::string rebuildTagIndexStatusPrefix(GraphSpaceID spaceId) { + return rebuildIndexStatusPrefix(spaceId, 'T'); + } + + static std::string rebuildEdgeIndexStatusPrefix(GraphSpaceID spaceId) { + return rebuildIndexStatusPrefix(spaceId, 'E'); + } + + static std::string indexSpaceKey(const std::string& name); + + static std::string indexTagKey(GraphSpaceID spaceId, const std::string& name); + + static std::string indexEdgeKey(GraphSpaceID spaceId, const std::string& name); + + static std::string indexIndexKey(GraphSpaceID spaceId, const std::string& name); + + static std::string indexGroupKey(const std::string& name); + + static std::string indexZoneKey(const std::string& name); + + static std::string assembleSegmentKey(const std::string& segment, const std::string& key); + + static std::string userPrefix(); + + static std::string userKey(const std::string& account); + + static std::string userVal(const std::string& val); + + static std::string parseUser(folly::StringPiece key); + + static std::string parseUserPwd(folly::StringPiece val); + + static std::string roleKey(GraphSpaceID spaceId, const std::string& account); + + static std::string roleVal(meta::cpp2::RoleType roleType); + + static std::string parseRoleUser(folly::StringPiece key); + + static GraphSpaceID parseRoleSpace(folly::StringPiece key); + + static std::string rolesPrefix(); + + static std::string roleSpacePrefix(GraphSpaceID spaceId); + + static std::string parseRoleStr(folly::StringPiece key); + + static std::string configKey(const meta::cpp2::ConfigModule& module, const std::string& name); + + static std::string configKeyPrefix(const meta::cpp2::ConfigModule& module); + + static std::string configValue(const meta::cpp2::ConfigMode& valueMode, const Value& config); + + static ConfigName parseConfigKey(folly::StringPiece rawData); + + static meta::cpp2::ConfigItem parseConfigValue(folly::StringPiece rawData); + + static std::string snapshotKey(const std::string& name); + + static std::string snapshotVal(const meta::cpp2::SnapshotStatus& status, + const std::string& hosts); + + static meta::cpp2::SnapshotStatus parseSnapshotStatus(folly::StringPiece rawData); + + static std::string parseSnapshotHosts(folly::StringPiece rawData); + + static std::string parseSnapshotName(folly::StringPiece rawData); + + static const std::string& snapshotPrefix(); + + static std::string serializeHostAddr(const HostAddr& host); + + static HostAddr deserializeHostAddr(folly::StringPiece str); + + static std::string balanceTaskKey( + BalanceID balanceId, GraphSpaceID spaceId, PartitionID partId, HostAddr src, HostAddr dst); + + static std::string balanceTaskVal(BalanceTaskStatus status, + BalanceTaskResult retult, + int64_t startTime, + int64_t endTime); + + static std::string balanceTaskPrefix(BalanceID balanceId); + + static std::string balancePlanKey(BalanceID id); + + static std::string balancePlanVal(BalanceStatus status); + + static std::string balancePlanPrefix(); + + static BalanceID parseBalanceID(const folly::StringPiece& rawKey); + + static BalanceStatus parseBalanceStatus(const folly::StringPiece& rawVal); + + static std::tuple parseBalanceTaskKey( + const folly::StringPiece& rawKey); + + static std::tuple parseBalanceTaskVal( + const folly::StringPiece& rawVal); + + static std::string groupKey(const std::string& group); + + static std::string groupVal(const std::vector& zones); + + static const std::string& groupPrefix(); + + static std::string parseGroupName(folly::StringPiece rawData); + + static std::vector parseZoneNames(folly::StringPiece rawData); + + static std::string zoneKey(const std::string& zone); + + static std::string zoneVal(const std::vector& hosts); + + static const std::string& zonePrefix(); + + static std::string parseZoneName(folly::StringPiece rawData); + + static std::vector parseZoneHosts(folly::StringPiece rawData); + + static std::string listenerKey(GraphSpaceID spaceId, + PartitionID partId, + meta::cpp2::ListenerType type); + + static std::string listenerPrefix(GraphSpaceID spaceId); + + static std::string listenerPrefix(GraphSpaceID spaceId, meta::cpp2::ListenerType type); + + static meta::cpp2::ListenerType parseListenerType(folly::StringPiece rawData); + + static GraphSpaceID parseListenerSpace(folly::StringPiece rawData); + + static PartitionID parseListenerPart(folly::StringPiece rawData); + + static std::string statsKey(GraphSpaceID spaceId); + + static std::string statsVal(const meta::cpp2::StatsItem& statsItem); + + static meta::cpp2::StatsItem parseStatsVal(folly::StringPiece rawData); + + static const std::string& statsKeyPrefix(); + + static GraphSpaceID parseStatsSpace(folly::StringPiece rawData); + + static std::string fulltextServiceKey(); + + static std::string fulltextServiceVal(meta::cpp2::FTServiceType type, + const std::vector& clients); + + static std::vector parseFTClients(folly::StringPiece rawData); + + static const std::string& sessionPrefix(); + + static std::string sessionKey(SessionID sessionId); + + static std::string sessionVal(const meta::cpp2::Session& session); + + static SessionID getSessionId(const folly::StringPiece& key); + + static meta::cpp2::Session parseSessionVal(const folly::StringPiece& val); + + static std::string fulltextIndexKey(const std::string& indexName); + + static std::string fulltextIndexVal(const meta::cpp2::FTIndex& index); + + static std::string parsefulltextIndexName(folly::StringPiece key); + + static meta::cpp2::FTIndex parsefulltextIndex(folly::StringPiece val); + + static std::string fulltextIndexPrefix(); + + static std::string genTimestampStr(); + + static GraphSpaceID parseEdgesKeySpaceID(folly::StringPiece key); + static GraphSpaceID parseTagsKeySpaceID(folly::StringPiece key); + static GraphSpaceID parseIndexesKeySpaceID(folly::StringPiece key); + static GraphSpaceID parseIndexStatusKeySpaceID(folly::StringPiece key); + static GraphSpaceID parseIndexKeySpaceID(folly::StringPiece key); + static GraphSpaceID parseDefaultKeySpaceID(folly::StringPiece key); + + static std::string idKey(); + + static std::string localIdKey(GraphSpaceID spaceId); + + static GraphSpaceID parseLocalIdSpace(folly::StringPiece rawData); + + static std::string getIndexTable(); + + static std::unordered_map>> + getTableMaps(); + + static std::unordered_map> getSystemInfoMaps(); + + static std::unordered_map> getSystemTableMaps(); +}; + +} // namespace nebula + +#endif // META_KEY_UTILS_H_ diff --git a/src/common/utils/test/CMakeLists.txt b/src/common/utils/test/CMakeLists.txt index d02df8575dc..5e78743cf97 100644 --- a/src/common/utils/test/CMakeLists.txt +++ b/src/common/utils/test/CMakeLists.txt @@ -43,6 +43,9 @@ nebula_add_test( OperationKeyUtilsTest.cpp OBJECTS $ + $ + $ + $ $ $ $ @@ -54,3 +57,28 @@ nebula_add_test( gtest ${THRIFT_LIBRARIES} ) + +nebula_add_test( + NAME + meta_key_utils_test + SOURCES + MetaKeyUtilsTest.cpp + OBJECTS + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + LIBRARIES + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + gtest +) diff --git a/src/meta/test/MetaServiceUtilsTest.cpp b/src/common/utils/test/MetaKeyUtilsTest.cpp similarity index 62% rename from src/meta/test/MetaServiceUtilsTest.cpp rename to src/common/utils/test/MetaKeyUtilsTest.cpp index e21b828b4d9..dcc4aca964e 100644 --- a/src/meta/test/MetaServiceUtilsTest.cpp +++ b/src/common/utils/test/MetaKeyUtilsTest.cpp @@ -10,29 +10,28 @@ #include "common/base/Base.h" #include "common/fs/TempFile.h" #include "common/network/NetworkUtils.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" namespace nebula { -namespace meta { -TEST(MetaServiceUtilsTest, SpaceKeyTest) { - auto prefix = MetaServiceUtils::spacePrefix(); +TEST(MetaKeyUtilsTest, SpaceKeyTest) { + auto prefix = MetaKeyUtils::spacePrefix(); ASSERT_EQ("__spaces__", prefix); - auto spaceKey = MetaServiceUtils::spaceKey(101); - ASSERT_EQ(101, MetaServiceUtils::spaceId(spaceKey)); - cpp2::SpaceDesc spaceDesc; + auto spaceKey = MetaKeyUtils::spaceKey(101); + ASSERT_EQ(101, MetaKeyUtils::spaceId(spaceKey)); + meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default"); spaceDesc.set_partition_num(100); spaceDesc.set_replica_factor(3); - auto spaceVal = MetaServiceUtils::spaceVal(spaceDesc); - ASSERT_EQ("default", MetaServiceUtils::spaceName(spaceVal)); - ASSERT_EQ(100, MetaServiceUtils::parseSpace(spaceVal).get_partition_num()); - ASSERT_EQ(3, MetaServiceUtils::parseSpace(spaceVal).get_replica_factor()); + auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc); + ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal)); + ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num()); + ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor()); } -TEST(MetaServiceUtilsTest, PartKeyTest) { - auto partKey = MetaServiceUtils::partKey(0, 1); - auto prefix = MetaServiceUtils::partPrefix(0); +TEST(MetaKeyUtilsTest, PartKeyTest) { + auto partKey = MetaKeyUtils::partKey(0, 1); + auto prefix = MetaKeyUtils::partPrefix(0); ASSERT_EQ("__parts__", prefix.substr(0, prefix.size() - sizeof(GraphSpaceID))); ASSERT_EQ(0, *reinterpret_cast(prefix.c_str() + prefix.size() - @@ -44,9 +43,9 @@ TEST(MetaServiceUtilsTest, PartKeyTest) { for (int i = 0; i < 10; i++) { hosts.emplace_back(std::to_string(i * 20 + 1), i * 20 + 2); } - auto partVal = MetaServiceUtils::partVal(hosts); + auto partVal = MetaKeyUtils::partVal(hosts); ASSERT_GE(partVal.size(), 10 * sizeof(int32_t) * 2); - auto result = MetaServiceUtils::parsePartVal(partVal); + auto result = MetaKeyUtils::parsePartVal(partVal); ASSERT_EQ(hosts.size(), result.size()); for (int i = 0; i < 10; i++) { ASSERT_EQ(std::to_string(i * 20 + 1), result[i].host); @@ -54,7 +53,7 @@ TEST(MetaServiceUtilsTest, PartKeyTest) { } } -TEST(MetaServiceUtilsTest, storeStrIpCodecTest) { +TEST(MetaKeyUtilsTest, storeStrIpCodecTest) { int N = 4; std::vector hostnames(N); std::vector ports; @@ -67,8 +66,8 @@ TEST(MetaServiceUtilsTest, storeStrIpCodecTest) { { // kPartsTable : value - auto encodedVal = MetaServiceUtils::partValV2(hosts); - auto decodedVal = MetaServiceUtils::parsePartVal(encodedVal); + auto encodedVal = MetaKeyUtils::partValV2(hosts); + auto decodedVal = MetaKeyUtils::parsePartVal(encodedVal); ASSERT_EQ(hosts.size(), decodedVal.size()); for (int i = 0; i < N; i++) { LOG(INFO) << folly::format("hosts[{}]={}:{}", i, hostnames[i], ports[i]); @@ -79,8 +78,8 @@ TEST(MetaServiceUtilsTest, storeStrIpCodecTest) { { // kHostsTable : key - auto key = MetaServiceUtils::hostKey(hostnames[0], ports[0]); - auto host = MetaServiceUtils::parseHostKeyV2(key); + auto key = MetaKeyUtils::hostKey(hostnames[0], ports[0]); + auto host = MetaKeyUtils::parseHostKeyV2(key); ASSERT_EQ(host.host, hostnames[0]); ASSERT_EQ(host.port, ports[0]); } @@ -90,16 +89,16 @@ TEST(MetaServiceUtilsTest, storeStrIpCodecTest) { GraphSpaceID spaceId = 1; PartitionID partId = 10; int64_t termId = 999999; - auto key = MetaServiceUtils::leaderKey(spaceId, partId); - auto parsedLeaderKey = MetaServiceUtils::parseLeaderKeyV3(key); + auto key = MetaKeyUtils::leaderKey(spaceId, partId); + auto parsedLeaderKey = MetaKeyUtils::parseLeaderKeyV3(key); EXPECT_EQ(parsedLeaderKey.first, spaceId); EXPECT_EQ(parsedLeaderKey.second, partId); HostAddr host; host.host = "Hello-Kitty"; host.port = 9527; - auto leaderVal = MetaServiceUtils::leaderValV3(host, termId); - auto parsedVal = MetaServiceUtils::parseLeaderValV3(leaderVal); + auto leaderVal = MetaKeyUtils::leaderValV3(host, termId); + auto parsedVal = MetaKeyUtils::parseLeaderValV3(leaderVal); EXPECT_EQ(std::get<0>(parsedVal), host); EXPECT_EQ(std::get<1>(parsedVal), termId); @@ -126,7 +125,7 @@ std::string leaderKeyV1(uint32_t ip, Port port) { return key; } -TEST(MetaServiceUtilsTest, storeStrIpBackwardCompatibilityTest) { +TEST(MetaKeyUtilsTest, storeStrIpBackwardCompatibilityTest) { int N = 4; std::vector hostnames(N); std::vector ips(N); @@ -142,68 +141,67 @@ TEST(MetaServiceUtilsTest, storeStrIpBackwardCompatibilityTest) { { // kHostsTable : key auto encodedVal = hostKeyV1(ips[0], ports[0]); - auto decodedVal = MetaServiceUtils::parseHostKey(encodedVal); + auto decodedVal = MetaKeyUtils::parseHostKey(encodedVal); ASSERT_EQ(decodedVal.host, hostnames[0]); ASSERT_EQ(decodedVal.port, ports[0]); } } -TEST(MetaServiceUtilsTest, HostKeyTest) { - auto hostKey = MetaServiceUtils::hostKey("10", 11); - const auto& prefix = MetaServiceUtils::hostPrefix(); +TEST(MetaKeyUtilsTest, HostKeyTest) { + auto hostKey = MetaKeyUtils::hostKey("10", 11); + const auto& prefix = MetaKeyUtils::hostPrefix(); ASSERT_EQ("__hosts__", prefix); - auto addr = MetaServiceUtils::parseHostKey(hostKey); + auto addr = MetaKeyUtils::parseHostKey(hostKey); ASSERT_EQ("10", addr.host); ASSERT_EQ(11, addr.port); } -TEST(MetaServiceUtilsTest, TagTest) { - cpp2::Schema schema; - std::vector cols; +TEST(MetaKeyUtilsTest, TagTest) { + meta::cpp2::Schema schema; + std::vector cols; for (auto i = 1; i <= 3; i++) { - cpp2::ColumnDef column; + meta::cpp2::ColumnDef column; column.set_name(folly::stringPrintf("col_%d", i)); - column.type.set_type(cpp2::PropertyType::INT64); + column.type.set_type(meta::cpp2::PropertyType::INT64); cols.emplace_back(std::move(column)); } for (auto i = 4; i <= 6; i++) { - cpp2::ColumnDef column; + meta::cpp2::ColumnDef column; column.set_name(folly::stringPrintf("col_%d", i)); - column.type.set_type(cpp2::PropertyType::FLOAT); + column.type.set_type(meta::cpp2::PropertyType::FLOAT); cols.emplace_back(std::move(column)); } for (auto i = 7; i < 10; i++) { - cpp2::ColumnDef column; + meta::cpp2::ColumnDef column; column.set_name(folly::stringPrintf("col_%d", i)); - column.type.set_type(cpp2::PropertyType::STRING); + column.type.set_type(meta::cpp2::PropertyType::STRING); cols.emplace_back(std::move(column)); } schema.set_columns(std::move(cols)); - auto val = MetaServiceUtils::schemaVal("test_tag", schema); - auto parsedSchema = MetaServiceUtils::parseSchema(val); + auto val = MetaKeyUtils::schemaVal("test_tag", schema); + auto parsedSchema = MetaKeyUtils::parseSchema(val); ASSERT_EQ(parsedSchema, schema); } -TEST(MetaServiceUtilsTest, GroupTest) { - auto groupKey = MetaServiceUtils::groupKey("test_group"); - ASSERT_EQ("test_group", MetaServiceUtils::parseGroupName(groupKey)); +TEST(MetaKeyUtilsTest, GroupTest) { + auto groupKey = MetaKeyUtils::groupKey("test_group"); + ASSERT_EQ("test_group", MetaKeyUtils::parseGroupName(groupKey)); std::vector zones = {"zone_0", "zone_1", "zone_2"}; - auto groupValue = MetaServiceUtils::groupVal(zones); - ASSERT_EQ(zones, MetaServiceUtils::parseZoneNames(groupValue)); + auto groupValue = MetaKeyUtils::groupVal(zones); + ASSERT_EQ(zones, MetaKeyUtils::parseZoneNames(groupValue)); } -TEST(MetaServiceUtilsTest, ZoneTest) { - auto zoneKey = MetaServiceUtils::zoneKey("test_zone"); - ASSERT_EQ("test_zone", MetaServiceUtils::parseZoneName(zoneKey)); +TEST(MetaKeyUtilsTest, ZoneTest) { + auto zoneKey = MetaKeyUtils::zoneKey("test_zone"); + ASSERT_EQ("test_zone", MetaKeyUtils::parseZoneName(zoneKey)); std::vector nodes = {{"0", 0}, {"1", 1}, {"2", 2}}; - auto zoneValue = MetaServiceUtils::zoneVal(nodes); - ASSERT_EQ(nodes, MetaServiceUtils::parseZoneHosts(zoneValue)); + auto zoneValue = MetaKeyUtils::zoneVal(nodes); + ASSERT_EQ(nodes, MetaKeyUtils::parseZoneHosts(zoneValue)); } -} // namespace meta } // namespace nebula int main(int argc, char** argv) { diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 565c2717f54..e58fecb2c0d 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -41,6 +41,7 @@ set(storage_meta_deps $ $ $ + $ $ $ $ diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 0174859311d..2de31bbd373 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -16,12 +16,12 @@ #include "common/ssl/SSLConfig.h" #include "common/thread/GenericThreadPool.h" #include "common/time/TimezoneInfo.h" +#include "common/utils/MetaKeyUtils.h" #include "kvstore/NebulaStore.h" #include "kvstore/PartManager.h" #include "meta/ActiveHostsMan.h" #include "meta/KVBasedClusterIdMan.h" #include "meta/MetaServiceHandler.h" -#include "meta/MetaServiceUtils.h" #include "meta/MetaVersionMan.h" #include "meta/RootUserMan.h" #include "meta/http/MetaHttpDownloadHandler.h" @@ -78,7 +78,7 @@ std::unique_ptr initKV(std::vector p nebula::HostAddr localhost) { auto partMan = std::make_unique(); // The meta server has only one space (0), one part (0) - partMan->addPart(nebula::meta::kDefaultSpaceId, nebula::meta::kDefaultPartId, std::move(peers)); + partMan->addPart(nebula::kDefaultSpaceId, nebula::kDefaultPartId, std::move(peers)); // folly IOThreadPoolExecutor auto ioPool = std::make_shared(FLAGS_num_io_threads); std::shared_ptr threadManager( @@ -99,7 +99,7 @@ std::unique_ptr initKV(std::vector p LOG(INFO) << "Waiting for the leader elected..."; nebula::HostAddr leader; while (true) { - auto ret = kvstore->partLeader(nebula::meta::kDefaultSpaceId, nebula::meta::kDefaultPartId); + auto ret = kvstore->partLeader(nebula::kDefaultSpaceId, nebula::kDefaultPartId); if (!nebula::ok(ret)) { LOG(ERROR) << "Nebula store init failed"; return nullptr; @@ -297,7 +297,7 @@ int main(int argc, char* argv[]) { /** * Only leader part needed. */ - auto ret = gKVStore->partLeader(nebula::meta::kDefaultSpaceId, nebula::meta::kDefaultPartId); + auto ret = gKVStore->partLeader(nebula::kDefaultSpaceId, nebula::kDefaultPartId); if (!nebula::ok(ret)) { LOG(ERROR) << "Part leader get failed"; return EXIT_FAILURE; diff --git a/src/kvstore/test/CMakeLists.txt b/src/kvstore/test/CMakeLists.txt index aa039b23854..75a807479cb 100644 --- a/src/kvstore/test/CMakeLists.txt +++ b/src/kvstore/test/CMakeLists.txt @@ -4,6 +4,7 @@ set(KVSTORE_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/meta/ActiveHostsMan.cpp b/src/meta/ActiveHostsMan.cpp index e20ba8a1f9d..68919104cb2 100644 --- a/src/meta/ActiveHostsMan.cpp +++ b/src/meta/ActiveHostsMan.cpp @@ -30,7 +30,7 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, for (auto& spaceLeaders : *allLeaders) { auto spaceId = spaceLeaders.first; for (auto& partLeader : spaceLeaders.second) { - auto key = MetaServiceUtils::leaderKey(spaceId, partLeader.get_part_id()); + auto key = MetaKeyUtils::leaderKey(spaceId, partLeader.get_part_id()); leaderKeys.emplace_back(std::move(key)); terms.emplace_back(partLeader.get_term()); } @@ -47,7 +47,7 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, nebula::cpp2::ErrorCode code; for (auto i = 0U; i != leaderKeys.size(); ++i) { if (statusVec[i].ok()) { - std::tie(std::ignore, term, code) = MetaServiceUtils::parseLeaderValV3(vals[i]); + std::tie(std::ignore, term, code) = MetaKeyUtils::parseLeaderValV3(vals[i]); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(WARNING) << apache::thrift::util::enumNameSafe(code); continue; @@ -57,7 +57,7 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, } } // write directly if not exist, or update if has greater term - auto val = MetaServiceUtils::leaderValV3(hostAddr, terms[i]); + auto val = MetaKeyUtils::leaderValV3(hostAddr, terms[i]); data.emplace_back(std::make_pair(leaderKeys[i], std::move(val))); } } @@ -66,8 +66,7 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, if (!data.empty()) { hasUpdate = true; } - data.emplace_back(MetaServiceUtils::hostKey(hostAddr.host, hostAddr.port), - HostInfo::encodeV2(info)); + data.emplace_back(MetaKeyUtils::hostKey(hostAddr.host, hostAddr.port), HostInfo::encodeV2(info)); folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock()); folly::Baton baton; @@ -89,7 +88,7 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, ErrorOr> ActiveHostsMan::getActiveHosts( kvstore::KVStore* kv, int32_t expiredTTL, cpp2::HostRole role) { - const auto& prefix = MetaServiceUtils::hostPrefix(); + const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; auto retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -103,7 +102,7 @@ ErrorOr> ActiveHostsMan::getActiv 1000; auto now = time::WallClock::fastNowInMilliSec(); while (iter->valid()) { - auto host = MetaServiceUtils::parseHostKey(iter->key()); + auto host = MetaKeyUtils::parseHostKey(iter->key()); HostInfo info = HostInfo::decode(iter->val()); if (info.role_ == role) { if (now - info.lastHBTimeInMilliSec_ < threshold) { @@ -120,7 +119,7 @@ ErrorOr> ActiveHostsMan::getActiv kvstore::KVStore* kv, const std::string& zoneName, int32_t expiredTTL) { std::vector activeHosts; std::string zoneValue; - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get zone " << zoneName @@ -128,7 +127,7 @@ ErrorOr> ActiveHostsMan::getActiv return retCode; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(zoneValue)); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(zoneValue)); auto now = time::WallClock::fastNowInMilliSec(); int64_t threshold = (expiredTTL == 0 ? FLAGS_heartbeat_interval_secs * FLAGS_expired_time_factor : expiredTTL) * @@ -151,7 +150,7 @@ ErrorOr> ActiveHostsMan::getActiv kvstore::KVStore* kv, GraphSpaceID spaceId, int32_t expiredTTL) { std::string spaceValue; std::vector activeHosts; - auto spaceKey = MetaServiceUtils::spaceKey(spaceId); + auto spaceKey = MetaKeyUtils::spaceKey(spaceId); auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, spaceKey, &spaceValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get space failed, error: " << apache::thrift::util::enumNameSafe(retCode); @@ -159,8 +158,8 @@ ErrorOr> ActiveHostsMan::getActiv } std::string groupValue; - auto space = MetaServiceUtils::parseSpace(std::move(spaceValue)); - auto groupKey = MetaServiceUtils::groupKey(*space.group_name_ref()); + auto space = MetaKeyUtils::parseSpace(std::move(spaceValue)); + auto groupKey = MetaKeyUtils::groupKey(*space.group_name_ref()); retCode = kv->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get group " << *space.group_name_ref() @@ -168,7 +167,7 @@ ErrorOr> ActiveHostsMan::getActiv return retCode; } - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(groupValue)); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); for (const auto& zoneName : zoneNames) { auto hostsRet = getActiveHostsInZone(kv, zoneName, expiredTTL); if (!nebula::ok(hostsRet)) { @@ -208,7 +207,7 @@ ErrorOr ActiveHostsMan::isLived(kvstore::KVStore* ErrorOr ActiveHostsMan::getHostInfo(kvstore::KVStore* kv, const HostAddr& host) { - auto hostKey = MetaServiceUtils::hostKey(host.host, host.port); + auto hostKey = MetaKeyUtils::hostKey(host.host, host.port); std::string hostValue; auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, hostKey, &hostValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -223,8 +222,8 @@ nebula::cpp2::ErrorCode LastUpdateTimeMan::update(kvstore::KVStore* kv, const int64_t timeInMilliSec) { CHECK_NOTNULL(kv); std::vector data; - data.emplace_back(MetaServiceUtils::lastUpdateTimeKey(), - MetaServiceUtils::lastUpdateTimeVal(timeInMilliSec)); + data.emplace_back(MetaKeyUtils::lastUpdateTimeKey(), + MetaKeyUtils::lastUpdateTimeVal(timeInMilliSec)); folly::SharedMutex::WriteHolder wHolder(LockUtils::lastUpdateTimeLock()); folly::Baton baton; @@ -240,7 +239,7 @@ nebula::cpp2::ErrorCode LastUpdateTimeMan::update(kvstore::KVStore* kv, ErrorOr LastUpdateTimeMan::get(kvstore::KVStore* kv) { CHECK_NOTNULL(kv); - auto key = MetaServiceUtils::lastUpdateTimeKey(); + auto key = MetaKeyUtils::lastUpdateTimeKey(); std::string val; auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, key, &val); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { diff --git a/src/meta/ActiveHostsMan.h b/src/meta/ActiveHostsMan.h index d922c0b7ab0..0f2ec18f7f0 100644 --- a/src/meta/ActiveHostsMan.h +++ b/src/meta/ActiveHostsMan.h @@ -10,8 +10,8 @@ #include #include "common/base/Base.h" +#include "common/utils/MetaKeyUtils.h" #include "kvstore/KVStore.h" -#include "meta/MetaServiceUtils.h" namespace nebula { namespace meta { diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 90c4835dd0b..1336615c590 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -132,6 +132,7 @@ set(meta_test_deps $ $ $ + $ $ $ $ diff --git a/src/meta/MetaServiceHandler.cpp b/src/meta/MetaServiceHandler.cpp index d00c5185be8..6ad92a8e29f 100644 --- a/src/meta/MetaServiceHandler.cpp +++ b/src/meta/MetaServiceHandler.cpp @@ -6,7 +6,7 @@ #include "meta/MetaServiceHandler.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/processors/admin/BalanceProcessor.h" #include "meta/processors/admin/CreateBackupProcessor.h" #include "meta/processors/admin/CreateSnapshotProcessor.h" diff --git a/src/meta/MetaServiceUtils.cpp b/src/meta/MetaServiceUtils.cpp index de0de19effb..50ce3b91e89 100644 --- a/src/meta/MetaServiceUtils.cpp +++ b/src/meta/MetaServiceUtils.cpp @@ -6,95 +6,12 @@ #include "meta/MetaServiceUtils.h" -#include -#include -#include - -#include - -#include "common/network/NetworkUtils.h" -#include "processors/Common.h" +#include "common/utils/MetaKeyUtils.h" +#include "interface/gen-cpp2/meta_types.h" namespace nebula { namespace meta { -// Systemtable means that it does not contain any space information(id).false -// means that the backup should be skipped. -static const std::unordered_map> systemTableMaps = { - {"users", {"__users__", true}}, - {"hosts", {"__hosts__", false}}, - {"snapshots", {"__snapshots__", false}}, - {"configs", {"__configs__", true}}, - {"groups", {"__groups__", true}}, - {"zones", {"__zones__", true}}, - {"ft_service", {"__ft_service__", false}}, - {"sessions", {"__sessions__", true}}}; - -// SystemInfo will always be backuped -static const std::unordered_map> systemInfoMaps{ - {"autoIncrementId", {"__id__", true}}, {"lastUpdateTime", {"__last_update_time__", true}}}; - -// name => {prefix, parseSpaceid}, nullptr means that the backup should be skipped. -static const std::unordered_map< - std::string, - std::pair>> - tableMaps = { - {"spaces", {"__spaces__", MetaServiceUtils::spaceId}}, - {"parts", {"__parts__", MetaServiceUtils::parsePartKeySpaceId}}, - {"tags", {"__tags__", MetaServiceUtils::parseTagsKeySpaceID}}, - {"edges", {"__edges__", MetaServiceUtils::parseEdgesKeySpaceID}}, - {"indexes", {"__indexes__", MetaServiceUtils::parseIndexesKeySpaceID}}, - // Index tables are handled separately. - {"index", {"__index__", nullptr}}, - {"index_status", {"__index_status__", MetaServiceUtils::parseIndexStatusKeySpaceID}}, - {"roles", {"__roles__", MetaServiceUtils::parseRoleSpace}}, - {"leaders", {"__leaders__", nullptr}}, - {"leader_terms", {"__leader_terms__", nullptr}}, - {"listener", {"__listener__", nullptr}}, - {"stats", {"__stats__", MetaServiceUtils::parseStatsSpace}}, - {"balance_task", {"__balance_task__", nullptr}}, - {"balance_plan", {"__balance_plan__", nullptr}}, - {"ft_index", {"__ft_index__", nullptr}}, - {"local_id", {"__local_id__", MetaServiceUtils::parseLocalIdSpace}}}; - -// clang-format off -static const std::string kSpacesTable = tableMaps.at("spaces").first; // NOLINT -static const std::string kPartsTable = tableMaps.at("parts").first; // NOLINT -static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT -static const std::string kTagsTable = tableMaps.at("tags").first; // NOLINT -static const std::string kEdgesTable = tableMaps.at("edges").first; // NOLINT -static const std::string kIndexesTable = tableMaps.at("indexes").first; // NOLINT -static const std::string kIndexTable = tableMaps.at("index").first; // NOLINT -static const std::string kIndexStatusTable = tableMaps.at("index_status").first; // NOLINT -static const std::string kUsersTable = systemTableMaps.at("users").first; // NOLINT -static const std::string kRolesTable = tableMaps.at("roles").first; // NOLINT -static const std::string kConfigsTable = systemTableMaps.at("configs").first; // NOLINT -static const std::string kSnapshotsTable = systemTableMaps.at("snapshots").first; // NOLINT -static const std::string kLeadersTable = tableMaps.at("leaders").first; // NOLINT -static const std::string kLeaderTermsTable = tableMaps.at("leader_terms").first; // NOLINT -static const std::string kGroupsTable = systemTableMaps.at("groups").first; // NOLINT -static const std::string kZonesTable = systemTableMaps.at("zones").first; // NOLINT -static const std::string kListenerTable = tableMaps.at("listener").first; // NOLINT - -// Used to record the number of vertices and edges in the space -// The number of vertices of each tag in the space -// The number of edges of each edgetype in the space -static const std::string kStatsTable = tableMaps.at("stats").first; // NOLINT -static const std::string kBalanceTaskTable = tableMaps.at("balance_task").first; // NOLINT -static const std::string kBalancePlanTable = tableMaps.at("balance_plan").first; // NOLINT -static const std::string kLocalIdTable = tableMaps.at("local_id").first; // NOLINT - -const std::string kFTIndexTable = tableMaps.at("ft_index").first; // NOLINT -const std::string kFTServiceTable = systemTableMaps.at("ft_service").first; // NOLINT -const std::string kSessionsTable = systemTableMaps.at("sessions").first; // NOLINT - -const std::string kIdKey = systemInfoMaps.at("autoIncrementId").first; // NOLINT -const std::string kLastUpdateTimeTable = systemInfoMaps.at("lastUpdateTime").first; // NOLINT - -// clang-format on - -const int kMaxIpAddrLen = 15; // '255.255.255.255' - namespace { nebula::cpp2::ErrorCode backupTable(kvstore::KVStore* kvstore, const std::string& backupName, @@ -117,566 +34,6 @@ nebula::cpp2::ErrorCode backupTable(kvstore::KVStore* kvstore, } } // namespace -std::string MetaServiceUtils::lastUpdateTimeKey() { - std::string key; - key.reserve(kLastUpdateTimeTable.size()); - key.append(kLastUpdateTimeTable.data(), kLastUpdateTimeTable.size()); - return key; -} - -std::string MetaServiceUtils::lastUpdateTimeVal(const int64_t timeInMilliSec) { - std::string val; - val.reserve(sizeof(int64_t)); - val.append(reinterpret_cast(&timeInMilliSec), sizeof(int64_t)); - return val; -} - -std::string MetaServiceUtils::spaceKey(GraphSpaceID spaceId) { - std::string key; - key.reserve(kSpacesTable.size() + sizeof(GraphSpaceID)); - key.append(kSpacesTable.data(), kSpacesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -std::string MetaServiceUtils::spaceVal(const cpp2::SpaceDesc& spaceDesc) { - std::string val; - apache::thrift::CompactSerializer::serialize(spaceDesc, &val); - return val; -} - -cpp2::SpaceDesc MetaServiceUtils::parseSpace(folly::StringPiece rawData) { - cpp2::SpaceDesc spaceDesc; - apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); - return spaceDesc; -} - -const std::string& MetaServiceUtils::spacePrefix() { return kSpacesTable; } - -GraphSpaceID MetaServiceUtils::spaceId(folly::StringPiece rawKey) { - return *reinterpret_cast(rawKey.data() + kSpacesTable.size()); -} - -std::string MetaServiceUtils::spaceName(folly::StringPiece rawVal) { - return parseSpace(rawVal).get_space_name(); -} - -std::string MetaServiceUtils::partKey(GraphSpaceID spaceId, PartitionID partId) { - std::string key; - key.reserve(kPartsTable.size() + sizeof(GraphSpaceID) + sizeof(PartitionID)); - key.append(kPartsTable.data(), kPartsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&partId), sizeof(PartitionID)); - return key; -} - -GraphSpaceID MetaServiceUtils::parsePartKeySpaceId(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kPartsTable.size()); -} - -PartitionID MetaServiceUtils::parsePartKeyPartId(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kPartsTable.size() + - sizeof(GraphSpaceID)); -} - -std::string MetaServiceUtils::partVal(const std::vector& hosts) { - return partValV2(hosts); -} - -// dataVer(int) + vectorSize(size_t) + vector of (strIp(string) + port(int)) -std::string MetaServiceUtils::partValV2(const std::vector& hosts) { - std::string encodedVal; - int dataVersion = 2; - encodedVal.append(reinterpret_cast(&dataVersion), sizeof(int)) - .append(network::NetworkUtils::toHostsStr(hosts)); - return encodedVal; -} - -std::string MetaServiceUtils::partPrefix() { - std::string prefix; - prefix.reserve(kPartsTable.size() + sizeof(GraphSpaceID)); - prefix.append(kPartsTable.data(), kPartsTable.size()); - return prefix; -} - -std::string MetaServiceUtils::partPrefix(GraphSpaceID spaceId) { - std::string prefix; - prefix.reserve(kPartsTable.size() + sizeof(GraphSpaceID)); - prefix.append(kPartsTable.data(), kPartsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return prefix; -} - -std::vector MetaServiceUtils::parsePartVal(folly::StringPiece val, int partNum) { - static const size_t unitSizeV1 = sizeof(int64_t); - if (unitSizeV1 * partNum == val.size()) { - return parsePartValV1(val); - } - int dataVer = *reinterpret_cast(val.data()); - UNUSED(dataVer); // currently if not ver1, it must be v2 - val.advance(sizeof(int)); - return parsePartValV2(val); -} - -// partion val is ip(int) + port(int) -std::vector MetaServiceUtils::parsePartValV1(folly::StringPiece val) { - std::vector hosts; - static const size_t unitSize = sizeof(int32_t) * 2; - auto hostsNum = val.size() / unitSize; - hosts.reserve(hostsNum); - VLOG(3) << "Total size:" << val.size() << ", host size:" << unitSize << ", host num:" << hostsNum; - for (decltype(hostsNum) i = 0; i < hostsNum; i++) { - HostAddr h; - uint32_t ip = *reinterpret_cast(val.data() + i * unitSize); - h.host = network::NetworkUtils::intToIPv4(ip); - h.port = *reinterpret_cast(val.data() + i * unitSize + sizeof(int32_t)); - hosts.emplace_back(std::move(h)); - } - return hosts; -} - -// dataVer(int) + vectorSize(size_t) + vector of (strIpV4(string) + port(int)) -std::vector MetaServiceUtils::parsePartValV2(folly::StringPiece val) { - std::vector ret; - auto hostsOrErr = network::NetworkUtils::toHosts(val.str()); - if (hostsOrErr.ok()) { - ret = std::move(hostsOrErr.value()); - } else { - LOG(ERROR) << "invalid input for parsePartValV2()"; - } - return ret; -} - -std::string MetaServiceUtils::hostKey(std::string addr, Port port) { return hostKeyV2(addr, port); } - -std::string MetaServiceUtils::hostKeyV2(std::string addr, Port port) { - std::string key; - HostAddr h(addr, port); - key.append(kHostsTable.data(), kHostsTable.size()).append(MetaServiceUtils::serializeHostAddr(h)); - return key; -} - -const std::string& MetaServiceUtils::hostPrefix() { return kHostsTable; } - -HostAddr MetaServiceUtils::parseHostKey(folly::StringPiece key) { - if (key.size() == kHostsTable.size() + sizeof(int64_t)) { - return parseHostKeyV1(key); - } - return parseHostKeyV2(key); -} - -HostAddr MetaServiceUtils::parseHostKeyV1(folly::StringPiece key) { - HostAddr host; - key.advance(kHostsTable.size()); - uint32_t ip = *reinterpret_cast(key.begin()); - host.host = network::NetworkUtils::intToIPv4(ip); - host.port = *reinterpret_cast(key.begin() + sizeof(uint32_t)); - return host; -} - -HostAddr MetaServiceUtils::parseHostKeyV2(folly::StringPiece key) { - key.advance(kHostsTable.size()); - return MetaServiceUtils::deserializeHostAddr(key); -} - -std::string MetaServiceUtils::leaderKey(std::string addr, Port port) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - return leaderKeyV2(addr, port); -} - -std::string MetaServiceUtils::leaderKeyV2(std::string addr, Port port) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - std::string key; - HostAddr h(addr, port); - - key.reserve(kLeadersTable.size() + kMaxIpAddrLen + sizeof(Port)); - key.append(kLeadersTable.data(), kLeadersTable.size()) - .append(MetaServiceUtils::serializeHostAddr(h)); - return key; -} - -std::string MetaServiceUtils::leaderKey(GraphSpaceID spaceId, PartitionID partId) { - std::string key; - key.reserve(kLeaderTermsTable.size() + sizeof(GraphSpaceID) + sizeof(PartitionID)); - key.append(kLeaderTermsTable.data(), kLeaderTermsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&partId), sizeof(PartitionID)); - return key; -} - -std::string MetaServiceUtils::leaderVal(const LeaderParts& leaderParts) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - std::string value; - value.reserve(512); - for (const auto& spaceEntry : leaderParts) { - GraphSpaceID spaceId = spaceEntry.first; - value.append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - size_t leaderCount = spaceEntry.second.size(); - value.append(reinterpret_cast(&leaderCount), sizeof(size_t)); - for (const auto& partId : spaceEntry.second) { - value.append(reinterpret_cast(&partId), sizeof(PartitionID)); - } - } - return value; -} - -// v3: dataVer(int) + lenOfHost(8) + HostAddr(varchar) + term(int64_t) -std::string MetaServiceUtils::leaderValV3(const HostAddr& h, int64_t term) { - std::string leaderVal; - leaderVal.reserve(256); - - int dataVersion = 3; - - auto sHost = serializeHostAddr(h); - auto lenOfHost = sHost.size(); - - leaderVal.append(reinterpret_cast(&dataVersion), sizeof(dataVersion)) - .append(reinterpret_cast(&lenOfHost), sizeof(lenOfHost)) - .append(sHost) - .append(reinterpret_cast(&term), sizeof(term)); - return leaderVal; -} - -// v3: dataVer(int) + lenOfHost(8) + HostAddr(varchar) + term(int64_t) -std::tuple MetaServiceUtils::parseLeaderValV3( - folly::StringPiece val) { - std::tuple ret; - std::get<2>(ret) = nebula::cpp2::ErrorCode::SUCCEEDED; - int dataVer = *reinterpret_cast(val.data()); - if (dataVer != 3) { - std::get<2>(ret) = nebula::cpp2::ErrorCode::E_INVALID_PARM; - return ret; - } - - CHECK_GE(val.size(), sizeof(int)); - val.advance(sizeof(int)); - auto lenOfHost = *reinterpret_cast(val.data()); - - val.advance(sizeof(size_t)); - CHECK_GE(val.size(), lenOfHost); - std::get<0>(ret) = MetaServiceUtils::deserializeHostAddr(val.subpiece(0, lenOfHost)); - - val.advance(lenOfHost); - std::get<1>(ret) = *reinterpret_cast(val.data()); - return ret; -} - -const std::string& MetaServiceUtils::leaderPrefix() { return kLeaderTermsTable; } - -std::string MetaServiceUtils::leaderPrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kLeaderTermsTable.size() + sizeof(GraphSpaceID)); - key.append(kLeaderTermsTable.data(), kLeaderTermsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -HostAddr MetaServiceUtils::parseLeaderKey(folly::StringPiece key) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - if (key.size() == kLeadersTable.size() + sizeof(int64_t)) { - return parseLeaderKeyV1(key); - } - return parseLeaderKeyV2(key); -} - -// input should be a pair of int32_t -HostAddr MetaServiceUtils::parseLeaderKeyV1(folly::StringPiece key) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - HostAddr host; - CHECK_EQ(key.size(), kLeadersTable.size() + sizeof(int64_t)); - key.advance(kLeadersTable.size()); - auto ip = *reinterpret_cast(key.begin()); - host.host = network::NetworkUtils::intToIPv4(ip); - host.port = *reinterpret_cast(key.begin() + sizeof(ip)); - return host; -} - -HostAddr MetaServiceUtils::parseLeaderKeyV2(folly::StringPiece key) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - key.advance(kLeadersTable.size()); - return MetaServiceUtils::deserializeHostAddr(key); -} - -std::pair MetaServiceUtils::parseLeaderKeyV3(folly::StringPiece key) { - std::pair ret; - ret.first = *reinterpret_cast(key.data() + kLeaderTermsTable.size()); - ret.second = *reinterpret_cast(key.data() + kLeaderTermsTable.size() + - sizeof(GraphSpaceID)); - return ret; -} - -LeaderParts MetaServiceUtils::parseLeaderValV1(folly::StringPiece val) { - LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); - LeaderParts leaderParts; - size_t size = val.size(); - // decode leader info - size_t offset = 0; - while (offset + sizeof(GraphSpaceID) + sizeof(size_t) < size) { - GraphSpaceID spaceId = *reinterpret_cast(val.data() + offset); - offset += sizeof(GraphSpaceID); - size_t leaderCount = *reinterpret_cast(val.data() + offset); - offset += sizeof(size_t); - std::vector partIds; - for (size_t i = 0; i < leaderCount && offset < size; i++) { - partIds.emplace_back(*reinterpret_cast(val.data() + offset)); - offset += sizeof(PartitionID); - } - leaderParts.emplace(spaceId, std::move(partIds)); - } - return leaderParts; -} - -std::string MetaServiceUtils::schemaEdgePrefix(GraphSpaceID spaceId, EdgeType edgeType) { - std::string key; - key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(edgeType)); - key.append(kEdgesTable.data(), kEdgesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&edgeType), sizeof(edgeType)); - return key; -} - -std::string MetaServiceUtils::schemaEdgesPrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID)); - key.append(kEdgesTable.data(), kEdgesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -std::string MetaServiceUtils::schemaEdgeKey(GraphSpaceID spaceId, - EdgeType edgeType, - SchemaVer version) { - auto storageVer = std::numeric_limits::max() - version; - std::string key; - key.reserve(kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(EdgeType) + sizeof(SchemaVer)); - key.append(kEdgesTable.data(), kEdgesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&edgeType), sizeof(EdgeType)) - .append(reinterpret_cast(&storageVer), sizeof(SchemaVer)); - return key; -} - -std::string MetaServiceUtils::schemaVal(const std::string& name, const cpp2::Schema& schema) { - auto len = name.size(); - std::string val, sval; - apache::thrift::CompactSerializer::serialize(schema, &sval); - val.reserve(sizeof(int32_t) + name.size() + sval.size()); - val.append(reinterpret_cast(&len), sizeof(int32_t)).append(name).append(sval); - return val; -} - -EdgeType MetaServiceUtils::parseEdgeType(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kEdgesTable.size() + sizeof(GraphSpaceID)); -} - -SchemaVer MetaServiceUtils::parseEdgeVersion(folly::StringPiece key) { - auto offset = kEdgesTable.size() + sizeof(GraphSpaceID) + sizeof(EdgeType); - return std::numeric_limits::max() - - *reinterpret_cast(key.begin() + offset); -} - -GraphSpaceID MetaServiceUtils::parseEdgesKeySpaceID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kEdgesTable.size()); -} - -std::string MetaServiceUtils::schemaTagKey(GraphSpaceID spaceId, TagID tagId, SchemaVer version) { - auto storageVer = std::numeric_limits::max() - version; - std::string key; - key.reserve(kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID) + sizeof(SchemaVer)); - key.append(kTagsTable.data(), kTagsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&tagId), sizeof(TagID)) - .append(reinterpret_cast(&storageVer), sizeof(SchemaVer)); - return key; -} - -TagID MetaServiceUtils::parseTagId(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kTagsTable.size() + sizeof(GraphSpaceID)); -} - -SchemaVer MetaServiceUtils::parseTagVersion(folly::StringPiece key) { - auto offset = kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID); - return std::numeric_limits::max() - - *reinterpret_cast(key.begin() + offset); -} - -std::string MetaServiceUtils::schemaTagPrefix(GraphSpaceID spaceId, TagID tagId) { - std::string key; - key.reserve(kTagsTable.size() + sizeof(GraphSpaceID) + sizeof(TagID)); - key.append(kTagsTable.data(), kTagsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&tagId), sizeof(TagID)); - return key; -} - -std::string MetaServiceUtils::schemaTagsPrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kTagsTable.size() + sizeof(GraphSpaceID)); - key.append(kTagsTable.data(), kTagsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -GraphSpaceID MetaServiceUtils::parseTagsKeySpaceID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kTagsTable.size()); -} - -cpp2::Schema MetaServiceUtils::parseSchema(folly::StringPiece rawData) { - cpp2::Schema schema; - int32_t offset = sizeof(int32_t) + *reinterpret_cast(rawData.begin()); - auto schval = rawData.subpiece(offset, rawData.size() - offset); - apache::thrift::CompactSerializer::deserialize(schval, schema); - return schema; -} - -std::string MetaServiceUtils::indexKey(GraphSpaceID spaceID, IndexID indexID) { - std::string key; - key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID) + sizeof(IndexID)); - key.append(kIndexesTable.data(), kIndexesTable.size()) - .append(reinterpret_cast(&spaceID), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&indexID), sizeof(IndexID)); - return key; -} - -std::string MetaServiceUtils::indexVal(const nebula::meta::cpp2::IndexItem& item) { - std::string value; - apache::thrift::CompactSerializer::serialize(item, &value); - return value; -} - -std::string MetaServiceUtils::indexPrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID)); - key.append(kIndexesTable.data(), kIndexesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -GraphSpaceID MetaServiceUtils::parseIndexesKeySpaceID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kIndexesTable.size()); -} - -IndexID MetaServiceUtils::parseIndexesKeyIndexID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kIndexesTable.size() + - sizeof(GraphSpaceID)); -} - -nebula::meta::cpp2::IndexItem MetaServiceUtils::parseIndex(const folly::StringPiece& rawData) { - nebula::meta::cpp2::IndexItem item; - apache::thrift::CompactSerializer::deserialize(rawData, item); - return item; -} - -// This method should replace with JobManager when it ready. -std::string MetaServiceUtils::rebuildIndexStatus(GraphSpaceID space, - char type, - const std::string& indexName) { - std::string key; - key.reserve(64); - key.append(kIndexStatusTable.data(), kIndexStatusTable.size()) - .append(reinterpret_cast(&space), sizeof(GraphSpaceID)) - .append(1, type) - .append(indexName); - return key; -} - -// This method should replace with JobManager when it ready. -std::string MetaServiceUtils::rebuildIndexStatusPrefix(GraphSpaceID space, char type) { - std::string key; - key.reserve(kIndexStatusTable.size() + sizeof(GraphSpaceID) + sizeof(char)); - key.append(kIndexStatusTable.data(), kIndexStatusTable.size()) - .append(reinterpret_cast(&space), sizeof(GraphSpaceID)) - .append(1, type); - return key; -} - -std::string MetaServiceUtils::rebuildIndexStatusPrefix() { - std::string key; - key.reserve(kIndexStatusTable.size()); - key.append(kIndexStatusTable.data(), kIndexStatusTable.size()); - return key; -} - -GraphSpaceID MetaServiceUtils::parseIndexStatusKeySpaceID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kIndexStatusTable.size()); -} - -std::string MetaServiceUtils::indexSpaceKey(const std::string& name) { - EntryType type = EntryType::SPACE; - std::string key; - key.reserve(64); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(type)) - .append(name); - return key; -} - -std::string MetaServiceUtils::indexTagKey(GraphSpaceID spaceId, const std::string& name) { - EntryType type = EntryType::TAG; - std::string key; - key.reserve(128); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(type)) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(name); - return key; -} - -std::string MetaServiceUtils::indexEdgeKey(GraphSpaceID spaceId, const std::string& name) { - EntryType type = EntryType::EDGE; - std::string key; - key.reserve(128); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(type)) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(name); - return key; -} - -GraphSpaceID MetaServiceUtils::parseIndexKeySpaceID(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kIndexTable.size() + - sizeof(EntryType)); -} - -std::string MetaServiceUtils::indexIndexKey(GraphSpaceID spaceID, const std::string& indexName) { - EntryType type = EntryType::INDEX; - std::string key; - key.reserve(128); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(type)) - .append(reinterpret_cast(&spaceID), sizeof(GraphSpaceID)) - .append(indexName); - return key; -} - -std::string MetaServiceUtils::indexGroupKey(const std::string& name) { - EntryType type = EntryType::GROUP; - std::string key; - key.reserve(128); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(EntryType)) - .append(name); - return key; -} - -std::string MetaServiceUtils::indexZoneKey(const std::string& name) { - EntryType type = EntryType::ZONE; - std::string key; - key.reserve(128); - key.append(kIndexTable.data(), kIndexTable.size()) - .append(reinterpret_cast(&type), sizeof(type)) - .append(name); - return key; -} - -std::string MetaServiceUtils::assembleSegmentKey(const std::string& segment, - const std::string& key) { - std::string segmentKey; - segmentKey.reserve(64); - segmentKey.append(segment).append(key.data(), key.size()); - return segmentKey; -} - nebula::cpp2::ErrorCode MetaServiceUtils::alterColumnDefs(std::vector& cols, cpp2::SchemaProp& prop, const cpp2::ColumnDef col, @@ -795,213 +152,6 @@ nebula::cpp2::ErrorCode MetaServiceUtils::alterSchemaProp(std::vector(&pwdLen), sizeof(size_t)).append(val); - return key; -} - -std::string MetaServiceUtils::parseUser(folly::StringPiece key) { - return key.subpiece(kUsersTable.size(), key.size() - kUsersTable.size()).str(); -} - -std::string MetaServiceUtils::parseUserPwd(folly::StringPiece val) { - auto len = *reinterpret_cast(val.data()); - return val.subpiece(sizeof(size_t), len).str(); -} - -std::string MetaServiceUtils::roleKey(GraphSpaceID spaceId, const std::string& account) { - std::string key; - key.reserve(kRolesTable.size() + sizeof(GraphSpaceID) + account.size()); - key.append(kRolesTable.data(), kRolesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(account); - return key; -} - -std::string MetaServiceUtils::roleVal(cpp2::RoleType roleType) { - std::string val; - val.reserve(sizeof(cpp2::RoleType)); - val.append(reinterpret_cast(&roleType), sizeof(cpp2::RoleType)); - return val; -} - -std::string MetaServiceUtils::parseRoleUser(folly::StringPiece key) { - auto offset = kRolesTable.size() + sizeof(GraphSpaceID); - return key.subpiece(offset, key.size() - offset).str(); -} - -GraphSpaceID MetaServiceUtils::parseRoleSpace(folly::StringPiece key) { - return *reinterpret_cast(key.data() + kRolesTable.size()); -} - -std::string MetaServiceUtils::rolesPrefix() { return kRolesTable; } - -std::string MetaServiceUtils::roleSpacePrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kRolesTable.size() + sizeof(GraphSpaceID)); - key.append(kRolesTable.data(), kRolesTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -std::string MetaServiceUtils::parseRoleStr(folly::StringPiece key) { - auto* c = reinterpret_cast(&key); - auto type = *reinterpret_cast(c); - std::string role; - switch (type) { - case cpp2::RoleType::GOD: { - role = "GOD"; - break; - } - case cpp2::RoleType::ADMIN: { - role = "ADMIN"; - break; - } - case cpp2::RoleType::DBA: { - role = "DBA"; - break; - } - case cpp2::RoleType::USER: { - role = "USER"; - break; - } - case cpp2::RoleType::GUEST: { - role = "GUEST"; - break; - } - } - return role; -} - -std::string MetaServiceUtils::configKey(const cpp2::ConfigModule& module, const std::string& name) { - int32_t nSize = name.size(); - std::string key; - key.reserve(128); - key.append(kConfigsTable.data(), kConfigsTable.size()) - .append(reinterpret_cast(&module), sizeof(cpp2::ConfigModule)) - .append(reinterpret_cast(&nSize), sizeof(int32_t)) - .append(name); - return key; -} - -std::string MetaServiceUtils::configKeyPrefix(const cpp2::ConfigModule& module) { - std::string key; - key.reserve(128); - key.append(kConfigsTable.data(), kConfigsTable.size()); - if (module != cpp2::ConfigModule::ALL) { - key.append(reinterpret_cast(&module), sizeof(cpp2::ConfigModule)); - } - return key; -} - -std::string MetaServiceUtils::configValue(const cpp2::ConfigMode& valueMode, const Value& value) { - std::string val, cVal; - apache::thrift::CompactSerializer::serialize(value, &cVal); - val.reserve(sizeof(cpp2::ConfigMode) + cVal.size()); - val.append(reinterpret_cast(&valueMode), sizeof(cpp2::ConfigMode)).append(cVal); - return val; -} - -ConfigName MetaServiceUtils::parseConfigKey(folly::StringPiece rawKey) { - std::string key; - auto offset = kConfigsTable.size(); - auto module = *reinterpret_cast(rawKey.data() + offset); - offset += sizeof(cpp2::ConfigModule); - int32_t nSize = *reinterpret_cast(rawKey.data() + offset); - offset += sizeof(int32_t); - auto name = rawKey.subpiece(offset, nSize); - return {module, name.str()}; -} - -cpp2::ConfigItem MetaServiceUtils::parseConfigValue(folly::StringPiece rawData) { - int32_t offset = 0; - cpp2::ConfigMode mode = *reinterpret_cast(rawData.data() + offset); - offset += sizeof(cpp2::ConfigMode); - Value value; - apache::thrift::CompactSerializer::deserialize(rawData.subpiece(offset, rawData.size() - offset), - value); - - cpp2::ConfigItem item; - item.set_mode(mode); - item.set_value(value); - return item; -} - -std::string MetaServiceUtils::snapshotKey(const std::string& name) { - std::string key; - key.reserve(kSnapshotsTable.size() + name.size()); - key.append(kSnapshotsTable.data(), kSnapshotsTable.size()).append(name); - return key; -} - -std::string MetaServiceUtils::snapshotVal(const cpp2::SnapshotStatus& status, - const std::string& hosts) { - std::string val; - val.reserve(sizeof(cpp2::SnapshotStatus) + sizeof(hosts)); - val.append(reinterpret_cast(&status), sizeof(cpp2::SnapshotStatus)).append(hosts); - return val; -} - -cpp2::SnapshotStatus MetaServiceUtils::parseSnapshotStatus(folly::StringPiece rawData) { - return *reinterpret_cast(rawData.data()); -} - -std::string MetaServiceUtils::parseSnapshotHosts(folly::StringPiece rawData) { - return rawData - .subpiece(sizeof(cpp2::SnapshotStatus), rawData.size() - sizeof(cpp2::SnapshotStatus)) - .str(); -} - -std::string MetaServiceUtils::parseSnapshotName(folly::StringPiece rawData) { - int32_t offset = kSnapshotsTable.size(); - return rawData.subpiece(offset, rawData.size() - offset).str(); -} - -const std::string& MetaServiceUtils::snapshotPrefix() { return kSnapshotsTable; } - -std::string MetaServiceUtils::serializeHostAddr(const HostAddr& host) { - std::string ret; - ret.reserve(sizeof(size_t) + 15 + sizeof(Port)); // 255.255.255.255 - size_t len = host.host.size(); - ret.append(reinterpret_cast(&len), sizeof(size_t)) - .append(host.host.data(), len) - .append(reinterpret_cast(&host.port), sizeof(Port)); - return ret; -} - -HostAddr MetaServiceUtils::deserializeHostAddr(folly::StringPiece raw) { - HostAddr addr; - CHECK_GE(raw.size(), sizeof(size_t) + sizeof(Port)); // host may be "" - size_t offset = 0; - size_t len = *reinterpret_cast(raw.begin() + offset); - offset += sizeof(size_t); - addr.host = std::string(raw.begin() + offset, len); - offset += len; - addr.port = *reinterpret_cast(raw.begin() + offset); - return addr; -} - -std::string MetaServiceUtils::genTimestampStr() { - char ch[60]; - std::time_t t = std::time(nullptr); - std::strftime(ch, sizeof(ch), "%Y_%m_%d_%H_%M_%S", localtime(&t)); - return ch; -} - -std::string MetaServiceUtils::idKey() { return kIdKey; } - std::function MetaServiceUtils::spaceFilter( const std::unordered_set& spaces, std::function parseSpace) { @@ -1027,22 +177,23 @@ ErrorOr> MetaServiceUtils::bac const std::unordered_set& spaces, const std::string& backupName, const std::vector* spaceName) { + auto indexTable = MetaKeyUtils::getIndexTable(); return kvstore->backupTable( kDefaultSpaceId, backupName, - kIndexTable, - [spaces, spaceName](const folly::StringPiece& key) -> bool { + indexTable, + [spaces, spaceName, indexTable](const folly::StringPiece& key) -> bool { if (spaces.empty()) { return false; } - auto type = *reinterpret_cast(key.data() + kIndexTable.size()); + auto type = *reinterpret_cast(key.data() + indexTable.size()); if (type == EntryType::SPACE) { if (spaceName == nullptr) { return false; } - auto sn = key.subpiece(kIndexTable.size() + sizeof(EntryType), - key.size() - kIndexTable.size() - sizeof(EntryType)) + auto sn = key.subpiece(indexTable.size() + sizeof(EntryType), + key.size() - indexTable.size() - sizeof(EntryType)) .str(); LOG(INFO) << "sn was " << sn; auto it = std::find_if( @@ -1054,7 +205,7 @@ ErrorOr> MetaServiceUtils::bac return false; } - auto id = parseIndexKeySpaceID(key); + auto id = MetaKeyUtils::parseIndexKeySpaceID(key); auto it = spaces.find(id); if (it == spaces.end()) { return true; @@ -1070,9 +221,10 @@ ErrorOr> MetaServiceUtils::bac const std::string& backupName, const std::vector* spaceNames) { std::vector files; - files.reserve(tableMaps.size()); + auto tables = MetaKeyUtils::getTableMaps(); + files.reserve(tables.size()); - for (const auto& table : tableMaps) { + for (const auto& table : tables) { if (table.second.second == nullptr) { LOG(INFO) << table.first << " table skipped"; continue; @@ -1086,7 +238,8 @@ ErrorOr> MetaServiceUtils::bac } if (spaceNames == nullptr) { - for (const auto& table : systemTableMaps) { + auto sysTables = MetaKeyUtils::getSystemTableMaps(); + for (const auto& table : sysTables) { if (!table.second.second) { LOG(INFO) << table.first << " table skipped"; continue; @@ -1099,7 +252,8 @@ ErrorOr> MetaServiceUtils::bac } } - for (const auto& table : systemInfoMaps) { + auto sysInfos = MetaKeyUtils::getSystemInfoMaps(); + for (const auto& table : sysInfos) { if (!table.second.second) { LOG(INFO) << table.first << " table skipped"; continue; @@ -1128,311 +282,5 @@ ErrorOr> MetaServiceUtils::bac return files; } -std::string MetaServiceUtils::balanceTaskKey( - BalanceID balanceId, GraphSpaceID spaceId, PartitionID partId, HostAddr src, HostAddr dst) { - std::string str; - str.reserve(64); - str.append(reinterpret_cast(kBalanceTaskTable.data()), kBalanceTaskTable.size()) - .append(reinterpret_cast(&balanceId), sizeof(BalanceID)) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&partId), sizeof(PartitionID)) - .append(serializeHostAddr(src)) - .append(serializeHostAddr(dst)); - return str; -} - -std::string MetaServiceUtils::balanceTaskVal(BalanceTaskStatus status, - BalanceTaskResult retult, - int64_t startTime, - int64_t endTime) { - std::string val; - val.reserve(32); - val.append(reinterpret_cast(&status), sizeof(BalanceTaskStatus)) - .append(reinterpret_cast(&retult), sizeof(BalanceTaskResult)) - .append(reinterpret_cast(&startTime), sizeof(int64_t)) - .append(reinterpret_cast(&endTime), sizeof(int64_t)); - return val; -} - -std::string MetaServiceUtils::balanceTaskPrefix(BalanceID balanceId) { - std::string prefix; - prefix.reserve(32); - prefix.append(reinterpret_cast(kBalanceTaskTable.data()), kBalanceTaskTable.size()) - .append(reinterpret_cast(&balanceId), sizeof(BalanceID)); - return prefix; -} - -std::string MetaServiceUtils::balancePlanKey(BalanceID id) { - CHECK_GE(id, 0); - // make the balance id is stored in decend order - auto encode = folly::Endian::big(std::numeric_limits::max() - id); - std::string key; - key.reserve(sizeof(BalanceID) + kBalancePlanTable.size()); - key.append(reinterpret_cast(kBalancePlanTable.data()), kBalancePlanTable.size()) - .append(reinterpret_cast(&encode), sizeof(BalanceID)); - return key; -} - -std::string MetaServiceUtils::balancePlanVal(BalanceStatus status) { - std::string val; - val.reserve(sizeof(BalanceStatus)); - val.append(reinterpret_cast(&status), sizeof(BalanceStatus)); - return val; -} - -std::string MetaServiceUtils::balancePlanPrefix() { return kBalancePlanTable; } - -std::tuple -MetaServiceUtils::parseBalanceTaskKey(const folly::StringPiece& rawKey) { - uint32_t offset = kBalanceTaskTable.size(); - auto balanceId = *reinterpret_cast(rawKey.begin() + offset); - offset += sizeof(balanceId); - auto spaceId = *reinterpret_cast(rawKey.begin() + offset); - offset += sizeof(GraphSpaceID); - auto partId = *reinterpret_cast(rawKey.begin() + offset); - offset += sizeof(PartitionID); - auto src = MetaServiceUtils::deserializeHostAddr({rawKey, offset}); - offset += src.host.size() + sizeof(size_t) + sizeof(uint32_t); - auto dst = MetaServiceUtils::deserializeHostAddr({rawKey, offset}); - return std::make_tuple(balanceId, spaceId, partId, src, dst); -} - -std::tuple -MetaServiceUtils::parseBalanceTaskVal(const folly::StringPiece& rawVal) { - int32_t offset = 0; - auto status = *reinterpret_cast(rawVal.begin() + offset); - offset += sizeof(BalanceTaskStatus); - auto ret = *reinterpret_cast(rawVal.begin() + offset); - offset += sizeof(BalanceTaskResult); - auto start = *reinterpret_cast(rawVal.begin() + offset); - offset += sizeof(int64_t); - auto end = *reinterpret_cast(rawVal.begin() + offset); - return std::make_tuple(status, ret, start, end); -} - -std::string MetaServiceUtils::groupKey(const std::string& group) { - std::string key; - key.reserve(kGroupsTable.size() + group.size()); - key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); - return key; -} - -BalanceID MetaServiceUtils::parseBalanceID(const folly::StringPiece& rawKey) { - auto decode = *reinterpret_cast(rawKey.begin() + kBalancePlanTable.size()); - auto id = std::numeric_limits::max() - folly::Endian::big(decode); - CHECK_GE(id, 0); - return id; -} - -BalanceStatus MetaServiceUtils::parseBalanceStatus(const folly::StringPiece& rawVal) { - return static_cast(*rawVal.begin()); -} - -std::string MetaServiceUtils::groupVal(const std::vector& zones) { - return folly::join(",", zones); -} - -const std::string& MetaServiceUtils::groupPrefix() { return kGroupsTable; } - -std::string MetaServiceUtils::parseGroupName(folly::StringPiece rawData) { - return rawData.subpiece(kGroupsTable.size(), rawData.size()).toString(); -} - -std::vector MetaServiceUtils::parseZoneNames(folly::StringPiece rawData) { - std::vector zones; - folly::split(',', rawData.str(), zones); - return zones; -} - -std::string MetaServiceUtils::zoneKey(const std::string& zone) { - std::string key; - key.reserve(kZonesTable.size() + zone.size()); - key.append(kZonesTable.data(), kZonesTable.size()).append(zone); - return key; -} - -std::string MetaServiceUtils::zoneVal(const std::vector& hosts) { - std::string value; - value.append(network::NetworkUtils::toHostsStr(hosts)); - return value; -} - -const std::string& MetaServiceUtils::zonePrefix() { return kZonesTable; } - -std::string MetaServiceUtils::parseZoneName(folly::StringPiece rawData) { - return rawData.subpiece(kZonesTable.size(), rawData.size()).toString(); -} - -std::vector MetaServiceUtils::parseZoneHosts(folly::StringPiece rawData) { - std::vector addresses; - auto hostsOrErr = network::NetworkUtils::toHosts(rawData.str()); - if (hostsOrErr.ok()) { - addresses = std::move(hostsOrErr.value()); - } else { - LOG(ERROR) << "invalid input for parseZoneHosts()"; - } - return addresses; -} - -std::string MetaServiceUtils::listenerKey(GraphSpaceID spaceId, - PartitionID partId, - cpp2::ListenerType type) { - std::string key; - key.reserve(kListenerTable.size() + sizeof(GraphSpaceID) + sizeof(cpp2::ListenerType) + - sizeof(PartitionID)); - key.append(kListenerTable.data(), kListenerTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&type), sizeof(cpp2::ListenerType)) - .append(reinterpret_cast(&partId), sizeof(PartitionID)); - return key; -} - -std::string MetaServiceUtils::listenerPrefix(GraphSpaceID spaceId) { - std::string key; - key.reserve(kListenerTable.size() + sizeof(GraphSpaceID)); - key.append(kListenerTable.data(), kListenerTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -std::string MetaServiceUtils::listenerPrefix(GraphSpaceID spaceId, cpp2::ListenerType type) { - std::string key; - key.reserve(kListenerTable.size() + sizeof(GraphSpaceID) + sizeof(cpp2::ListenerType)); - key.append(kListenerTable.data(), kListenerTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)) - .append(reinterpret_cast(&type), sizeof(cpp2::ListenerType)); - return key; -} - -cpp2::ListenerType MetaServiceUtils::parseListenerType(folly::StringPiece rawData) { - auto offset = kListenerTable.size() + sizeof(GraphSpaceID); - return *reinterpret_cast(rawData.data() + offset); -} - -GraphSpaceID MetaServiceUtils::parseListenerSpace(folly::StringPiece rawData) { - auto offset = kListenerTable.size(); - return *reinterpret_cast(rawData.data() + offset); -} - -PartitionID MetaServiceUtils::parseListenerPart(folly::StringPiece rawData) { - auto offset = kListenerTable.size() + sizeof(cpp2::ListenerType) + sizeof(GraphSpaceID); - return *reinterpret_cast(rawData.data() + offset); -} - -std::string MetaServiceUtils::statsKey(GraphSpaceID spaceId) { - std::string key; - key.reserve(kStatsTable.size() + sizeof(GraphSpaceID)); - key.append(kStatsTable.data(), kStatsTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -std::string MetaServiceUtils::statsVal(const cpp2::StatsItem& statsItem) { - std::string val; - apache::thrift::CompactSerializer::serialize(statsItem, &val); - return val; -} - -cpp2::StatsItem MetaServiceUtils::parseStatsVal(folly::StringPiece rawData) { - cpp2::StatsItem statsItem; - apache::thrift::CompactSerializer::deserialize(rawData, statsItem); - return statsItem; -} - -GraphSpaceID MetaServiceUtils::parseStatsSpace(folly::StringPiece rawData) { - auto offset = kStatsTable.size(); - return *reinterpret_cast(rawData.data() + offset); -} - -const std::string& MetaServiceUtils::statsKeyPrefix() { return kStatsTable; } - -std::string MetaServiceUtils::fulltextServiceKey() { - std::string key; - key.reserve(kFTServiceTable.size()); - key.append(kFTServiceTable.data(), kFTServiceTable.size()); - return key; -} - -std::string MetaServiceUtils::fulltextServiceVal(cpp2::FTServiceType type, - const std::vector& clients) { - std::string val, cval; - apache::thrift::CompactSerializer::serialize(clients, &cval); - val.reserve(sizeof(cpp2::FTServiceType) + cval.size()); - val.append(reinterpret_cast(&type), sizeof(cpp2::FTServiceType)).append(cval); - return val; -} - -std::vector MetaServiceUtils::parseFTClients(folly::StringPiece rawData) { - std::vector clients; - int32_t offset = sizeof(cpp2::FTServiceType); - auto clientsRaw = rawData.subpiece(offset, rawData.size() - offset); - apache::thrift::CompactSerializer::deserialize(clientsRaw, clients); - return clients; -} - -const std::string& MetaServiceUtils::sessionPrefix() { return kSessionsTable; } - -std::string MetaServiceUtils::sessionKey(SessionID sessionId) { - std::string key; - key.reserve(kSessionsTable.size() + sizeof(sessionId)); - key.append(kSessionsTable.data(), kSessionsTable.size()) - .append(reinterpret_cast(&sessionId), sizeof(SessionID)); - return key; -} - -std::string MetaServiceUtils::sessionVal(const meta::cpp2::Session& session) { - std::string val; - apache::thrift::CompactSerializer::serialize(session, &val); - return val; -} - -SessionID MetaServiceUtils::getSessionId(const folly::StringPiece& key) { - return *reinterpret_cast(key.data() + kSessionsTable.size()); -} - -meta::cpp2::Session MetaServiceUtils::parseSessionVal(const folly::StringPiece& val) { - meta::cpp2::Session session; - apache::thrift::CompactSerializer::deserialize(val, session); - return session; -} - -std::string MetaServiceUtils::fulltextIndexKey(const std::string& indexName) { - std::string key; - key.reserve(kFTIndexTable.size() + indexName.size()); - key.append(kFTIndexTable.data(), kFTIndexTable.size()).append(indexName); - return key; -} - -std::string MetaServiceUtils::fulltextIndexVal(const cpp2::FTIndex& index) { - std::string val; - apache::thrift::CompactSerializer::serialize(index, &val); - return val; -} - -std::string MetaServiceUtils::parsefulltextIndexName(folly::StringPiece key) { - return key.subpiece(kFTIndexTable.size(), key.size()).toString(); -} - -cpp2::FTIndex MetaServiceUtils::parsefulltextIndex(folly::StringPiece val) { - cpp2::FTIndex ftIndex; - apache::thrift::CompactSerializer::deserialize(val, ftIndex); - return ftIndex; -} - -std::string MetaServiceUtils::fulltextIndexPrefix() { return kFTIndexTable; } - -std::string MetaServiceUtils::localIdKey(GraphSpaceID spaceId) { - std::string key; - key.reserve(kLocalIdTable.size() + sizeof(GraphSpaceID)); - key.append(kLocalIdTable.data(), kLocalIdTable.size()) - .append(reinterpret_cast(&spaceId), sizeof(GraphSpaceID)); - return key; -} - -GraphSpaceID MetaServiceUtils::parseLocalIdSpace(folly::StringPiece rawData) { - auto offset = kLocalIdTable.size(); - return *reinterpret_cast(rawData.data() + offset); -} - } // namespace meta } // namespace nebula diff --git a/src/meta/MetaServiceUtils.h b/src/meta/MetaServiceUtils.h index 63d00b82257..af40b6f8af9 100644 --- a/src/meta/MetaServiceUtils.h +++ b/src/meta/MetaServiceUtils.h @@ -17,336 +17,22 @@ namespace nebula { namespace meta { -enum class EntryType : int8_t { - SPACE = 0x01, - TAG = 0x02, - EDGE = 0x03, - INDEX = 0x04, - CONFIG = 0x05, - GROUP = 0x06, - ZONE = 0x07, -}; - -using ConfigName = std::pair; -using LeaderParts = std::unordered_map>; - class MetaServiceUtils final { public: MetaServiceUtils() = delete; - static std::string lastUpdateTimeKey(); - - static std::string lastUpdateTimeVal(const int64_t timeInMilliSec); - - static std::string spaceKey(GraphSpaceID spaceId); - - static std::string spaceVal(const cpp2::SpaceDesc& spaceDesc); - - static cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); - - static const std::string& spacePrefix(); - - static GraphSpaceID spaceId(folly::StringPiece rawKey); - - static std::string spaceName(folly::StringPiece rawVal); - - static std::string partKey(GraphSpaceID spaceId, PartitionID partId); - - static GraphSpaceID parsePartKeySpaceId(folly::StringPiece key); - - static PartitionID parsePartKeyPartId(folly::StringPiece key); - - static std::string partVal(const std::vector& hosts); - - static std::string partValV1(const std::vector& hosts); - - static std::string partValV2(const std::vector& hosts); - - static std::string partPrefix(); - - static std::string partPrefix(GraphSpaceID spaceId); - - static std::string encodeHostAddrV2(int ip, int port); - - static HostAddr decodeHostAddrV2(folly::StringPiece val, int& offset); - - static std::vector parsePartVal(folly::StringPiece val, int parNum = 0); - - static std::vector parsePartValV1(folly::StringPiece val); - - static std::vector parsePartValV2(folly::StringPiece val); - - static std::string hostKey(std::string ip, Port port); - - static std::string hostKeyV2(std::string addr, Port port); - - static const std::string& hostPrefix(); - - static HostAddr parseHostKey(folly::StringPiece key); - - static HostAddr parseHostKeyV1(folly::StringPiece key); - - static HostAddr parseHostKeyV2(folly::StringPiece key); - - static std::string leaderKey(std::string ip, Port port); - - static std::string leaderKeyV2(std::string addr, Port port); - - static std::string leaderKey(GraphSpaceID spaceId, PartitionID partId); - - static std::string leaderVal(const LeaderParts& leaderParts); - - static std::string leaderValV3(const HostAddr& h, int64_t term); - - static std::string leaderPrefix(GraphSpaceID spaceId); - - static const std::string& leaderPrefix(); - - static HostAddr parseLeaderKey(folly::StringPiece key); - - static HostAddr parseLeaderKeyV1(folly::StringPiece key); - - static std::pair parseLeaderKeyV3(folly::StringPiece key); - - static HostAddr parseLeaderKeyV2(folly::StringPiece key); - - static LeaderParts parseLeaderValV1(folly::StringPiece val); - - static std::tuple parseLeaderValV3( - folly::StringPiece val); - - static std::string schemaVal(const std::string& name, const cpp2::Schema& schema); - - static std::string schemaEdgePrefix(GraphSpaceID spaceId, EdgeType edgeType); - - static std::string schemaEdgesPrefix(GraphSpaceID spaceId); - - static std::string schemaEdgeKey(GraphSpaceID spaceId, EdgeType edgeType, SchemaVer version); - - static EdgeType parseEdgeType(folly::StringPiece key); - - static SchemaVer parseEdgeVersion(folly::StringPiece key); - - static std::string schemaTagKey(GraphSpaceID spaceId, TagID tagId, SchemaVer version); - - static TagID parseTagId(folly::StringPiece key); - - static SchemaVer parseTagVersion(folly::StringPiece key); - - static std::string schemaTagPrefix(GraphSpaceID spaceId, TagID tagId); - - static std::string schemaTagsPrefix(GraphSpaceID spaceId); - - static cpp2::Schema parseSchema(folly::StringPiece rawData); - - static std::string indexKey(GraphSpaceID spaceId, IndexID indexID); - - static std::string indexVal(const cpp2::IndexItem& item); - - static std::string indexPrefix(GraphSpaceID spaceId); - - static IndexID parseIndexesKeyIndexID(folly::StringPiece key); - - static cpp2::IndexItem parseIndex(const folly::StringPiece& rawData); - - static std::string rebuildIndexStatus(GraphSpaceID space, - char type, - const std::string& indexName); - - static std::string rebuildIndexStatusPrefix(GraphSpaceID spaceId, char type); - - static std::string rebuildIndexStatusPrefix(); - - static std::string rebuildTagIndexStatusPrefix(GraphSpaceID spaceId) { - return rebuildIndexStatusPrefix(spaceId, 'T'); - } - - static std::string rebuildEdgeIndexStatusPrefix(GraphSpaceID spaceId) { - return rebuildIndexStatusPrefix(spaceId, 'E'); - } - - static std::string indexSpaceKey(const std::string& name); - - static std::string indexTagKey(GraphSpaceID spaceId, const std::string& name); - - static std::string indexEdgeKey(GraphSpaceID spaceId, const std::string& name); - - static std::string indexIndexKey(GraphSpaceID spaceId, const std::string& name); - - static std::string indexGroupKey(const std::string& name); - - static std::string indexZoneKey(const std::string& name); - - static std::string assembleSegmentKey(const std::string& segment, const std::string& key); - static nebula::cpp2::ErrorCode alterColumnDefs(std::vector& cols, cpp2::SchemaProp& prop, const cpp2::ColumnDef col, const cpp2::AlterSchemaOp op, bool isEdge = false); - static std::string userPrefix(); - static nebula::cpp2::ErrorCode alterSchemaProp(std::vector& cols, cpp2::SchemaProp& schemaProp, cpp2::SchemaProp alterSchemaProp, bool existIndex, bool isEdge = false); - static std::string userKey(const std::string& account); - - static std::string userVal(const std::string& val); - - static std::string parseUser(folly::StringPiece key); - - static std::string parseUserPwd(folly::StringPiece val); - - static std::string roleKey(GraphSpaceID spaceId, const std::string& account); - - static std::string roleVal(cpp2::RoleType roleType); - - static std::string parseRoleUser(folly::StringPiece key); - - static GraphSpaceID parseRoleSpace(folly::StringPiece key); - - static std::string rolesPrefix(); - - static std::string roleSpacePrefix(GraphSpaceID spaceId); - - static std::string parseRoleStr(folly::StringPiece key); - - static std::string configKey(const cpp2::ConfigModule& module, const std::string& name); - - static std::string configKeyPrefix(const cpp2::ConfigModule& module); - - static std::string configValue(const cpp2::ConfigMode& valueMode, const Value& config); - - static ConfigName parseConfigKey(folly::StringPiece rawData); - - static cpp2::ConfigItem parseConfigValue(folly::StringPiece rawData); - - static std::string snapshotKey(const std::string& name); - - static std::string snapshotVal(const cpp2::SnapshotStatus& status, const std::string& hosts); - - static cpp2::SnapshotStatus parseSnapshotStatus(folly::StringPiece rawData); - - static std::string parseSnapshotHosts(folly::StringPiece rawData); - - static std::string parseSnapshotName(folly::StringPiece rawData); - - static const std::string& snapshotPrefix(); - - static std::string serializeHostAddr(const HostAddr& host); - - static HostAddr deserializeHostAddr(folly::StringPiece str); - - static std::string balanceTaskKey( - BalanceID balanceId, GraphSpaceID spaceId, PartitionID partId, HostAddr src, HostAddr dst); - - static std::string balanceTaskVal(BalanceTaskStatus status, - BalanceTaskResult retult, - int64_t startTime, - int64_t endTime); - - static std::string balanceTaskPrefix(BalanceID balanceId); - - static std::string balancePlanKey(BalanceID id); - - static std::string balancePlanVal(BalanceStatus status); - - static std::string balancePlanPrefix(); - - static BalanceID parseBalanceID(const folly::StringPiece& rawKey); - - static BalanceStatus parseBalanceStatus(const folly::StringPiece& rawVal); - - static std::tuple parseBalanceTaskKey( - const folly::StringPiece& rawKey); - - static std::tuple parseBalanceTaskVal( - const folly::StringPiece& rawVal); - - static std::string groupKey(const std::string& group); - - static std::string groupVal(const std::vector& zones); - - static const std::string& groupPrefix(); - - static std::string parseGroupName(folly::StringPiece rawData); - - static std::vector parseZoneNames(folly::StringPiece rawData); - - static std::string zoneKey(const std::string& zone); - - static std::string zoneVal(const std::vector& hosts); - - static const std::string& zonePrefix(); - - static std::string parseZoneName(folly::StringPiece rawData); - - static std::vector parseZoneHosts(folly::StringPiece rawData); - - static std::string listenerKey(GraphSpaceID spaceId, PartitionID partId, cpp2::ListenerType type); - - static std::string listenerPrefix(GraphSpaceID spaceId); - - static std::string listenerPrefix(GraphSpaceID spaceId, cpp2::ListenerType type); - - static cpp2::ListenerType parseListenerType(folly::StringPiece rawData); - - static GraphSpaceID parseListenerSpace(folly::StringPiece rawData); - - static PartitionID parseListenerPart(folly::StringPiece rawData); - - static std::string statsKey(GraphSpaceID spaceId); - - static std::string statsVal(const cpp2::StatsItem& statsItem); - - static cpp2::StatsItem parseStatsVal(folly::StringPiece rawData); - - static const std::string& statsKeyPrefix(); - - static GraphSpaceID parseStatsSpace(folly::StringPiece rawData); - - static std::string fulltextServiceKey(); - - static std::string fulltextServiceVal(cpp2::FTServiceType type, - const std::vector& clients); - - static std::vector parseFTClients(folly::StringPiece rawData); - - static const std::string& sessionPrefix(); - - static std::string sessionKey(SessionID sessionId); - - static std::string sessionVal(const meta::cpp2::Session& session); - - static SessionID getSessionId(const folly::StringPiece& key); - - static meta::cpp2::Session parseSessionVal(const folly::StringPiece& val); - - static std::string fulltextIndexKey(const std::string& indexName); - - static std::string fulltextIndexVal(const cpp2::FTIndex& index); - - static std::string parsefulltextIndexName(folly::StringPiece key); - - static cpp2::FTIndex parsefulltextIndex(folly::StringPiece val); - - static std::string fulltextIndexPrefix(); - - static std::string genTimestampStr(); - - static GraphSpaceID parseEdgesKeySpaceID(folly::StringPiece key); - static GraphSpaceID parseTagsKeySpaceID(folly::StringPiece key); - static GraphSpaceID parseIndexesKeySpaceID(folly::StringPiece key); - static GraphSpaceID parseIndexStatusKeySpaceID(folly::StringPiece key); - static GraphSpaceID parseIndexKeySpaceID(folly::StringPiece key); - static GraphSpaceID parseDefaultKeySpaceID(folly::StringPiece key); - - static std::string idKey(); - // backup static ErrorOr> backupIndex( kvstore::KVStore* kvstore, @@ -363,10 +49,6 @@ class MetaServiceUtils final { const std::unordered_set& spaces, const std::string& backupName, const std::vector* spaceName); - - static std::string localIdKey(GraphSpaceID spaceId); - - static GraphSpaceID parseLocalIdSpace(folly::StringPiece rawData); }; } // namespace meta diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index e3551632500..efa8fc72073 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -35,7 +35,7 @@ MetaVersion MetaVersionMan::getMetaVersionFromKV(kvstore::KVStore* kv) { // static MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { - const auto& hostPrefix = nebula::meta::MetaServiceUtils::hostPrefix(); + const auto& hostPrefix = nebula::MetaKeyUtils::hostPrefix(); std::unique_ptr iter; auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, hostPrefix, &iter, true); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -75,8 +75,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { // static Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { CHECK_NOTNULL(kv); - auto snapshot = - folly::format("META_UPGRADE_SNAPSHOT_{}", MetaServiceUtils::genTimestampStr()).str(); + auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); if (meteRet.isLeftType()) { LOG(ERROR) << "Create snapshot failed: " << snapshot; @@ -311,9 +310,9 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { upgrader.printIndexes(iter->val()); } auto oldItem = meta::v1::MetaServiceUtilsV1::parseIndex(iter->val()); - auto spaceId = MetaServiceUtils::parseIndexesKeySpaceID(iter->key()); + auto spaceId = MetaKeyUtils::parseIndexesKeySpaceID(iter->key()); status = upgrader.deleteKeyVal( - MetaServiceUtils::indexIndexKey(spaceId, oldItem.get_index_name())); + MetaKeyUtils::indexIndexKey(spaceId, oldItem.get_index_name())); if (!status.ok()) { LOG(ERROR) << status; return status; diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 2224758ff40..dc0117cd2b3 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -8,8 +8,8 @@ #define META_METAVERSIONMAN_H_ #include "common/base/Base.h" +#include "common/utils/MetaKeyUtils.h" #include "kvstore/KVStore.h" -#include "meta/MetaServiceUtils.h" namespace nebula { namespace meta { diff --git a/src/meta/RootUserMan.h b/src/meta/RootUserMan.h index dafb41c70b7..9ea8a88dc97 100644 --- a/src/meta/RootUserMan.h +++ b/src/meta/RootUserMan.h @@ -9,8 +9,8 @@ #include "common/base/Base.h" #include "common/encryption/MD5Utils.h" +#include "common/utils/MetaKeyUtils.h" #include "kvstore/KVStore.h" -#include "meta/MetaServiceUtils.h" namespace nebula { namespace meta { @@ -21,7 +21,7 @@ namespace meta { class RootUserMan { public: static bool isUserExists(kvstore::KVStore* kv) { - auto userKey = MetaServiceUtils::userKey("root"); + auto userKey = MetaKeyUtils::userKey("root"); std::string val; auto result = kv->get(kDefaultSpaceId, kDefaultPartId, userKey, &val); if (result == nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -36,10 +36,10 @@ class RootUserMan { static bool initRootUser(kvstore::KVStore* kv) { LOG(INFO) << "Init root user"; auto encodedPwd = encryption::MD5Utils::md5Encode("nebula"); - auto userKey = MetaServiceUtils::userKey("root"); - auto userVal = MetaServiceUtils::userVal(std::move(encodedPwd)); - auto roleKey = MetaServiceUtils::roleKey(kDefaultSpaceId, "root"); - auto roleVal = MetaServiceUtils::roleVal(cpp2::RoleType::GOD); + auto userKey = MetaKeyUtils::userKey("root"); + auto userVal = MetaKeyUtils::userVal(std::move(encodedPwd)); + auto roleKey = MetaKeyUtils::roleKey(kDefaultSpaceId, "root"); + auto roleVal = MetaKeyUtils::roleVal(cpp2::RoleType::GOD); std::vector data; data.emplace_back(std::move(userKey), std::move(userVal)); diff --git a/src/meta/common/MetaCommon.h b/src/meta/common/MetaCommon.h index a1bef8e6ae4..cd5e6758c44 100644 --- a/src/meta/common/MetaCommon.h +++ b/src/meta/common/MetaCommon.h @@ -9,6 +9,7 @@ #include "common/base/Base.h" #include "common/base/Status.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/meta_types.h" #include "kvstore/KVStore.h" #include "meta/processors/Common.h" diff --git a/src/meta/http/MetaHttpDownloadHandler.cpp b/src/meta/http/MetaHttpDownloadHandler.cpp index 1912728333e..5ddfd8b85cb 100644 --- a/src/meta/http/MetaHttpDownloadHandler.cpp +++ b/src/meta/http/MetaHttpDownloadHandler.cpp @@ -17,7 +17,7 @@ #include "common/network/NetworkUtils.h" #include "common/process/ProcessUtils.h" #include "common/thread/GenericThreadPool.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "webservice/Common.h" #include "webservice/WebService.h" @@ -132,7 +132,7 @@ bool MetaHttpDownloadHandler::dispatchSSTFiles(const std::string &hdfsHost, folly::split("\n", result.value(), files, true); std::unique_ptr iter; - auto prefix = MetaServiceUtils::partPrefix(spaceID_); + auto prefix = MetaKeyUtils::partPrefix(spaceID_); auto ret = kvstore_->prefix(0, 0, prefix, &iter); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Fetch Parts Failed"; @@ -145,7 +145,7 @@ bool MetaHttpDownloadHandler::dispatchSSTFiles(const std::string &hdfsHost, auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - for (auto host : MetaServiceUtils::parsePartVal(iter->val())) { + for (auto host : MetaKeyUtils::parsePartVal(iter->val())) { auto address = HostAddr(host.host, host.port); auto addressIter = hostPartition.find(address); if (addressIter == hostPartition.end()) { diff --git a/src/meta/http/MetaHttpIngestHandler.cpp b/src/meta/http/MetaHttpIngestHandler.cpp index 0f73c47f5a4..ba381600459 100644 --- a/src/meta/http/MetaHttpIngestHandler.cpp +++ b/src/meta/http/MetaHttpIngestHandler.cpp @@ -14,7 +14,7 @@ #include "common/network/NetworkUtils.h" #include "common/process/ProcessUtils.h" #include "common/thread/GenericThreadPool.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "webservice/Common.h" #include "webservice/WebService.h" @@ -106,7 +106,7 @@ void MetaHttpIngestHandler::onError(ProxygenError error) noexcept { bool MetaHttpIngestHandler::ingestSSTFiles(GraphSpaceID space) { std::unique_ptr iter; - auto prefix = MetaServiceUtils::partPrefix(space); + auto prefix = MetaKeyUtils::partPrefix(space); static const GraphSpaceID metaSpaceId = 0; static const PartitionID metaPartId = 0; @@ -118,7 +118,7 @@ bool MetaHttpIngestHandler::ingestSSTFiles(GraphSpaceID space) { std::set storageIPs; while (iter->valid()) { - for (auto &host : MetaServiceUtils::parsePartVal(iter->val())) { + for (auto &host : MetaKeyUtils::parsePartVal(iter->val())) { if (storageIPs.count(host.host) == 0) { storageIPs.insert(std::move(host.host)); } diff --git a/src/meta/http/MetaHttpReplaceHostHandler.cpp b/src/meta/http/MetaHttpReplaceHostHandler.cpp index 970012e90a5..fd9294f56fe 100644 --- a/src/meta/http/MetaHttpReplaceHostHandler.cpp +++ b/src/meta/http/MetaHttpReplaceHostHandler.cpp @@ -14,7 +14,7 @@ #include "common/network/NetworkUtils.h" #include "common/process/ProcessUtils.h" #include "common/thread/GenericThreadPool.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/processors/Common.h" #include "webservice/Common.h" #include "webservice/WebService.h" @@ -107,7 +107,7 @@ void MetaHttpReplaceHostHandler::onError(ProxygenError error) noexcept { bool MetaHttpReplaceHostHandler::replaceHost(std::string ipv4From, std::string ipv4To) { folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock()); - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); std::unique_ptr iter; auto kvRet = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, spacePrefix, &iter); if (kvRet != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -118,7 +118,7 @@ bool MetaHttpReplaceHostHandler::replaceHost(std::string ipv4From, std::string i std::vector allSpaceId; while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); allSpaceId.emplace_back(spaceId); iter->next(); } @@ -126,7 +126,7 @@ bool MetaHttpReplaceHostHandler::replaceHost(std::string ipv4From, std::string i std::vector data; for (const auto& spaceId : allSpaceId) { - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); kvRet = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, partPrefix, &iter); if (kvRet != nebula::cpp2::ErrorCode::SUCCEEDED) { errMsg_ = folly::stringPrintf("can't get partPrefix=%s", partPrefix.c_str()); @@ -136,7 +136,7 @@ bool MetaHttpReplaceHostHandler::replaceHost(std::string ipv4From, std::string i while (iter->valid()) { bool needUpdate = false; - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& host : partHosts) { if (host.host == ipv4From) { needUpdate = true; @@ -144,7 +144,7 @@ bool MetaHttpReplaceHostHandler::replaceHost(std::string ipv4From, std::string i } } if (needUpdate) { - data.emplace_back(iter->key(), MetaServiceUtils::partVal(partHosts)); + data.emplace_back(iter->key(), MetaKeyUtils::partVal(partHosts)); } iter->next(); } diff --git a/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp b/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp index 62bbd608236..e5fd4355649 100644 --- a/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp +++ b/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp @@ -150,23 +150,23 @@ std::vector dumpKVStore(kvstore::KVStore* kvstore) { std::vector ret; // Get all spaces std::vector allSpaceId; - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); std::unique_ptr iter; auto kvRet = kvstore->prefix(kDefaultSpaceId, kDefaultPartId, spacePrefix, &iter); EXPECT_EQ(kvRet, nebula::cpp2::ErrorCode::SUCCEEDED); while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); allSpaceId.emplace_back(spaceId); iter->next(); } LOG(INFO) << "allSpaceId.size()=" << allSpaceId.size(); for (const auto& spaceId : allSpaceId) { - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); kvRet = kvstore->prefix(kDefaultSpaceId, kDefaultPartId, partPrefix, &iter); EXPECT_EQ(kvRet, nebula::cpp2::ErrorCode::SUCCEEDED); while (iter->valid()) { - auto hostAddrs = MetaServiceUtils::parsePartVal(iter->val()); + auto hostAddrs = MetaKeyUtils::parsePartVal(iter->val()); LOG(INFO) << "hostAddrs.size()=" << hostAddrs.size(); for (auto& hostAddr : hostAddrs) { ret.emplace_back(hostAddr.host); diff --git a/src/meta/processors/BaseProcessor-inl.h b/src/meta/processors/BaseProcessor-inl.h index 79c4378ad69..a947aa07aa5 100644 --- a/src/meta/processors/BaseProcessor-inl.h +++ b/src/meta/processors/BaseProcessor-inl.h @@ -120,7 +120,7 @@ ErrorOr> BaseProcessor:: template ErrorOr> BaseProcessor::allHosts() { std::vector hosts; - const auto& prefix = MetaServiceUtils::hostPrefix(); + const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; auto code = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -141,7 +141,7 @@ ErrorOr> BaseProcessor::all template ErrorOr BaseProcessor::autoIncrementId() { folly::SharedMutex::WriteHolder holder(LockUtils::idLock()); - const std::string kIdKey = MetaServiceUtils::idKey(); + const std::string kIdKey = MetaKeyUtils::idKey(); int32_t id; std::string val; auto ret = kvstore_->get(kDefaultSpaceId, kDefaultPartId, kIdKey, &val); @@ -195,7 +195,7 @@ ErrorOr BaseProcessor::autoIncrementIdIn folly::SharedMutex::WriteHolder wHolder(LockUtils::localIdLock()); folly::SharedMutex::ReadHolder rHolder(LockUtils::idLock()); - auto localIdkey = MetaServiceUtils::localIdKey(spaceId); + auto localIdkey = MetaKeyUtils::localIdKey(spaceId); int32_t id; std::string val; auto ret = kvstore_->get(kDefaultSpaceId, kDefaultPartId, localIdkey, &val); @@ -235,7 +235,7 @@ ErrorOr BaseProcessor::autoIncrementIdIn template nebula::cpp2::ErrorCode BaseProcessor::spaceExist(GraphSpaceID spaceId) { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - auto spaceKey = MetaServiceUtils::spaceKey(spaceId); + auto spaceKey = MetaKeyUtils::spaceKey(spaceId); auto ret = doGet(std::move(spaceKey)); if (nebula::ok(ret)) { return nebula::cpp2::ErrorCode::SUCCEEDED; @@ -249,7 +249,7 @@ nebula::cpp2::ErrorCode BaseProcessor::spaceExist(GraphSpaceID spaceId) { template nebula::cpp2::ErrorCode BaseProcessor::userExist(const std::string& account) { - auto userKey = MetaServiceUtils::userKey(account); + auto userKey = MetaKeyUtils::userKey(account); auto ret = doGet(std::move(userKey)); if (nebula::ok(ret)) { return nebula::cpp2::ErrorCode::SUCCEEDED; @@ -273,7 +273,7 @@ nebula::cpp2::ErrorCode BaseProcessor::hostExist(const std::string& hostKe template ErrorOr BaseProcessor::getSpaceId( const std::string& name) { - auto indexKey = MetaServiceUtils::indexSpaceKey(name); + auto indexKey = MetaKeyUtils::indexSpaceKey(name); auto ret = doGet(indexKey); if (nebula::ok(ret)) { return *reinterpret_cast(nebula::value(ret).c_str()); @@ -288,7 +288,7 @@ ErrorOr BaseProcessor::getSpaceId( template ErrorOr BaseProcessor::getTagId(GraphSpaceID spaceId, const std::string& name) { - auto indexKey = MetaServiceUtils::indexTagKey(spaceId, name); + auto indexKey = MetaKeyUtils::indexTagKey(spaceId, name); std::string val; auto ret = doGet(std::move(indexKey)); if (nebula::ok(ret)) { @@ -304,7 +304,7 @@ ErrorOr BaseProcessor::getTagId(GraphSpace template ErrorOr BaseProcessor::getEdgeType( GraphSpaceID spaceId, const std::string& name) { - auto indexKey = MetaServiceUtils::indexEdgeKey(spaceId, name); + auto indexKey = MetaKeyUtils::indexEdgeKey(spaceId, name); auto ret = doGet(std::move(indexKey)); if (nebula::ok(ret)) { return *reinterpret_cast(nebula::value(ret).c_str()); @@ -319,7 +319,7 @@ ErrorOr BaseProcessor::getEdgeType( template ErrorOr BaseProcessor::getLatestTagSchema( GraphSpaceID spaceId, const TagID tagId) { - const auto& key = MetaServiceUtils::schemaTagPrefix(spaceId, tagId); + const auto& key = MetaKeyUtils::schemaTagPrefix(spaceId, tagId); auto ret = doPrefix(key); if (!nebula::ok(ret)) { LOG(ERROR) << "Tag Prefix " << key << " failed"; @@ -328,7 +328,7 @@ ErrorOr BaseProcessor::getLatestTag auto iter = nebula::value(ret).get(); if (iter->valid()) { - return MetaServiceUtils::parseSchema(iter->val()); + return MetaKeyUtils::parseSchema(iter->val()); } else { LOG(ERROR) << "Tag Prefix " << key << " not found"; return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; @@ -338,7 +338,7 @@ ErrorOr BaseProcessor::getLatestTag template ErrorOr BaseProcessor::getLatestEdgeSchema( GraphSpaceID spaceId, const EdgeType edgeType) { - const auto& key = MetaServiceUtils::schemaEdgePrefix(spaceId, edgeType); + const auto& key = MetaKeyUtils::schemaEdgePrefix(spaceId, edgeType); auto ret = doPrefix(key); if (!nebula::ok(ret)) { LOG(ERROR) << "Edge Prefix " << key << " failed"; @@ -347,7 +347,7 @@ ErrorOr BaseProcessor::getLatestEdg auto iter = nebula::value(ret).get(); if (iter->valid()) { - return MetaServiceUtils::parseSchema(iter->val()); + return MetaKeyUtils::parseSchema(iter->val()); } else { LOG(ERROR) << "Edge Prefix " << key << " not found"; return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; @@ -357,7 +357,7 @@ ErrorOr BaseProcessor::getLatestEdg template ErrorOr BaseProcessor::getIndexID( GraphSpaceID spaceId, const std::string& indexName) { - auto indexKey = MetaServiceUtils::indexIndexKey(spaceId, indexName); + auto indexKey = MetaKeyUtils::indexIndexKey(spaceId, indexName); auto ret = doGet(indexKey); if (nebula::ok(ret)) { return *reinterpret_cast(nebula::value(ret).c_str()); @@ -372,10 +372,10 @@ ErrorOr BaseProcessor::getIndexID( template ErrorOr BaseProcessor::checkPassword( const std::string& account, const std::string& password) { - auto userKey = MetaServiceUtils::userKey(account); + auto userKey = MetaKeyUtils::userKey(account); auto ret = doGet(userKey); if (nebula::ok(ret)) { - return MetaServiceUtils::parseUserPwd(nebula::value(ret)) == password; + return MetaKeyUtils::parseUserPwd(nebula::value(ret)) == password; } return nebula::error(ret); } @@ -452,7 +452,7 @@ template ErrorOr> BaseProcessor::getIndexes( GraphSpaceID spaceId, int32_t tagOrEdge) { std::vector items; - const auto& indexPrefix = MetaServiceUtils::indexPrefix(spaceId); + const auto& indexPrefix = MetaKeyUtils::indexPrefix(spaceId); auto iterRet = doPrefix(indexPrefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -463,7 +463,7 @@ ErrorOr> BaseProcessorvalid()) { - auto item = MetaServiceUtils::parseIndex(indexIter->val()); + auto item = MetaKeyUtils::parseIndex(indexIter->val()); if (item.get_schema_id().getType() == nebula::cpp2::SchemaID::Type::tag_id && item.get_schema_id().get_tag_id() == tagOrEdge) { items.emplace_back(std::move(item)); @@ -478,7 +478,7 @@ ErrorOr> BaseProcessor ErrorOr BaseProcessor::getFTIndex( GraphSpaceID spaceId, int32_t tagOrEdge) { - const auto& indexPrefix = MetaServiceUtils::fulltextIndexPrefix(); + const auto& indexPrefix = MetaKeyUtils::fulltextIndexPrefix(); auto iterRet = doPrefix(indexPrefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -489,7 +489,7 @@ ErrorOr BaseProcessor::getFTIndex( auto indexIter = nebula::value(iterRet).get(); while (indexIter->valid()) { - auto index = MetaServiceUtils::parsefulltextIndex(indexIter->val()); + auto index = MetaKeyUtils::parsefulltextIndex(indexIter->val()); auto id = index.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type ? index.get_depend_schema().get_edge_type() : index.get_depend_schema().get_tag_id(); @@ -570,7 +570,7 @@ bool BaseProcessor::checkIndexExist(const std::vector template ErrorOr BaseProcessor::getGroupId( const std::string& groupName) { - auto indexKey = MetaServiceUtils::indexGroupKey(groupName); + auto indexKey = MetaKeyUtils::indexGroupKey(groupName); auto ret = doGet(std::move(indexKey)); if (nebula::ok(ret)) { return *reinterpret_cast(nebula::value(ret).c_str()); @@ -585,7 +585,7 @@ ErrorOr BaseProcessor::getGroupId( template ErrorOr BaseProcessor::getZoneId( const std::string& zoneName) { - auto indexKey = MetaServiceUtils::indexZoneKey(zoneName); + auto indexKey = MetaKeyUtils::indexZoneKey(zoneName); auto ret = doGet(std::move(indexKey)); if (nebula::ok(ret)) { return *reinterpret_cast(nebula::value(ret).c_str()); @@ -601,7 +601,7 @@ template nebula::cpp2::ErrorCode BaseProcessor::listenerExist(GraphSpaceID space, cpp2::ListenerType type) { folly::SharedMutex::ReadHolder rHolder(LockUtils::listenerLock()); - const auto& prefix = MetaServiceUtils::listenerPrefix(space, type); + const auto& prefix = MetaKeyUtils::listenerPrefix(space, type); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { return nebula::error(ret); diff --git a/src/meta/processors/BaseProcessor.h b/src/meta/processors/BaseProcessor.h index fac215e14d1..40ba29fe438 100644 --- a/src/meta/processors/BaseProcessor.h +++ b/src/meta/processors/BaseProcessor.h @@ -16,6 +16,7 @@ #include "common/charset/Charset.h" #include "common/network/NetworkUtils.h" #include "common/time/Duration.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/storage_types.h" #include "kvstore/KVStore.h" #include "meta/ActiveHostsMan.h" diff --git a/src/meta/processors/Common.h b/src/meta/processors/Common.h index d1b4c9a5959..f897cbd48a9 100644 --- a/src/meta/processors/Common.h +++ b/src/meta/processors/Common.h @@ -12,44 +12,6 @@ namespace nebula { namespace meta { -static const PartitionID kDefaultPartId = 0; -static const GraphSpaceID kDefaultSpaceId = 0; - -using BalanceID = int64_t; - -enum class BalanceTaskStatus : uint8_t { - START = 0x01, - CHANGE_LEADER = 0x02, - ADD_PART_ON_DST = 0x03, - ADD_LEARNER = 0x04, - CATCH_UP_DATA = 0x05, - MEMBER_CHANGE_ADD = 0x06, - MEMBER_CHANGE_REMOVE = 0x07, - UPDATE_PART_META = 0x08, // After this state, we can't rollback anymore. - REMOVE_PART_ON_SRC = 0x09, - CHECK = 0x0A, - END = 0xFF, -}; - -enum class BalanceTaskResult : uint8_t { - SUCCEEDED = 0x01, - FAILED = 0x02, - IN_PROGRESS = 0x03, - INVALID = 0x04, -}; - -enum class BalanceStatus : uint8_t { - NOT_START = 0x01, - IN_PROGRESS = 0x02, - SUCCEEDED = 0x03, - /** - * TODO(heng): Currently, after the plan failed, we will try to resume it - * when running "balance" again. But in many cases, the plan will be failed - * forever, it this cases, we should rollback the plan. - * */ - FAILED = 0x04, -}; - class LockUtils { public: LockUtils() = delete; diff --git a/src/meta/processors/admin/AdminClient.cpp b/src/meta/processors/admin/AdminClient.cpp index 704a1e896f4..cbad41e5646 100644 --- a/src/meta/processors/admin/AdminClient.cpp +++ b/src/meta/processors/admin/AdminClient.cpp @@ -8,10 +8,10 @@ #include +#include "common/utils/MetaKeyUtils.h" #include "common/utils/Utils.h" #include "kvstore/Part.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" @@ -224,7 +224,7 @@ folly::Future AdminClient::updateMeta(GraphSpaceID spaceId, folly::Promise pro; auto f = pro.getFuture(); std::vector data; - data.emplace_back(MetaServiceUtils::partKey(spaceId, partId), MetaServiceUtils::partVal(peers)); + data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(peers)); kv_->asyncMultiPut( kDefaultSpaceId, kDefaultPartId, @@ -524,11 +524,11 @@ void AdminClient::getResponse(std::vector hosts, ErrorOr> AdminClient::getPeers(GraphSpaceID spaceId, PartitionID partId) { CHECK_NOTNULL(kv_); - auto partKey = MetaServiceUtils::partKey(spaceId, partId); + auto partKey = MetaKeyUtils::partKey(spaceId, partId); std::string value; auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, partKey, &value); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { - return MetaServiceUtils::parsePartVal(value); + return MetaKeyUtils::parsePartVal(value); } return code; } diff --git a/src/meta/processors/admin/BalancePlan.cpp b/src/meta/processors/admin/BalancePlan.cpp index eabe54b2ccd..5cd459add1c 100644 --- a/src/meta/processors/admin/BalancePlan.cpp +++ b/src/meta/processors/admin/BalancePlan.cpp @@ -123,13 +123,12 @@ void BalancePlan::invoke() { nebula::cpp2::ErrorCode BalancePlan::saveInStore(bool onlyPlan) { CHECK_NOTNULL(kv_); std::vector data; - data.emplace_back(MetaServiceUtils::balancePlanKey(id_), - MetaServiceUtils::balancePlanVal(status_)); + data.emplace_back(MetaKeyUtils::balancePlanKey(id_), MetaKeyUtils::balancePlanVal(status_)); if (!onlyPlan) { for (auto& task : tasks_) { - data.emplace_back(MetaServiceUtils::balanceTaskKey( + data.emplace_back(MetaKeyUtils::balanceTaskKey( task.balanceId_, task.spaceId_, task.partId_, task.src_, task.dst_), - MetaServiceUtils::balanceTaskVal( + MetaKeyUtils::balanceTaskVal( task.status_, task.ret_, task.startTimeMs_, task.endTimeMs_)); } } @@ -152,7 +151,7 @@ nebula::cpp2::ErrorCode BalancePlan::saveInStore(bool onlyPlan) { nebula::cpp2::ErrorCode BalancePlan::recovery(bool resume) { CHECK_NOTNULL(kv_); - const auto& prefix = MetaServiceUtils::balanceTaskPrefix(id_); + const auto& prefix = MetaKeyUtils::balanceTaskPrefix(id_); std::unique_ptr iter; auto ret = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -165,7 +164,7 @@ nebula::cpp2::ErrorCode BalancePlan::recovery(bool resume) { task.kv_ = kv_; task.client_ = client_; { - auto tup = MetaServiceUtils::parseBalanceTaskKey(iter->key()); + auto tup = MetaKeyUtils::parseBalanceTaskKey(iter->key()); task.balanceId_ = std::get<0>(tup); task.spaceId_ = std::get<1>(tup); task.partId_ = std::get<2>(tup); @@ -174,7 +173,7 @@ nebula::cpp2::ErrorCode BalancePlan::recovery(bool resume) { task.taskIdStr_ = task.buildTaskId(); } { - auto tup = MetaServiceUtils::parseBalanceTaskVal(iter->val()); + auto tup = MetaKeyUtils::parseBalanceTaskVal(iter->val()); task.status_ = std::get<0>(tup); task.ret_ = std::get<1>(tup); task.startTimeMs_ = std::get<2>(tup); diff --git a/src/meta/processors/admin/BalanceTask.cpp b/src/meta/processors/admin/BalanceTask.cpp index e0f03a61cb9..2b90bd8ff43 100644 --- a/src/meta/processors/admin/BalanceTask.cpp +++ b/src/meta/processors/admin/BalanceTask.cpp @@ -236,8 +236,8 @@ void BalanceTask::rollback() { bool BalanceTask::saveInStore() { CHECK_NOTNULL(kv_); std::vector data; - data.emplace_back(MetaServiceUtils::balanceTaskKey(balanceId_, spaceId_, partId_, src_, dst_), - MetaServiceUtils::balanceTaskVal(status_, ret_, startTimeMs_, endTimeMs_)); + data.emplace_back(MetaKeyUtils::balanceTaskKey(balanceId_, spaceId_, partId_, src_, dst_), + MetaKeyUtils::balanceTaskVal(status_, ret_, startTimeMs_, endTimeMs_)); folly::Baton baton; bool ret = true; kv_->asyncMultiPut(kDefaultSpaceId, diff --git a/src/meta/processors/admin/Balancer.cpp b/src/meta/processors/admin/Balancer.cpp index 90e5b04d5ee..a9bd46aacbc 100644 --- a/src/meta/processors/admin/Balancer.cpp +++ b/src/meta/processors/admin/Balancer.cpp @@ -12,9 +12,9 @@ #include #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "kvstore/NebulaStore.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" @@ -92,7 +92,7 @@ ErrorOr Balancer::cleanLastInValidPlan() { if (running_) { return nebula::cpp2::ErrorCode::E_BALANCER_RUNNING; } - const auto& prefix = MetaServiceUtils::balancePlanPrefix(); + const auto& prefix = MetaKeyUtils::balancePlanPrefix(); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -101,9 +101,9 @@ ErrorOr Balancer::cleanLastInValidPlan() { } // There should be at most one invalid plan, and it must be the latest one if (iter->valid()) { - auto status = MetaServiceUtils::parseBalanceStatus(iter->val()); + auto status = MetaKeyUtils::parseBalanceStatus(iter->val()); if (status == BalanceStatus::FAILED) { - auto balanceId = MetaServiceUtils::parseBalanceID(iter->key()); + auto balanceId = MetaKeyUtils::parseBalanceID(iter->key()); folly::Baton baton; auto result = nebula::cpp2::ErrorCode::SUCCEEDED; // Only remove the plan will be enough @@ -134,7 +134,7 @@ nebula::cpp2::ErrorCode Balancer::recovery() { // if not leader. return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; } - const auto& prefix = MetaServiceUtils::balancePlanPrefix(); + const auto& prefix = MetaKeyUtils::balancePlanPrefix(); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -145,9 +145,9 @@ nebula::cpp2::ErrorCode Balancer::recovery() { // The balance plan is stored with balance id desc order, there should be at // most one failed or in_progress plan, and it must be the latest one if (iter->valid()) { - auto status = MetaServiceUtils::parseBalanceStatus(iter->val()); + auto status = MetaKeyUtils::parseBalanceStatus(iter->val()); if (status == BalanceStatus::IN_PROGRESS || status == BalanceStatus::FAILED) { - auto balanceId = MetaServiceUtils::parseBalanceID(iter->key()); + auto balanceId = MetaKeyUtils::parseBalanceID(iter->key()); corruptedPlans.emplace_back(balanceId); } } @@ -184,7 +184,7 @@ nebula::cpp2::ErrorCode Balancer::getAllSpaces( std::vector>& spaces) { // Get all spaces folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::spacePrefix(); + const auto& prefix = MetaKeyUtils::spacePrefix(); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -193,8 +193,8 @@ nebula::cpp2::ErrorCode Balancer::getAllSpaces( } while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); - auto properties = MetaServiceUtils::parseSpace(iter->val()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); + auto properties = MetaKeyUtils::parseSpace(iter->val()); bool zoned = properties.group_name_ref().has_value(); spaces.emplace_back(spaceId, *properties.replica_factor_ref(), zoned); iter->next(); @@ -563,7 +563,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space HostParts& hostParts, int32_t& totalParts) { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::partPrefix(spaceId); + const auto& prefix = MetaKeyUtils::partPrefix(spaceId); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -576,7 +576,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& ph : partHosts) { hostParts[ph].emplace_back(partId); } @@ -585,7 +585,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space } LOG(INFO) << "Host size: " << hostParts.size(); - auto key = MetaServiceUtils::spaceKey(spaceId); + auto key = MetaKeyUtils::spaceKey(spaceId); std::string value; retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, key, &value); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -594,7 +594,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space return retCode; } - auto properties = MetaServiceUtils::parseSpace(value); + auto properties = MetaKeyUtils::parseSpace(value); if (totalParts != properties.get_partition_num()) { LOG(ERROR) << "Partition number not equals"; LOG(ERROR) << totalParts << " : " << properties.get_partition_num(); @@ -605,7 +605,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space LOG(INFO) << "Replica " << replica; if (dependentOnGroup && properties.group_name_ref().has_value()) { auto groupName = *properties.group_name_ref(); - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); std::string groupValue; retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -614,7 +614,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space return retCode; } - int32_t zoneSize = MetaServiceUtils::parseZoneNames(std::move(groupValue)).size(); + int32_t zoneSize = MetaKeyUtils::parseZoneNames(std::move(groupValue)).size(); LOG(INFO) << "Zone Size " << zoneSize; innerBalance_ = (replica == zoneSize); @@ -653,7 +653,7 @@ ErrorOr Balancer::getHostParts(GraphSpaceID space nebula::cpp2::ErrorCode Balancer::assembleZoneParts(const std::string& groupName, HostParts& hostParts) { LOG(INFO) << "Balancer assembleZoneParts"; - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); std::string groupValue; auto retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -664,10 +664,10 @@ nebula::cpp2::ErrorCode Balancer::assembleZoneParts(const std::string& groupName // zoneHosts use to record this host belong to zone's hosts std::unordered_map, std::vector> zoneHosts; - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(groupValue)); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); for (auto zoneName : zoneNames) { LOG(INFO) << "Zone Name: " << zoneName; - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); std::string zoneValue; retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -676,7 +676,7 @@ nebula::cpp2::ErrorCode Balancer::assembleZoneParts(const std::string& groupName return retCode; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(zoneValue)); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(zoneValue)); for (const auto& host : hosts) { LOG(INFO) << "Host for zone " << host; auto pair = std::pair(std::move(host), zoneName); @@ -907,7 +907,7 @@ ErrorOr Balancer::buildLeaderBalancePlan( size_t leaderParts = 0; // store peers of all paritions in peerMap folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::partPrefix(spaceId); + const auto& prefix = MetaKeyUtils::partPrefix(spaceId); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -919,7 +919,7 @@ ErrorOr Balancer::buildLeaderBalancePlan( auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - auto peers = MetaServiceUtils::parsePartVal(iter->val()); + auto peers = MetaKeyUtils::parsePartVal(iter->val()); peersMap[partId] = std::move(peers); ++leaderParts; iter->next(); @@ -1149,7 +1149,7 @@ void Balancer::simplifyLeaderBalnacePlan(GraphSpaceID spaceId, LeaderBalancePlan nebula::cpp2::ErrorCode Balancer::collectZoneParts(const std::string& groupName, HostParts& hostParts) { - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); std::string groupValue; auto retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -1160,9 +1160,9 @@ nebula::cpp2::ErrorCode Balancer::collectZoneParts(const std::string& groupName, // zoneHosts use to record this host belong to zone's hosts std::unordered_map, std::vector> zoneHosts; - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(groupValue)); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); for (auto zoneName : zoneNames) { - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); std::string zoneValue; retCode = kv_->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -1171,7 +1171,7 @@ nebula::cpp2::ErrorCode Balancer::collectZoneParts(const std::string& groupName, return retCode; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(zoneValue)); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(zoneValue)); for (const auto& host : hosts) { auto pair = std::pair(std::move(host), zoneName); auto& hs = zoneHosts[std::move(pair)]; diff --git a/src/meta/processors/admin/CreateBackupProcessor.cpp b/src/meta/processors/admin/CreateBackupProcessor.cpp index 7ae8b59a4f5..2042a398df0 100644 --- a/src/meta/processors/admin/CreateBackupProcessor.cpp +++ b/src/meta/processors/admin/CreateBackupProcessor.cpp @@ -26,7 +26,7 @@ CreateBackupProcessor::spaceNameToId(const std::vector* backupSpace std::transform( backupSpaces->begin(), backupSpaces->end(), std::back_inserter(keys), [](auto& name) { - return MetaServiceUtils::indexSpaceKey(name); + return MetaKeyUtils::indexSpaceKey(name); }); auto result = doMultiGet(std::move(keys)); @@ -48,7 +48,7 @@ CreateBackupProcessor::spaceNameToId(const std::vector* backupSpace }); } else { - const auto& prefix = MetaServiceUtils::spacePrefix(); + const auto& prefix = MetaKeyUtils::spacePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -58,8 +58,8 @@ CreateBackupProcessor::spaceNameToId(const std::vector* backupSpace auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); - auto spaceName = MetaServiceUtils::spaceName(iter->val()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); + auto spaceName = MetaKeyUtils::spaceName(iter->val()); VLOG(3) << "List spaces " << spaceId << ", name " << spaceName; spaces.emplace(spaceId); iter->next(); @@ -128,11 +128,11 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) { // The entire process follows mostly snapshot logic. std::vector data; - auto backupName = folly::format("BACKUP_{}", MetaServiceUtils::genTimestampStr()).str(); + auto backupName = folly::format("BACKUP_{}", MetaKeyUtils::genTimestampStr()).str(); - data.emplace_back(MetaServiceUtils::snapshotKey(backupName), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, - NetworkUtils::toHostsStr(hosts))); + data.emplace_back( + MetaKeyUtils::snapshotKey(backupName), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, NetworkUtils::toHostsStr(hosts))); Snapshot::instance(kvstore_, client_)->setSpaces(spaces); // step 1 : Blocking all writes action for storage engines. @@ -181,8 +181,8 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) { // step 6 : update snapshot status from INVALID to VALID. data.emplace_back( - MetaServiceUtils::snapshotKey(backupName), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::VALID, NetworkUtils::toHostsStr(hosts))); + MetaKeyUtils::snapshotKey(backupName), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::VALID, NetworkUtils::toHostsStr(hosts))); auto putRet = doSyncPut(std::move(data)); if (putRet != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -202,14 +202,14 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) { for (auto id : spaces) { LOG(INFO) << "backup space " << id; cpp2::SpaceBackupInfo spaceInfo; - auto spaceKey = MetaServiceUtils::spaceKey(id); + auto spaceKey = MetaKeyUtils::spaceKey(id); auto spaceRet = doGet(spaceKey); if (!nebula::ok(spaceRet)) { handleErrorCode(nebula::error(spaceRet)); onFinished(); return; } - auto properties = MetaServiceUtils::parseSpace(nebula::value(spaceRet)); + auto properties = MetaKeyUtils::parseSpace(nebula::value(spaceRet)); // todo we should save partition info. auto it = snapshotInfo.find(id); DCHECK(it != snapshotInfo.end()); diff --git a/src/meta/processors/admin/CreateSnapshotProcessor.cpp b/src/meta/processors/admin/CreateSnapshotProcessor.cpp index a9dc6dc6e0c..c8f4c62bf0b 100644 --- a/src/meta/processors/admin/CreateSnapshotProcessor.cpp +++ b/src/meta/processors/admin/CreateSnapshotProcessor.cpp @@ -31,7 +31,7 @@ void CreateSnapshotProcessor::process(const cpp2::CreateSnapshotReq&) { return; } - auto snapshot = folly::format("SNAPSHOT_{}", MetaServiceUtils::genTimestampStr()).str(); + auto snapshot = folly::format("SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); folly::SharedMutex::WriteHolder wHolder(LockUtils::snapshotLock()); auto activeHostsRet = ActiveHostsMan::getActiveHosts(kvstore_); @@ -53,9 +53,9 @@ void CreateSnapshotProcessor::process(const cpp2::CreateSnapshotReq&) { // CREATING. // The purpose of this is to handle the failure of the checkpoint. std::vector data; - data.emplace_back(MetaServiceUtils::snapshotKey(snapshot), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, - NetworkUtils::toHostsStr(hosts))); + data.emplace_back( + MetaKeyUtils::snapshotKey(snapshot), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, NetworkUtils::toHostsStr(hosts))); auto putRet = doSyncPut(std::move(data)); if (putRet != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -105,8 +105,8 @@ void CreateSnapshotProcessor::process(const cpp2::CreateSnapshotReq&) { // step 6 : update snapshot status from INVALID to VALID. data.emplace_back( - MetaServiceUtils::snapshotKey(snapshot), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::VALID, NetworkUtils::toHostsStr(hosts))); + MetaKeyUtils::snapshotKey(snapshot), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::VALID, NetworkUtils::toHostsStr(hosts))); putRet = doSyncPut(std::move(data)); if (putRet != nebula::cpp2::ErrorCode::SUCCEEDED) { diff --git a/src/meta/processors/admin/DropSnapshotProcessor.cpp b/src/meta/processors/admin/DropSnapshotProcessor.cpp index 27a6a44294e..ade6c2b327d 100644 --- a/src/meta/processors/admin/DropSnapshotProcessor.cpp +++ b/src/meta/processors/admin/DropSnapshotProcessor.cpp @@ -17,7 +17,7 @@ void DropSnapshotProcessor::process(const cpp2::DropSnapshotReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::snapshotLock()); // Check snapshot is exists - auto key = MetaServiceUtils::snapshotKey(snapshot); + auto key = MetaKeyUtils::snapshotKey(snapshot); auto ret = doGet(std::move(key)); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -29,7 +29,7 @@ void DropSnapshotProcessor::process(const cpp2::DropSnapshotReq& req) { } auto val = nebula::value(ret); - auto hosts = MetaServiceUtils::parseSnapshotHosts(val); + auto hosts = MetaKeyUtils::parseSnapshotHosts(val); auto peersRet = NetworkUtils::toHosts(hosts); if (!peersRet.ok()) { LOG(ERROR) << "Get checkpoint hosts error"; @@ -45,8 +45,8 @@ void DropSnapshotProcessor::process(const cpp2::DropSnapshotReq& req) { LOG(ERROR) << "Drop snapshot error on storage engine"; // Need update the snapshot status to invalid, maybe some storage engine // drop done. - data.emplace_back(MetaServiceUtils::snapshotKey(snapshot), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, hosts)); + data.emplace_back(MetaKeyUtils::snapshotKey(snapshot), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, hosts)); auto putRet = doSyncPut(std::move(data)); if (putRet != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Update snapshot status error. " @@ -64,8 +64,8 @@ void DropSnapshotProcessor::process(const cpp2::DropSnapshotReq& req) { LOG(ERROR) << "Drop snapshot error on meta engine"; // Need update the snapshot status to invalid, maybe storage engines drop // done. - data.emplace_back(MetaServiceUtils::snapshotKey(snapshot), - MetaServiceUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, hosts)); + data.emplace_back(MetaKeyUtils::snapshotKey(snapshot), + MetaKeyUtils::snapshotVal(cpp2::SnapshotStatus::INVALID, hosts)); auto putRet = doSyncPut(std::move(data)); if (putRet != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Update snapshot status error. " @@ -77,7 +77,7 @@ void DropSnapshotProcessor::process(const cpp2::DropSnapshotReq& req) { return; } // Delete metadata of checkpoint - doRemove(MetaServiceUtils::snapshotKey(snapshot)); + doRemove(MetaKeyUtils::snapshotKey(snapshot)); LOG(INFO) << "Drop snapshot " << snapshot << " successfully"; } diff --git a/src/meta/processors/admin/ListClusterInfoProcessor.cpp b/src/meta/processors/admin/ListClusterInfoProcessor.cpp index d2188540bc6..c0b31f4f2ca 100644 --- a/src/meta/processors/admin/ListClusterInfoProcessor.cpp +++ b/src/meta/processors/admin/ListClusterInfoProcessor.cpp @@ -23,7 +23,7 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { return; } - const auto& prefix = MetaServiceUtils::hostPrefix(); + const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; auto ret = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter, true); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -35,7 +35,7 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { std::vector storages; while (iter->valid()) { - auto host = MetaServiceUtils::parseHostKey(iter->key()); + auto host = MetaKeyUtils::parseHostKey(iter->key()); HostInfo info = HostInfo::decode(iter->val()); if (info.role_ != cpp2::HostRole::STORAGE) { @@ -63,7 +63,7 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { return; } auto& map = mpm->partsMap(); - auto hosts = map[nebula::meta::kDefaultSpaceId][nebula::meta::kDefaultPartId].hosts_; + auto hosts = map[kDefaultSpaceId][kDefaultPartId].hosts_; LOG(INFO) << "meta servers count: " << hosts.size(); resp_.set_meta_servers(std::move(hosts)); diff --git a/src/meta/processors/admin/ListSnapshotsProcessor.cpp b/src/meta/processors/admin/ListSnapshotsProcessor.cpp index aa0c9a8ed3b..f35b90df267 100644 --- a/src/meta/processors/admin/ListSnapshotsProcessor.cpp +++ b/src/meta/processors/admin/ListSnapshotsProcessor.cpp @@ -12,7 +12,7 @@ namespace nebula { namespace meta { void ListSnapshotsProcessor::process(const cpp2::ListSnapshotsReq&) { - const auto& prefix = MetaServiceUtils::snapshotPrefix(); + const auto& prefix = MetaKeyUtils::snapshotPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -26,9 +26,9 @@ void ListSnapshotsProcessor::process(const cpp2::ListSnapshotsReq&) { std::vector snapshots; while (iter->valid()) { auto val = iter->val(); - auto name = MetaServiceUtils::parseSnapshotName(iter->key()); - auto status = MetaServiceUtils::parseSnapshotStatus(val); - auto hosts = MetaServiceUtils::parseSnapshotHosts(val); + auto name = MetaKeyUtils::parseSnapshotName(iter->key()); + auto status = MetaKeyUtils::parseSnapshotStatus(val); + auto hosts = MetaKeyUtils::parseSnapshotHosts(val); cpp2::Snapshot snapshot; snapshot.set_name(std::move(name)); snapshot.set_status(std::move(status)); diff --git a/src/meta/processors/admin/RestoreProcessor.cpp b/src/meta/processors/admin/RestoreProcessor.cpp index d1a542bde3b..5231c1d760f 100644 --- a/src/meta/processors/admin/RestoreProcessor.cpp +++ b/src/meta/processors/admin/RestoreProcessor.cpp @@ -17,7 +17,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInPartition(const HostAddr& bool direct) { folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock()); auto retCode = nebula::cpp2::ErrorCode::SUCCEEDED; - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto iterRet = doPrefix(spacePrefix); if (!nebula::ok(iterRet)) { retCode = nebula::error(iterRet); @@ -28,7 +28,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInPartition(const HostAddr& std::vector allSpaceId; while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); allSpaceId.emplace_back(spaceId); iter->next(); } @@ -37,7 +37,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInPartition(const HostAddr& std::vector data; for (const auto& spaceId : allSpaceId) { - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto iterPartRet = doPrefix(partPrefix); if (!nebula::ok(iterPartRet)) { retCode = nebula::error(iterPartRet); @@ -48,7 +48,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInPartition(const HostAddr& while (iter->valid()) { bool needUpdate = false; - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& host : partHosts) { if (host == ipv4From) { needUpdate = true; @@ -56,7 +56,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInPartition(const HostAddr& } } if (needUpdate) { - data.emplace_back(iter->key(), MetaServiceUtils::partVal(partHosts)); + data.emplace_back(iter->key(), MetaKeyUtils::partVal(partHosts)); } iter->next(); } @@ -87,7 +87,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInZone(const HostAddr& ipv4 bool direct) { folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock()); auto retCode = nebula::cpp2::ErrorCode::SUCCEEDED; - const auto& zonePrefix = MetaServiceUtils::zonePrefix(); + const auto& zonePrefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(zonePrefix); if (!nebula::ok(iterRet)) { retCode = nebula::error(iterRet); @@ -99,8 +99,8 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInZone(const HostAddr& ipv4 while (iter->valid()) { bool needUpdate = false; - auto zoneName = MetaServiceUtils::parseZoneName(iter->key()); - auto hosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); std::vector DesHosts; for (auto& host : hosts) { if (host == ipv4From) { @@ -109,7 +109,7 @@ nebula::cpp2::ErrorCode RestoreProcessor::replaceHostInZone(const HostAddr& ipv4 } } if (needUpdate) { - data.emplace_back(iter->key(), MetaServiceUtils::zoneVal(hosts)); + data.emplace_back(iter->key(), MetaKeyUtils::zoneVal(hosts)); } iter->next(); } diff --git a/src/meta/processors/admin/SnapShot.cpp b/src/meta/processors/admin/SnapShot.cpp index 4b468f47106..608ca1b0766 100644 --- a/src/meta/processors/admin/SnapShot.cpp +++ b/src/meta/processors/admin/SnapShot.cpp @@ -9,8 +9,8 @@ #include #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" @@ -108,7 +108,7 @@ nebula::cpp2::ErrorCode Snapshot::blockingWrites(storage::cpp2::EngineSignType s ErrorOr>> Snapshot::getSpacesHosts() { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::partPrefix(); + const auto& prefix = MetaKeyUtils::partPrefix(); std::unique_ptr iter; auto retCode = kv_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -120,8 +120,8 @@ Snapshot::getSpacesHosts() { std::map> hostsByspaces; for (; iter->valid(); iter->next()) { - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); - auto space = MetaServiceUtils::parsePartKeySpaceId(iter->key()); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); + auto space = MetaKeyUtils::parsePartKeySpaceId(iter->key()); if (!spaces_.empty()) { auto it = spaces_.find(space); if (it == spaces_.end()) { diff --git a/src/meta/processors/config/GetConfigProcessor.cpp b/src/meta/processors/config/GetConfigProcessor.cpp index 4b9565b2173..85a14b89f6f 100644 --- a/src/meta/processors/config/GetConfigProcessor.cpp +++ b/src/meta/processors/config/GetConfigProcessor.cpp @@ -44,7 +44,7 @@ void GetConfigProcessor::process(const cpp2::GetConfigReq& req) { nebula::cpp2::ErrorCode GetConfigProcessor::getOneConfig(const cpp2::ConfigModule& module, const std::string& name, std::vector& items) { - std::string configKey = MetaServiceUtils::configKey(module, name); + std::string configKey = MetaKeyUtils::configKey(module, name); auto ret = doGet(configKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -56,7 +56,7 @@ nebula::cpp2::ErrorCode GetConfigProcessor::getOneConfig(const cpp2::ConfigModul return retCode; } - cpp2::ConfigItem item = MetaServiceUtils::parseConfigValue(nebula::value(ret)); + cpp2::ConfigItem item = MetaKeyUtils::parseConfigValue(nebula::value(ret)); item.set_module(module); item.set_name(name); items.emplace_back(std::move(item)); diff --git a/src/meta/processors/config/ListConfigsProcessor.cpp b/src/meta/processors/config/ListConfigsProcessor.cpp index 28914508e57..9e88c3abfa0 100644 --- a/src/meta/processors/config/ListConfigsProcessor.cpp +++ b/src/meta/processors/config/ListConfigsProcessor.cpp @@ -12,7 +12,7 @@ namespace meta { void ListConfigsProcessor::process(const cpp2::ListConfigsReq& req) { folly::SharedMutex::ReadHolder rHolder(LockUtils::configLock()); - const auto& prefix = MetaServiceUtils::configKeyPrefix(req.get_module()); + const auto& prefix = MetaKeyUtils::configKeyPrefix(req.get_module()); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -27,8 +27,8 @@ void ListConfigsProcessor::process(const cpp2::ListConfigsReq& req) { while (iter->valid()) { auto key = iter->key(); auto value = iter->val(); - auto item = MetaServiceUtils::parseConfigValue(value); - auto configName = MetaServiceUtils::parseConfigKey(key); + auto item = MetaKeyUtils::parseConfigValue(value); + auto configName = MetaKeyUtils::parseConfigKey(key); item.set_module(configName.first); item.set_name(configName.second); items.emplace_back(std::move(item)); diff --git a/src/meta/processors/config/RegConfigProcessor.cpp b/src/meta/processors/config/RegConfigProcessor.cpp index a3d431da618..7440b2fd793 100644 --- a/src/meta/processors/config/RegConfigProcessor.cpp +++ b/src/meta/processors/config/RegConfigProcessor.cpp @@ -21,7 +21,7 @@ void RegConfigProcessor::process(const cpp2::RegConfigReq& req) { VLOG(1) << "Config name: " << name << ", mode: " << apache::thrift::util::enumNameSafe(mode) << ", module: " << apache::thrift::util::enumNameSafe(module) << ", value: " << value; - std::string configKey = MetaServiceUtils::configKey(module, name); + std::string configKey = MetaKeyUtils::configKey(module, name); // ignore config which has been registered before auto configRet = doGet(configKey); if (nebula::ok(configRet)) { @@ -36,7 +36,7 @@ void RegConfigProcessor::process(const cpp2::RegConfigReq& req) { } } - std::string configValue = MetaServiceUtils::configValue(mode, value); + std::string configValue = MetaKeyUtils::configValue(mode, value); data.emplace_back(std::move(configKey), std::move(configValue)); } diff --git a/src/meta/processors/config/SetConfigProcessor.cpp b/src/meta/processors/config/SetConfigProcessor.cpp index ace537391be..1a94e579db8 100644 --- a/src/meta/processors/config/SetConfigProcessor.cpp +++ b/src/meta/processors/config/SetConfigProcessor.cpp @@ -53,7 +53,7 @@ nebula::cpp2::ErrorCode SetConfigProcessor::setConfig(const cpp2::ConfigModule& const std::string& name, const Value& value, std::vector& data) { - std::string configKey = MetaServiceUtils::configKey(module, name); + std::string configKey = MetaKeyUtils::configKey(module, name); auto ret = doGet(std::move(configKey)); if (!nebula::ok(ret)) { auto retCode = nebula::error((ret)); @@ -65,12 +65,12 @@ nebula::cpp2::ErrorCode SetConfigProcessor::setConfig(const cpp2::ConfigModule& return retCode; } - cpp2::ConfigItem item = MetaServiceUtils::parseConfigValue(nebula::value(ret)); + cpp2::ConfigItem item = MetaKeyUtils::parseConfigValue(nebula::value(ret)); cpp2::ConfigMode curMode = item.get_mode(); if (curMode == cpp2::ConfigMode::IMMUTABLE) { return nebula::cpp2::ErrorCode::E_CONFIG_IMMUTABLE; } - std::string configValue = MetaServiceUtils::configValue(curMode, value); + std::string configValue = MetaKeyUtils::configValue(curMode, value); data.emplace_back(std::move(configKey), std::move(configValue)); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp index 9fa97cda12d..48a2956e3e1 100644 --- a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp @@ -72,7 +72,7 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) { } auto edgeType = nebula::value(edgeTypeRet); - const auto& prefix = MetaServiceUtils::indexPrefix(space); + const auto& prefix = MetaKeyUtils::indexPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -86,7 +86,7 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) { while (checkIter->valid()) { auto val = checkIter->val(); - auto item = MetaServiceUtils::parseIndex(val); + auto item = MetaKeyUtils::parseIndex(val); if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::edge_type || fields.size() > item.get_fields().size() || edgeType != item.get_schema_id().get_edge_type()) { @@ -188,9 +188,9 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) { item.set_comment(*req.comment_ref()); } - data.emplace_back(MetaServiceUtils::indexIndexKey(space, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(space, indexName), std::string(reinterpret_cast(&edgeIndex), sizeof(IndexID))); - data.emplace_back(MetaServiceUtils::indexKey(space, edgeIndex), MetaServiceUtils::indexVal(item)); + data.emplace_back(MetaKeyUtils::indexKey(space, edgeIndex), MetaKeyUtils::indexVal(item)); LOG(INFO) << "Create Edge Index " << indexName << ", edgeIndex " << edgeIndex; resp_.set_id(to(edgeIndex, EntryType::INDEX)); doSyncPutAndUpdate(std::move(data)); diff --git a/src/meta/processors/index/CreateTagIndexProcessor.cpp b/src/meta/processors/index/CreateTagIndexProcessor.cpp index 2f6a1c0f624..c81e993f50c 100644 --- a/src/meta/processors/index/CreateTagIndexProcessor.cpp +++ b/src/meta/processors/index/CreateTagIndexProcessor.cpp @@ -72,7 +72,7 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) { } auto tagID = nebula::value(tagIDRet); - const auto& prefix = MetaServiceUtils::indexPrefix(space); + const auto& prefix = MetaKeyUtils::indexPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -86,7 +86,7 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) { while (checkIter->valid()) { auto val = checkIter->val(); - auto item = MetaServiceUtils::parseIndex(val); + auto item = MetaKeyUtils::parseIndex(val); if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::tag_id || fields.size() > item.get_fields().size() || tagID != item.get_schema_id().get_tag_id()) { checkIter->next(); @@ -186,9 +186,9 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) { item.set_comment(*req.comment_ref()); } - data.emplace_back(MetaServiceUtils::indexIndexKey(space, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(space, indexName), std::string(reinterpret_cast(&tagIndex), sizeof(IndexID))); - data.emplace_back(MetaServiceUtils::indexKey(space, tagIndex), MetaServiceUtils::indexVal(item)); + data.emplace_back(MetaKeyUtils::indexKey(space, tagIndex), MetaKeyUtils::indexVal(item)); LOG(INFO) << "Create Tag Index " << indexName << ", tagIndex " << tagIndex; resp_.set_id(to(tagIndex, EntryType::INDEX)); doSyncPutAndUpdate(std::move(data)); diff --git a/src/meta/processors/index/DropEdgeIndexProcessor.cpp b/src/meta/processors/index/DropEdgeIndexProcessor.cpp index aefc8dc6bd5..788f0ceab4c 100644 --- a/src/meta/processors/index/DropEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/DropEdgeIndexProcessor.cpp @@ -37,8 +37,8 @@ void DropEdgeIndexProcessor::process(const cpp2::DropEdgeIndexReq& req) { auto edgeIndexID = nebula::value(edgeIndexIDRet); std::vector keys; - keys.emplace_back(MetaServiceUtils::indexIndexKey(spaceID, indexName)); - keys.emplace_back(MetaServiceUtils::indexKey(spaceID, edgeIndexID)); + keys.emplace_back(MetaKeyUtils::indexIndexKey(spaceID, indexName)); + keys.emplace_back(MetaKeyUtils::indexKey(spaceID, edgeIndexID)); LOG(INFO) << "Drop Edge Index " << indexName; resp_.set_id(to(edgeIndexID, EntryType::INDEX)); diff --git a/src/meta/processors/index/DropTagIndexProcessor.cpp b/src/meta/processors/index/DropTagIndexProcessor.cpp index d894c26bee9..7a0d8db983a 100644 --- a/src/meta/processors/index/DropTagIndexProcessor.cpp +++ b/src/meta/processors/index/DropTagIndexProcessor.cpp @@ -36,8 +36,8 @@ void DropTagIndexProcessor::process(const cpp2::DropTagIndexReq& req) { auto tagIndexID = nebula::value(tagIndexIDRet); std::vector keys; - keys.emplace_back(MetaServiceUtils::indexIndexKey(spaceID, indexName)); - keys.emplace_back(MetaServiceUtils::indexKey(spaceID, tagIndexID)); + keys.emplace_back(MetaKeyUtils::indexIndexKey(spaceID, indexName)); + keys.emplace_back(MetaKeyUtils::indexKey(spaceID, tagIndexID)); LOG(INFO) << "Drop Tag Index " << indexName; resp_.set_id(to(tagIndexID, EntryType::INDEX)); diff --git a/src/meta/processors/index/FTIndexProcessor.cpp b/src/meta/processors/index/FTIndexProcessor.cpp index df8777e944a..ef04c2e0378 100644 --- a/src/meta/processors/index/FTIndexProcessor.cpp +++ b/src/meta/processors/index/FTIndexProcessor.cpp @@ -18,9 +18,9 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { CHECK_SPACE_ID_AND_RETURN(index.get_space_id()); auto isEdge = index.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type; folly::SharedMutex::ReadHolder rHolder(isEdge ? LockUtils::edgeLock() : LockUtils::tagLock()); - auto schemaPrefix = isEdge ? MetaServiceUtils::schemaEdgePrefix( + auto schemaPrefix = isEdge ? MetaKeyUtils::schemaEdgePrefix( index.get_space_id(), index.get_depend_schema().get_edge_type()) - : MetaServiceUtils::schemaTagPrefix( + : MetaKeyUtils::schemaTagPrefix( index.get_space_id(), index.get_depend_schema().get_tag_id()); // Check tag or edge exist. @@ -43,7 +43,7 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { } // verify the columns. - auto schema = MetaServiceUtils::parseSchema(iter->val()); + auto schema = MetaKeyUtils::parseSchema(iter->val()); auto columns = schema.get_columns(); for (const auto& col : index.get_fields()) { auto targetCol = std::find_if( @@ -80,7 +80,7 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { } // Check fulltext index exist. - auto ftPrefix = MetaServiceUtils::fulltextIndexPrefix(); + auto ftPrefix = MetaKeyUtils::fulltextIndexPrefix(); auto ret = doPrefix(ftPrefix); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -93,8 +93,8 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { // If already have a full-text index that depend on this tag or edge, reject // to create it. even though it's a different column. while (it->valid()) { - auto indexName = MetaServiceUtils::parsefulltextIndexName(it->key()); - auto indexItem = MetaServiceUtils::parsefulltextIndex(it->val()); + auto indexName = MetaKeyUtils::parsefulltextIndexName(it->key()); + auto indexItem = MetaKeyUtils::parsefulltextIndex(it->val()); if (indexName == name) { LOG(ERROR) << "Fulltext index exist : " << indexName; handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); @@ -110,15 +110,14 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { it->next(); } std::vector data; - data.emplace_back(MetaServiceUtils::fulltextIndexKey(name), - MetaServiceUtils::fulltextIndexVal(index)); + data.emplace_back(MetaKeyUtils::fulltextIndexKey(name), MetaKeyUtils::fulltextIndexVal(index)); doSyncPutAndUpdate(std::move(data)); } void DropFTIndexProcessor::process(const cpp2::DropFTIndexReq& req) { CHECK_SPACE_ID_AND_RETURN(req.get_space_id()); folly::SharedMutex::WriteHolder wHolder(LockUtils::fulltextIndexLock()); - auto indexKey = MetaServiceUtils::fulltextIndexKey(req.get_fulltext_index_name()); + auto indexKey = MetaKeyUtils::fulltextIndexKey(req.get_fulltext_index_name()); auto ret = doGet(indexKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -139,7 +138,7 @@ void DropFTIndexProcessor::process(const cpp2::DropFTIndexReq& req) { void ListFTIndexesProcessor::process(const cpp2::ListFTIndexesReq&) { folly::SharedMutex::ReadHolder rHolder(LockUtils::fulltextIndexLock()); - const auto& prefix = MetaServiceUtils::fulltextIndexPrefix(); + const auto& prefix = MetaKeyUtils::fulltextIndexPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -153,8 +152,8 @@ void ListFTIndexesProcessor::process(const cpp2::ListFTIndexesReq&) { auto iter = nebula::value(iterRet).get(); std::unordered_map indexes; while (iter->valid()) { - auto name = MetaServiceUtils::parsefulltextIndexName(iter->key()); - auto index = MetaServiceUtils::parsefulltextIndex(iter->val()); + auto name = MetaKeyUtils::parsefulltextIndexName(iter->key()); + auto index = MetaKeyUtils::parsefulltextIndex(iter->val()); indexes.emplace(std::move(name), std::move(index)); iter->next(); } diff --git a/src/meta/processors/index/FTServiceProcessor.cpp b/src/meta/processors/index/FTServiceProcessor.cpp index d511f2e84ab..f3116d97aee 100644 --- a/src/meta/processors/index/FTServiceProcessor.cpp +++ b/src/meta/processors/index/FTServiceProcessor.cpp @@ -11,7 +11,7 @@ namespace meta { void SignInFTServiceProcessor::process(const cpp2::SignInFTServiceReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::fulltextServicesLock()); - auto serviceKey = MetaServiceUtils::fulltextServiceKey(); + auto serviceKey = MetaKeyUtils::fulltextServiceKey(); auto ret = doGet(serviceKey); if (nebula::ok(ret)) { LOG(ERROR) << "Fulltext already exists."; @@ -31,13 +31,13 @@ void SignInFTServiceProcessor::process(const cpp2::SignInFTServiceReq& req) { std::vector data; data.emplace_back(std::move(serviceKey), - MetaServiceUtils::fulltextServiceVal(req.get_type(), req.get_clients())); + MetaKeyUtils::fulltextServiceVal(req.get_type(), req.get_clients())); doSyncPutAndUpdate(std::move(data)); } void SignOutFTServiceProcessor::process(const cpp2::SignOutFTServiceReq&) { folly::SharedMutex::WriteHolder wHolder(LockUtils::fulltextServicesLock()); - auto serviceKey = MetaServiceUtils::fulltextServiceKey(); + auto serviceKey = MetaKeyUtils::fulltextServiceKey(); auto ret = doGet(serviceKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -57,7 +57,7 @@ void SignOutFTServiceProcessor::process(const cpp2::SignOutFTServiceReq&) { void ListFTClientsProcessor::process(const cpp2::ListFTClientsReq&) { folly::SharedMutex::ReadHolder rHolder(LockUtils::fulltextServicesLock()); - const auto& prefix = MetaServiceUtils::fulltextServiceKey(); + const auto& prefix = MetaKeyUtils::fulltextServiceKey(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -70,7 +70,7 @@ void ListFTClientsProcessor::process(const cpp2::ListFTClientsReq&) { auto iter = nebula::value(iterRet).get(); std::vector clients; if (iter->valid()) { - clients = MetaServiceUtils::parseFTClients(iter->val()); + clients = MetaKeyUtils::parseFTClients(iter->val()); } resp_.set_clients(std::move(clients)); handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); diff --git a/src/meta/processors/index/GetEdgeIndexProcessor.cpp b/src/meta/processors/index/GetEdgeIndexProcessor.cpp index 190cd1f044c..930a60f2758 100644 --- a/src/meta/processors/index/GetEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/GetEdgeIndexProcessor.cpp @@ -26,7 +26,7 @@ void GetEdgeIndexProcessor::process(const cpp2::GetEdgeIndexReq& req) { auto indexId = nebula::value(edgeIndexIDRet); LOG(INFO) << "Get Edge Index SpaceID: " << spaceID << " Index Name: " << indexName; - const auto& indexKey = MetaServiceUtils::indexKey(spaceID, indexId); + const auto& indexKey = MetaKeyUtils::indexKey(spaceID, indexId); auto indexItemRet = doGet(indexKey); if (!nebula::ok(indexItemRet)) { auto retCode = nebula::error(indexItemRet); @@ -40,7 +40,7 @@ void GetEdgeIndexProcessor::process(const cpp2::GetEdgeIndexReq& req) { return; } - auto item = MetaServiceUtils::parseIndex(nebula::value(indexItemRet)); + auto item = MetaKeyUtils::parseIndex(nebula::value(indexItemRet)); if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::edge_type) { LOG(ERROR) << "Get Edge Index Failed: Index Name " << indexName << " is not EdgeIndex"; resp_.set_code(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND); diff --git a/src/meta/processors/index/GetTagIndexProcessor.cpp b/src/meta/processors/index/GetTagIndexProcessor.cpp index 50e03c8f9d4..58ce08b075c 100644 --- a/src/meta/processors/index/GetTagIndexProcessor.cpp +++ b/src/meta/processors/index/GetTagIndexProcessor.cpp @@ -27,7 +27,7 @@ void GetTagIndexProcessor::process(const cpp2::GetTagIndexReq& req) { auto indexId = nebula::value(tagIndexIDRet); LOG(INFO) << "Get Tag Index SpaceID: " << spaceID << " Index Name: " << indexName; - const auto& indexKey = MetaServiceUtils::indexKey(spaceID, indexId); + const auto& indexKey = MetaKeyUtils::indexKey(spaceID, indexId); auto indexItemRet = doGet(indexKey); if (!nebula::ok(indexItemRet)) { auto retCode = nebula::error(indexItemRet); @@ -41,7 +41,7 @@ void GetTagIndexProcessor::process(const cpp2::GetTagIndexReq& req) { return; } - auto item = MetaServiceUtils::parseIndex(nebula::value(indexItemRet)); + auto item = MetaKeyUtils::parseIndex(nebula::value(indexItemRet)); if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::tag_id) { LOG(ERROR) << "Get Tag Index Failed: Index Name " << indexName << " is not TagIndex"; resp_.set_code(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND); diff --git a/src/meta/processors/index/ListEdgeIndexesProcessor.cpp b/src/meta/processors/index/ListEdgeIndexesProcessor.cpp index d40efa3e3f6..479fbc54cf0 100644 --- a/src/meta/processors/index/ListEdgeIndexesProcessor.cpp +++ b/src/meta/processors/index/ListEdgeIndexesProcessor.cpp @@ -14,7 +14,7 @@ void ListEdgeIndexesProcessor::process(const cpp2::ListEdgeIndexesReq& req) { CHECK_SPACE_ID_AND_RETURN(space); folly::SharedMutex::ReadHolder rHolder(LockUtils::edgeIndexLock()); - const auto& prefix = MetaServiceUtils::indexPrefix(space); + const auto& prefix = MetaKeyUtils::indexPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -29,7 +29,7 @@ void ListEdgeIndexesProcessor::process(const cpp2::ListEdgeIndexesReq& req) { std::vector items; while (iter->valid()) { auto val = iter->val(); - auto item = MetaServiceUtils::parseIndex(val); + auto item = MetaKeyUtils::parseIndex(val); if (item.get_schema_id().getType() == nebula::cpp2::SchemaID::Type::edge_type) { items.emplace_back(std::move(item)); } diff --git a/src/meta/processors/index/ListTagIndexesProcessor.cpp b/src/meta/processors/index/ListTagIndexesProcessor.cpp index 4036a7df7ec..63cd167d3de 100644 --- a/src/meta/processors/index/ListTagIndexesProcessor.cpp +++ b/src/meta/processors/index/ListTagIndexesProcessor.cpp @@ -14,7 +14,7 @@ void ListTagIndexesProcessor::process(const cpp2::ListTagIndexesReq& req) { CHECK_SPACE_ID_AND_RETURN(space); folly::SharedMutex::ReadHolder rHolder(LockUtils::tagIndexLock()); - const auto& prefix = MetaServiceUtils::indexPrefix(space); + const auto& prefix = MetaKeyUtils::indexPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -29,7 +29,7 @@ void ListTagIndexesProcessor::process(const cpp2::ListTagIndexesReq& req) { std::vector items; while (iter->valid()) { auto val = iter->val(); - auto item = MetaServiceUtils::parseIndex(val); + auto item = MetaKeyUtils::parseIndex(val); if (item.get_schema_id().getType() == nebula::cpp2::SchemaID::Type::tag_id) { items.emplace_back(std::move(item)); } diff --git a/src/meta/processors/job/BalanceJobExecutor.cpp b/src/meta/processors/job/BalanceJobExecutor.cpp index 480489e1fef..134c744aec1 100644 --- a/src/meta/processors/job/BalanceJobExecutor.cpp +++ b/src/meta/processors/job/BalanceJobExecutor.cpp @@ -6,7 +6,7 @@ #include "meta/processors/job/BalanceJobExecutor.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" namespace nebula { namespace meta { diff --git a/src/meta/processors/job/GetStatsProcessor.cpp b/src/meta/processors/job/GetStatsProcessor.cpp index 843cbf3d50e..9e6df1b45df 100644 --- a/src/meta/processors/job/GetStatsProcessor.cpp +++ b/src/meta/processors/job/GetStatsProcessor.cpp @@ -13,7 +13,7 @@ void GetStatsProcessor::process(const cpp2::GetStatsReq& req) { auto spaceId = req.get_space_id(); CHECK_SPACE_ID_AND_RETURN(spaceId); - auto statsKey = MetaServiceUtils::statsKey(spaceId); + auto statsKey = MetaKeyUtils::statsKey(spaceId); std::string val; auto ret = kvstore_->get(kDefaultSpaceId, kDefaultPartId, statsKey, &val); @@ -30,7 +30,7 @@ void GetStatsProcessor::process(const cpp2::GetStatsReq& req) { onFinished(); return; } - auto statsItem = MetaServiceUtils::parseStatsVal(val); + auto statsItem = MetaKeyUtils::parseStatsVal(val); auto statisJobStatus = statsItem.get_status(); if (statisJobStatus != cpp2::JobStatus::FINISHED) { LOG(ERROR) << "SpaceId " << spaceId diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index b083d16a155..6afc79c4377 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -14,10 +14,10 @@ #include "common/http/HttpClient.h" #include "common/time/WallClock.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/common_types.h" #include "kvstore/Common.h" #include "kvstore/KVIterator.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" #include "meta/processors/admin/AdminClient.h" @@ -586,7 +586,7 @@ nebula::cpp2::ErrorCode JobManager::save(const std::string& k, const std::string } ErrorOr JobManager::getSpaceId(const std::string& name) { - auto indexKey = MetaServiceUtils::indexSpaceKey(name); + auto indexKey = MetaKeyUtils::indexSpaceKey(name); std::string val; auto retCode = kvStore_->get(kDefaultSpaceId, kDefaultPartId, indexKey, &val); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { diff --git a/src/meta/processors/job/MetaJobExecutor.cpp b/src/meta/processors/job/MetaJobExecutor.cpp index 38041533908..fa849a18283 100644 --- a/src/meta/processors/job/MetaJobExecutor.cpp +++ b/src/meta/processors/job/MetaJobExecutor.cpp @@ -7,10 +7,10 @@ #include "meta/processors/job/MetaJobExecutor.h" #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "common/utils/Utils.h" #include "interface/gen-cpp2/common_types.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" #include "meta/processors/admin/AdminClient.h" @@ -58,7 +58,7 @@ std::unique_ptr MetaJobExecutorFactory::createMetaJobExecutor( ErrorOr MetaJobExecutor::getSpaceIdFromName( const std::string& spaceName) { - auto indexKey = MetaServiceUtils::indexSpaceKey(spaceName); + auto indexKey = MetaKeyUtils::indexSpaceKey(spaceName); std::string val; auto retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, indexKey, &val); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -71,7 +71,7 @@ ErrorOr MetaJobExecutor::getSpaceIdFromNa ErrOrHosts MetaJobExecutor::getTargetHost(GraphSpaceID spaceId) { std::unique_ptr iter; - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto retCode = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, partPrefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Fetch Parts Failed, error: " << apache::thrift::util::enumNameSafe(retCode); @@ -82,8 +82,8 @@ ErrOrHosts MetaJobExecutor::getTargetHost(GraphSpaceID spaceId) { std::unordered_map> hostAndPart; std::vector>> hosts; while (iter->valid()) { - auto part = MetaServiceUtils::parsePartKeyPartId(iter->key()); - auto targets = MetaServiceUtils::parsePartVal(iter->val()); + auto part = MetaKeyUtils::parsePartKeyPartId(iter->key()); + auto targets = MetaKeyUtils::parsePartVal(iter->val()); for (auto& target : targets) { hostAndPart[target].emplace_back(part); } @@ -96,7 +96,7 @@ ErrOrHosts MetaJobExecutor::getTargetHost(GraphSpaceID spaceId) { } ErrOrHosts MetaJobExecutor::getLeaderHost(GraphSpaceID space) { - const auto& hostPrefix = MetaServiceUtils::leaderPrefix(space); + const auto& hostPrefix = MetaKeyUtils::leaderPrefix(space); std::unique_ptr leaderIter; auto retCode = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, hostPrefix, &leaderIter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -109,9 +109,9 @@ ErrOrHosts MetaJobExecutor::getLeaderHost(GraphSpaceID space) { HostAddr host; nebula::cpp2::ErrorCode code; for (; leaderIter->valid(); leaderIter->next()) { - auto spaceAndPart = MetaServiceUtils::parseLeaderKeyV3(leaderIter->key()); + auto spaceAndPart = MetaKeyUtils::parseLeaderKeyV3(leaderIter->key()); auto partId = spaceAndPart.second; - std::tie(host, std::ignore, code) = MetaServiceUtils::parseLeaderValV3(leaderIter->val()); + std::tie(host, std::ignore, code) = MetaKeyUtils::parseLeaderValV3(leaderIter->val()); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { continue; } @@ -127,7 +127,7 @@ ErrOrHosts MetaJobExecutor::getLeaderHost(GraphSpaceID space) { } ErrOrHosts MetaJobExecutor::getListenerHost(GraphSpaceID space, cpp2::ListenerType type) { - const auto& prefix = MetaServiceUtils::listenerPrefix(space, type); + const auto& prefix = MetaKeyUtils::listenerPrefix(space, type); std::unique_ptr iter; auto ret = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -148,8 +148,8 @@ ErrOrHosts MetaJobExecutor::getListenerHost(GraphSpaceID space, cpp2::ListenerTy std::vector>> hosts; while (iter->valid()) { - auto host = MetaServiceUtils::deserializeHostAddr(iter->val()); - auto part = MetaServiceUtils::parseListenerPart(iter->key()); + auto host = MetaKeyUtils::deserializeHostAddr(iter->val()); + auto part = MetaKeyUtils::parseListenerPart(iter->key()); if (std::find(activeHosts.begin(), activeHosts.end(), host) == activeHosts.end()) { LOG(ERROR) << "Invalid host : " << network::NetworkUtils::toHostsStr({host}); return nebula::cpp2::ErrorCode::E_INVALID_HOST; diff --git a/src/meta/processors/job/RebuildJobExecutor.cpp b/src/meta/processors/job/RebuildJobExecutor.cpp index 14a7421a345..ca03dff8cae 100644 --- a/src/meta/processors/job/RebuildJobExecutor.cpp +++ b/src/meta/processors/job/RebuildJobExecutor.cpp @@ -7,9 +7,9 @@ #include "meta/processors/job/RebuildJobExecutor.h" #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "common/utils/Utils.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" @@ -32,7 +32,7 @@ nebula::cpp2::ErrorCode RebuildJobExecutor::prepare() { std::string indexValue; IndexID indexId = -1; for (auto i = 0u; i < paras_.size() - 1; i++) { - auto indexKey = MetaServiceUtils::indexIndexKey(space_, paras_[i]); + auto indexKey = MetaKeyUtils::indexIndexKey(space_, paras_[i]); auto retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, indexKey, &indexValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get indexKey error indexName: " << paras_[i] diff --git a/src/meta/processors/job/SimpleConcurrentJobExecutor.cpp b/src/meta/processors/job/SimpleConcurrentJobExecutor.cpp index e5364c4a1df..7517a304e0c 100644 --- a/src/meta/processors/job/SimpleConcurrentJobExecutor.cpp +++ b/src/meta/processors/job/SimpleConcurrentJobExecutor.cpp @@ -6,7 +6,7 @@ #include "meta/processors/job/SimpleConcurrentJobExecutor.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/processors/admin/AdminClient.h" namespace nebula { diff --git a/src/meta/processors/job/StatsJobExecutor.cpp b/src/meta/processors/job/StatsJobExecutor.cpp index 18ddbc745d5..6d0ec50eaef 100644 --- a/src/meta/processors/job/StatsJobExecutor.cpp +++ b/src/meta/processors/job/StatsJobExecutor.cpp @@ -6,8 +6,8 @@ #include "meta/processors/job/StatsJobExecutor.h" +#include "common/utils/MetaKeyUtils.h" #include "common/utils/Utils.h" -#include "meta/MetaServiceUtils.h" #include "meta/common/MetaCommon.h" #include "meta/processors/Common.h" @@ -54,8 +54,8 @@ nebula::cpp2::ErrorCode StatsJobExecutor::prepare() { // Set the status of the statis job to running cpp2::StatsItem statsItem; statsItem.set_status(cpp2::JobStatus::RUNNING); - auto statsKey = MetaServiceUtils::statsKey(space_); - auto statsVal = MetaServiceUtils::statsVal(statsItem); + auto statsKey = MetaKeyUtils::statsKey(space_); + auto statsVal = MetaKeyUtils::statsVal(statsItem); return save(statsKey, statsVal); } @@ -118,7 +118,7 @@ nebula::cpp2::ErrorCode StatsJobExecutor::saveSpecialTaskStatus(const cpp2::Repo return nebula::cpp2::ErrorCode::SUCCEEDED; } cpp2::StatsItem statsItem; - auto statsKey = MetaServiceUtils::statsKey(space_); + auto statsKey = MetaKeyUtils::statsKey(space_); auto tempKey = toTempKey(req.get_job_id()); std::string val; auto ret = kvstore_->get(kDefaultSpaceId, kDefaultPartId, tempKey, &val); @@ -134,9 +134,9 @@ nebula::cpp2::ErrorCode StatsJobExecutor::saveSpecialTaskStatus(const cpp2::Repo return ret; } - statsItem = MetaServiceUtils::parseStatsVal(val); + statsItem = MetaKeyUtils::parseStatsVal(val); addStats(statsItem, *req.stats_ref()); - auto statsVal = MetaServiceUtils::statsVal(statsItem); + auto statsVal = MetaKeyUtils::statsVal(statsItem); return save(tempKey, statsVal); } @@ -149,12 +149,12 @@ nebula::cpp2::ErrorCode StatsJobExecutor::saveSpecialTaskStatus(const cpp2::Repo * @return std::string */ std::string StatsJobExecutor::toTempKey(int32_t jobId) { - std::string key = MetaServiceUtils::statsKey(space_); + std::string key = MetaKeyUtils::statsKey(space_); return key.append(reinterpret_cast(&jobId), sizeof(int32_t)); } nebula::cpp2::ErrorCode StatsJobExecutor::finish(bool exeSuccessed) { - auto statsKey = MetaServiceUtils::statsKey(space_); + auto statsKey = MetaKeyUtils::statsKey(space_); auto tempKey = toTempKey(jobId_); std::string val; auto ret = kvstore_->get(kDefaultSpaceId, kDefaultPartId, tempKey, &val); @@ -162,13 +162,13 @@ nebula::cpp2::ErrorCode StatsJobExecutor::finish(bool exeSuccessed) { LOG(ERROR) << "Can't find the statis data, spaceId : " << space_; return ret; } - auto statsItem = MetaServiceUtils::parseStatsVal(val); + auto statsItem = MetaKeyUtils::parseStatsVal(val); if (exeSuccessed) { statsItem.set_status(cpp2::JobStatus::FINISHED); } else { statsItem.set_status(cpp2::JobStatus::FAILED); } - auto statsVal = MetaServiceUtils::statsVal(statsItem); + auto statsVal = MetaKeyUtils::statsVal(statsItem); auto retCode = save(statsKey, statsVal); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Sace statis data failed, error " << apache::thrift::util::enumNameSafe(retCode); diff --git a/src/meta/processors/job/TaskDescription.cpp b/src/meta/processors/job/TaskDescription.cpp index 1a4634b66ab..4b6a36aed37 100644 --- a/src/meta/processors/job/TaskDescription.cpp +++ b/src/meta/processors/job/TaskDescription.cpp @@ -9,7 +9,7 @@ #include #include "common/time/WallClock.h" -#include "meta/MetaServiceUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/processors/job/JobStatus.h" #include "meta/processors/job/JobUtils.h" @@ -81,7 +81,7 @@ std::string TaskDescription::archiveKey() { std::string TaskDescription::taskVal() { std::string str; str.reserve(128); - str.append(MetaServiceUtils::serializeHostAddr(dest_)) + str.append(MetaKeyUtils::serializeHostAddr(dest_)) .append(reinterpret_cast(&status_), sizeof(Status)) .append(reinterpret_cast(&startTime_), sizeof(startTime_)) .append(reinterpret_cast(&stopTime_), sizeof(stopTime_)); @@ -100,7 +100,7 @@ std::tuple TaskDescription::parseVal( size_t offset = 0; folly::StringPiece raw = rawVal; - HostAddr host = MetaServiceUtils::deserializeHostAddr(raw); + HostAddr host = MetaKeyUtils::deserializeHostAddr(raw); offset += sizeof(size_t); offset += host.host.size(); offset += sizeof(uint32_t); diff --git a/src/meta/processors/kv/GetProcessor.cpp b/src/meta/processors/kv/GetProcessor.cpp index 07c2844f9b8..497a63b004d 100644 --- a/src/meta/processors/kv/GetProcessor.cpp +++ b/src/meta/processors/kv/GetProcessor.cpp @@ -10,7 +10,7 @@ namespace nebula { namespace meta { void GetProcessor::process(const cpp2::GetReq& req) { - auto key = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_key()); + auto key = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_key()); auto result = doGet(key); if (!nebula::ok(result)) { auto retCode = nebula::error(result); diff --git a/src/meta/processors/kv/MultiGetProcessor.cpp b/src/meta/processors/kv/MultiGetProcessor.cpp index 69be2c3cac1..cc924f26717 100644 --- a/src/meta/processors/kv/MultiGetProcessor.cpp +++ b/src/meta/processors/kv/MultiGetProcessor.cpp @@ -12,7 +12,7 @@ namespace meta { void MultiGetProcessor::process(const cpp2::MultiGetReq& req) { std::vector keys; for (auto& key : req.get_keys()) { - keys.emplace_back(MetaServiceUtils::assembleSegmentKey(req.get_segment(), key)); + keys.emplace_back(MetaKeyUtils::assembleSegmentKey(req.get_segment(), key)); } auto result = doMultiGet(std::move(keys)); diff --git a/src/meta/processors/kv/MultiPutProcessor.cpp b/src/meta/processors/kv/MultiPutProcessor.cpp index 87a3a76925e..f141b0b502d 100644 --- a/src/meta/processors/kv/MultiPutProcessor.cpp +++ b/src/meta/processors/kv/MultiPutProcessor.cpp @@ -13,8 +13,7 @@ void MultiPutProcessor::process(const cpp2::MultiPutReq& req) { CHECK_SEGMENT(req.get_segment()); std::vector data; for (auto& pair : req.get_pairs()) { - data.emplace_back(MetaServiceUtils::assembleSegmentKey(req.get_segment(), pair.key), - pair.value); + data.emplace_back(MetaKeyUtils::assembleSegmentKey(req.get_segment(), pair.key), pair.value); } doPut(std::move(data)); } diff --git a/src/meta/processors/kv/RemoveProcessor.cpp b/src/meta/processors/kv/RemoveProcessor.cpp index 402e8dcf91a..372f3682cc7 100644 --- a/src/meta/processors/kv/RemoveProcessor.cpp +++ b/src/meta/processors/kv/RemoveProcessor.cpp @@ -11,7 +11,7 @@ namespace meta { void RemoveProcessor::process(const cpp2::RemoveReq& req) { CHECK_SEGMENT(req.get_segment()); - auto key = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_key()); + auto key = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_key()); doRemove(key); } diff --git a/src/meta/processors/kv/RemoveRangeProcessor.cpp b/src/meta/processors/kv/RemoveRangeProcessor.cpp index e35550e9605..569029f2549 100644 --- a/src/meta/processors/kv/RemoveRangeProcessor.cpp +++ b/src/meta/processors/kv/RemoveRangeProcessor.cpp @@ -11,8 +11,8 @@ namespace meta { void RemoveRangeProcessor::process(const cpp2::RemoveRangeReq& req) { CHECK_SEGMENT(req.get_segment()); - auto start = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_start()); - auto end = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_end()); + auto start = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_start()); + auto end = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_end()); doRemoveRange(start, end); } diff --git a/src/meta/processors/kv/ScanProcessor.cpp b/src/meta/processors/kv/ScanProcessor.cpp index 94ec62874b4..557c411012e 100644 --- a/src/meta/processors/kv/ScanProcessor.cpp +++ b/src/meta/processors/kv/ScanProcessor.cpp @@ -10,8 +10,8 @@ namespace nebula { namespace meta { void ScanProcessor::process(const cpp2::ScanReq& req) { - auto start = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_start()); - auto end = MetaServiceUtils::assembleSegmentKey(req.get_segment(), req.get_end()); + auto start = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_start()); + auto end = MetaKeyUtils::assembleSegmentKey(req.get_segment(), req.get_end()); auto result = doScan(start, end); if (!nebula::ok(result)) { auto retCode = nebula::error(result); diff --git a/src/meta/processors/listener/ListenerProcessor.cpp b/src/meta/processors/listener/ListenerProcessor.cpp index 40d143134aa..01f8bcd1eb5 100644 --- a/src/meta/processors/listener/ListenerProcessor.cpp +++ b/src/meta/processors/listener/ListenerProcessor.cpp @@ -35,7 +35,7 @@ void AddListenerProcessor::process(const cpp2::AddListenerReq& req) { // TODO : (sky) if type is elasticsearch, need check text search service. folly::SharedMutex::WriteHolder wHolder(LockUtils::listenerLock()); folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::partPrefix(space); + const auto& prefix = MetaKeyUtils::partPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -48,13 +48,13 @@ void AddListenerProcessor::process(const cpp2::AddListenerReq& req) { std::vector parts; auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - parts.emplace_back(MetaServiceUtils::parsePartKeyPartId(iter->key())); + parts.emplace_back(MetaKeyUtils::parsePartKeyPartId(iter->key())); iter->next(); } std::vector data; for (size_t i = 0; i < parts.size(); i++) { - data.emplace_back(MetaServiceUtils::listenerKey(space, parts[i], type), - MetaServiceUtils::serializeHostAddr(hosts[i % hosts.size()])); + data.emplace_back(MetaKeyUtils::listenerKey(space, parts[i], type), + MetaKeyUtils::serializeHostAddr(hosts[i % hosts.size()])); } doSyncPutAndUpdate(std::move(data)); } @@ -77,7 +77,7 @@ void RemoveListenerProcessor::process(const cpp2::RemoveListenerReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::listenerLock()); std::vector keys; - const auto& prefix = MetaServiceUtils::listenerPrefix(space, type); + const auto& prefix = MetaKeyUtils::listenerPrefix(space, type); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -99,7 +99,7 @@ void ListListenerProcessor::process(const cpp2::ListListenerReq& req) { auto space = req.get_space_id(); CHECK_SPACE_ID_AND_RETURN(space); folly::SharedMutex::ReadHolder rHolder(LockUtils::listenerLock()); - const auto& prefix = MetaServiceUtils::listenerPrefix(space); + const auto& prefix = MetaKeyUtils::listenerPrefix(space); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -124,9 +124,9 @@ void ListListenerProcessor::process(const cpp2::ListListenerReq& req) { auto iter = nebula::value(iterRet).get(); while (iter->valid()) { cpp2::ListenerInfo listener; - listener.set_type(MetaServiceUtils::parseListenerType(iter->key())); - listener.set_host(MetaServiceUtils::deserializeHostAddr(iter->val())); - listener.set_part_id(MetaServiceUtils::parseListenerPart(iter->key())); + listener.set_type(MetaKeyUtils::parseListenerType(iter->key())); + listener.set_host(MetaKeyUtils::deserializeHostAddr(iter->val())); + listener.set_part_id(MetaKeyUtils::parseListenerPart(iter->key())); if (std::find(activeHosts.begin(), activeHosts.end(), *listener.host_ref()) != activeHosts.end()) { listener.set_status(cpp2::HostStatus::ONLINE); diff --git a/src/meta/processors/parts/CreateSpaceAsProcessor.cpp b/src/meta/processors/parts/CreateSpaceAsProcessor.cpp index 7f9497ca67d..09fa9a04bb8 100644 --- a/src/meta/processors/parts/CreateSpaceAsProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceAsProcessor.cpp @@ -103,7 +103,7 @@ void CreateSpaceAsProcessor::process(const cpp2::CreateSpaceAsReq &req) { ErrorOr> CreateSpaceAsProcessor::makeNewSpaceData( GraphSpaceID oldSpaceId, GraphSpaceID newSpaceId, const std::string &spaceName) { - auto oldSpaceKey = MetaServiceUtils::spaceKey(oldSpaceId); + auto oldSpaceKey = MetaKeyUtils::spaceKey(oldSpaceId); auto oldSpaceVal = doGet(oldSpaceKey); if (!nebula::ok(oldSpaceVal)) { LOG(ERROR) << "Create Space Failed : Generate new space id failed"; @@ -112,21 +112,21 @@ ErrorOr> CreateSpaceAsProcesso } std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey(spaceName), + data.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName), std::string(reinterpret_cast(&newSpaceId), sizeof(newSpaceId))); - cpp2::SpaceDesc spaceDesc = MetaServiceUtils::parseSpace(nebula::value(oldSpaceVal)); + cpp2::SpaceDesc spaceDesc = MetaKeyUtils::parseSpace(nebula::value(oldSpaceVal)); spaceDesc.set_space_name(spaceName); - data.emplace_back(MetaServiceUtils::spaceKey(newSpaceId), MetaServiceUtils::spaceVal(spaceDesc)); + data.emplace_back(MetaKeyUtils::spaceKey(newSpaceId), MetaKeyUtils::spaceVal(spaceDesc)); - auto prefix = MetaServiceUtils::partPrefix(oldSpaceId); + auto prefix = MetaKeyUtils::partPrefix(oldSpaceId); auto partPrefix = doPrefix(prefix); if (!nebula::ok(partPrefix)) { return nebula::error(partPrefix); } auto iter = nebula::value(partPrefix).get(); for (; iter->valid(); iter->next()) { - auto partId = MetaServiceUtils::parsePartKeyPartId(iter->key()); - data.emplace_back(MetaServiceUtils::partKey(newSpaceId, partId), iter->val()); + auto partId = MetaKeyUtils::parsePartKeyPartId(iter->key()); + data.emplace_back(MetaKeyUtils::partKey(newSpaceId, partId), iter->val()); } return data; } @@ -134,7 +134,7 @@ ErrorOr> CreateSpaceAsProcesso ErrorOr> CreateSpaceAsProcessor::makeNewTags( GraphSpaceID oldSpaceId, GraphSpaceID newSpaceId) { folly::SharedMutex::ReadHolder rHolder(LockUtils::tagLock()); - auto prefix = MetaServiceUtils::schemaTagsPrefix(oldSpaceId); + auto prefix = MetaKeyUtils::schemaTagsPrefix(oldSpaceId); auto tagPrefix = doPrefix(prefix); if (!nebula::ok(tagPrefix)) { if (nebula::error(tagPrefix) == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { @@ -149,14 +149,14 @@ ErrorOr> CreateSpaceAsProcesso for (; iter->valid(); iter->next()) { auto val = iter->val(); - auto tagId = MetaServiceUtils::parseTagId(iter->key()); + auto tagId = MetaKeyUtils::parseTagId(iter->key()); auto tagNameLen = *reinterpret_cast(val.data()); auto tagName = val.subpiece(sizeof(int32_t), tagNameLen).str(); - data.emplace_back(MetaServiceUtils::indexTagKey(newSpaceId, tagName), + data.emplace_back(MetaKeyUtils::indexTagKey(newSpaceId, tagName), std::string(reinterpret_cast(&tagId), sizeof(tagId))); - auto tagVer = MetaServiceUtils::parseTagVersion(iter->key()); - auto key = MetaServiceUtils::schemaTagKey(newSpaceId, tagId, tagVer); + auto tagVer = MetaKeyUtils::parseTagVersion(iter->key()); + auto key = MetaKeyUtils::schemaTagKey(newSpaceId, tagId, tagVer); data.emplace_back(std::move(key), val.str()); } return data; @@ -165,7 +165,7 @@ ErrorOr> CreateSpaceAsProcesso ErrorOr> CreateSpaceAsProcessor::makeNewEdges( GraphSpaceID oldSpaceId, GraphSpaceID newSpaceId) { folly::SharedMutex::ReadHolder rHolder(LockUtils::edgeLock()); - auto prefix = MetaServiceUtils::schemaEdgesPrefix(oldSpaceId); + auto prefix = MetaKeyUtils::schemaEdgesPrefix(oldSpaceId); auto edgePrefix = doPrefix(prefix); if (!nebula::ok(edgePrefix)) { if (nebula::error(edgePrefix) == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { @@ -180,14 +180,14 @@ ErrorOr> CreateSpaceAsProcesso for (; iter->valid(); iter->next()) { auto val = iter->val(); - auto edgeType = MetaServiceUtils::parseEdgeType(iter->key()); + auto edgeType = MetaKeyUtils::parseEdgeType(iter->key()); auto edgeNameLen = *reinterpret_cast(val.data()); auto edgeName = val.subpiece(sizeof(int32_t), edgeNameLen).str(); - data.emplace_back(MetaServiceUtils::indexTagKey(newSpaceId, edgeName), + data.emplace_back(MetaKeyUtils::indexTagKey(newSpaceId, edgeName), std::string(reinterpret_cast(&edgeType), sizeof(edgeType))); - auto ver = MetaServiceUtils::parseEdgeVersion(iter->key()); - auto key = MetaServiceUtils::schemaEdgeKey(newSpaceId, edgeType, ver); + auto ver = MetaKeyUtils::parseEdgeVersion(iter->key()); + auto key = MetaKeyUtils::schemaEdgeKey(newSpaceId, edgeType, ver); data.emplace_back(std::move(key), val.str()); } return data; @@ -196,7 +196,7 @@ ErrorOr> CreateSpaceAsProcesso ErrorOr> CreateSpaceAsProcessor::makeNewIndexes( GraphSpaceID oldSpaceId, GraphSpaceID newSpaceId) { folly::SharedMutex::ReadHolder rHolder(LockUtils::edgeLock()); - auto prefix = MetaServiceUtils::indexPrefix(oldSpaceId); + auto prefix = MetaKeyUtils::indexPrefix(oldSpaceId); auto indexPrefix = doPrefix(prefix); if (!nebula::ok(indexPrefix)) { if (nebula::error(indexPrefix) == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { @@ -211,16 +211,15 @@ ErrorOr> CreateSpaceAsProcesso for (; iter->valid(); iter->next()) { auto val = iter->val(); - auto indexId = MetaServiceUtils::parseIndexesKeyIndexID(iter->key()); + auto indexId = MetaKeyUtils::parseIndexesKeyIndexID(iter->key()); - cpp2::IndexItem idxItem = MetaServiceUtils::parseIndex(val.str()); + cpp2::IndexItem idxItem = MetaKeyUtils::parseIndex(val.str()); auto indexName = idxItem.get_index_name(); - data.emplace_back(MetaServiceUtils::indexIndexKey(newSpaceId, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(newSpaceId, indexName), std::string(reinterpret_cast(&indexId), sizeof(indexId))); - data.emplace_back(MetaServiceUtils::indexKey(newSpaceId, indexId), - MetaServiceUtils::indexVal(idxItem)); + data.emplace_back(MetaKeyUtils::indexKey(newSpaceId, indexId), MetaKeyUtils::indexVal(idxItem)); } return data; } diff --git a/src/meta/processors/parts/CreateSpaceProcessor.cpp b/src/meta/processors/parts/CreateSpaceProcessor.cpp index 182e663c40e..168d79db489 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceProcessor.cpp @@ -107,15 +107,15 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { auto spaceId = nebula::value(idRet); std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey(spaceName), + data.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName), std::string(reinterpret_cast(&spaceId), sizeof(spaceId))); - data.emplace_back(MetaServiceUtils::spaceKey(spaceId), MetaServiceUtils::spaceVal(properties)); + data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; if (properties.group_name_ref().has_value()) { auto& groupName = *properties.group_name_ref(); LOG(INFO) << "Create Space on group: " << groupName; - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); auto ret = doGet(groupKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -128,7 +128,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { return; } - auto zones = MetaServiceUtils::parseZoneNames(nebula::value(ret)); + auto zones = MetaKeyUtils::parseZoneNames(nebula::value(ret)); int32_t zoneNum = zones.size(); if (replicaFactor > zoneNum) { LOG(ERROR) << "Replication number should less than or equal to zone number."; @@ -152,7 +152,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { hostLoading_ = std::move(nebula::value(hostLoadingRet)); std::unordered_map zoneHosts; for (auto& zone : zones) { - auto zoneKey = MetaServiceUtils::zoneKey(zone); + auto zoneKey = MetaKeyUtils::zoneKey(zone); auto zoneValueRet = doGet(std::move(zoneKey)); if (!nebula::ok(zoneValueRet)) { code = nebula::error(zoneValueRet); @@ -163,7 +163,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { break; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); for (auto& host : hosts) { auto hostIter = hostLoading_.find(host); if (hostIter == hostLoading_.end()) { @@ -212,8 +212,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { } VLOG(3) << "Space " << spaceId << " part " << partId << " hosts " << ss.str(); - data.emplace_back(MetaServiceUtils::partKey(spaceId, partId), - MetaServiceUtils::partVal(partHosts)); + data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); } } else { auto hostsRet = ActiveHostsMan::getActiveHosts(kvstore_); @@ -243,8 +242,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { for (auto partId = 1; partId <= partitionNum; partId++) { auto partHosts = pickHosts(partId, hosts, replicaFactor); - data.emplace_back(MetaServiceUtils::partKey(spaceId, partId), - MetaServiceUtils::partVal(partHosts)); + data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); } } @@ -273,7 +271,7 @@ Hosts CreateSpaceProcessor::pickHosts(PartitionID partId, ErrorOr> CreateSpaceProcessor::getHostLoading() { - const auto& prefix = MetaServiceUtils::partPrefix(); + const auto& prefix = MetaKeyUtils::partPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { @@ -284,7 +282,7 @@ CreateSpaceProcessor::getHostLoading() { std::unordered_map result; auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto hosts = MetaServiceUtils::parsePartVal(iter->val()); + auto hosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& host : hosts) { result[host]++; } diff --git a/src/meta/processors/parts/DropSpaceProcessor.cpp b/src/meta/processors/parts/DropSpaceProcessor.cpp index 34588e7442f..a459dbd11cb 100644 --- a/src/meta/processors/parts/DropSpaceProcessor.cpp +++ b/src/meta/processors/parts/DropSpaceProcessor.cpp @@ -36,7 +36,7 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { std::vector deleteKeys; // 1. Delete related part meta data. - auto prefix = MetaServiceUtils::partPrefix(spaceId); + auto prefix = MetaKeyUtils::partPrefix(spaceId); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -54,11 +54,11 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { } // 2. Delete this space data - deleteKeys.emplace_back(MetaServiceUtils::indexSpaceKey(spaceName)); - deleteKeys.emplace_back(MetaServiceUtils::spaceKey(spaceId)); + deleteKeys.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName)); + deleteKeys.emplace_back(MetaKeyUtils::spaceKey(spaceId)); // 3. Delete related role data. - auto rolePrefix = MetaServiceUtils::roleSpacePrefix(spaceId); + auto rolePrefix = MetaKeyUtils::roleSpacePrefix(spaceId); auto roleRet = doPrefix(rolePrefix); if (!nebula::ok(roleRet)) { auto retCode = nebula::error(roleRet); @@ -71,14 +71,14 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { auto roleIter = nebula::value(roleRet).get(); while (roleIter->valid()) { - VLOG(3) << "Revoke role " << MetaServiceUtils::parseRoleStr(roleIter->val()) << " for user " - << MetaServiceUtils::parseRoleUser(roleIter->key()); + VLOG(3) << "Revoke role " << MetaKeyUtils::parseRoleStr(roleIter->val()) << " for user " + << MetaKeyUtils::parseRoleUser(roleIter->key()); deleteKeys.emplace_back(roleIter->key()); roleIter->next(); } // 4. Delete listener meta data - auto lstPrefix = MetaServiceUtils::listenerPrefix(spaceId); + auto lstPrefix = MetaKeyUtils::listenerPrefix(spaceId); auto lstRet = doPrefix(rolePrefix); if (!nebula::ok(lstRet)) { auto retCode = nebula::error(lstRet); @@ -96,11 +96,11 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { } // 5. Delete related statis data - auto statiskey = MetaServiceUtils::statsKey(spaceId); + auto statiskey = MetaKeyUtils::statsKey(spaceId); deleteKeys.emplace_back(statiskey); // 6. Delte related fulltext index meta data - auto ftPrefix = MetaServiceUtils::fulltextIndexPrefix(); + auto ftPrefix = MetaKeyUtils::fulltextIndexPrefix(); auto ftRet = doPrefix(ftPrefix); if (!nebula::ok(ftRet)) { auto retCode = nebula::error(ftRet); @@ -112,7 +112,7 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { } auto ftIter = nebula::value(ftRet).get(); while (ftIter->valid()) { - auto index = MetaServiceUtils::parsefulltextIndex(ftIter->val()); + auto index = MetaKeyUtils::parsefulltextIndex(ftIter->val()); if (index.get_space_id() == spaceId) { deleteKeys.emplace_back(ftIter->key()); } @@ -120,7 +120,7 @@ void DropSpaceProcessor::process(const cpp2::DropSpaceReq& req) { } // 7. Delete local_id meta data - auto localIdkey = MetaServiceUtils::localIdKey(spaceId); + auto localIdkey = MetaKeyUtils::localIdKey(spaceId); deleteKeys.emplace_back(localIdkey); doSyncMultiRemoveAndUpdate(std::move(deleteKeys)); diff --git a/src/meta/processors/parts/GetPartsAllocProcessor.cpp b/src/meta/processors/parts/GetPartsAllocProcessor.cpp index 46ec08b4fa6..9e26df3aa70 100644 --- a/src/meta/processors/parts/GetPartsAllocProcessor.cpp +++ b/src/meta/processors/parts/GetPartsAllocProcessor.cpp @@ -12,7 +12,7 @@ namespace meta { void GetPartsAllocProcessor::process(const cpp2::GetPartsAllocReq& req) { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); auto spaceId = req.get_space_id(); - auto prefix = MetaServiceUtils::partPrefix(spaceId); + auto prefix = MetaKeyUtils::partPrefix(spaceId); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -28,7 +28,7 @@ void GetPartsAllocProcessor::process(const cpp2::GetPartsAllocReq& req) { auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - std::vector partHosts = MetaServiceUtils::parsePartVal(iter->val()); + std::vector partHosts = MetaKeyUtils::parsePartVal(iter->val()); parts.emplace(partId, std::move(partHosts)); iter->next(); } @@ -44,7 +44,7 @@ void GetPartsAllocProcessor::process(const cpp2::GetPartsAllocReq& req) { std::unordered_map GetPartsAllocProcessor::getTerm(GraphSpaceID spaceId) { std::unordered_map ret; - auto spaceKey = MetaServiceUtils::spaceKey(spaceId); + auto spaceKey = MetaKeyUtils::spaceKey(spaceId); auto spaceVal = doGet(spaceKey); if (!nebula::ok(spaceVal)) { auto rc = nebula::error(spaceVal); @@ -54,14 +54,14 @@ std::unordered_map GetPartsAllocProcessor::getTerm(GraphSpa return ret; } - auto properties = MetaServiceUtils::parseSpace(nebula::value(spaceVal)); + auto properties = MetaKeyUtils::parseSpace(nebula::value(spaceVal)); auto partNum = properties.get_partition_num(); std::vector partIdVec; std::vector leaderKeys; for (auto partId = 1; partId <= partNum; ++partId) { partIdVec.emplace_back(partId); - leaderKeys.emplace_back(MetaServiceUtils::leaderKey(spaceId, partId)); + leaderKeys.emplace_back(MetaKeyUtils::leaderKey(spaceId, partId)); } std::vector vals; @@ -76,7 +76,7 @@ std::unordered_map GetPartsAllocProcessor::getTerm(GraphSpa TermID term; for (auto i = 0U; i != vals.size(); ++i) { if (statusVec[i].ok()) { - std::tie(std::ignore, term, code) = MetaServiceUtils::parseLeaderValV3(vals[i]); + std::tie(std::ignore, term, code) = MetaKeyUtils::parseLeaderValV3(vals[i]); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(WARNING) << apache::thrift::util::enumNameSafe(code); LOG(INFO) << folly::sformat("term of part {} is invalid", partIdVec[i]); diff --git a/src/meta/processors/parts/GetSpaceProcessor.cpp b/src/meta/processors/parts/GetSpaceProcessor.cpp index 4e3aa1c924e..c15906a49c0 100644 --- a/src/meta/processors/parts/GetSpaceProcessor.cpp +++ b/src/meta/processors/parts/GetSpaceProcessor.cpp @@ -24,7 +24,7 @@ void GetSpaceProcessor::process(const cpp2::GetSpaceReq& req) { } auto spaceId = nebula::value(spaceRet); - std::string spaceKey = MetaServiceUtils::spaceKey(spaceId); + std::string spaceKey = MetaKeyUtils::spaceKey(spaceId); auto ret = doGet(spaceKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -35,7 +35,7 @@ void GetSpaceProcessor::process(const cpp2::GetSpaceReq& req) { return; } - auto properties = MetaServiceUtils::parseSpace(nebula::value(ret)); + auto properties = MetaKeyUtils::parseSpace(nebula::value(ret)); VLOG(3) << "Get Space SpaceName: " << spaceName << ", Partition Num " << properties.get_partition_num() << ", Replica Factor " << properties.get_replica_factor(); diff --git a/src/meta/processors/parts/ListHostsProcessor.cpp b/src/meta/processors/parts/ListHostsProcessor.cpp index cea7e8b7d66..c812b59c0e3 100644 --- a/src/meta/processors/parts/ListHostsProcessor.cpp +++ b/src/meta/processors/parts/ListHostsProcessor.cpp @@ -100,7 +100,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro if (role == cpp2::HostRole::META) { return allMetaHostsStatus(); } - const auto& hostPrefix = MetaServiceUtils::hostPrefix(); + const auto& hostPrefix = MetaKeyUtils::hostPrefix(); auto ret = doPrefix(hostPrefix); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -120,7 +120,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro } cpp2::HostItem item; - auto host = MetaServiceUtils::parseHostKey(iter->key()); + auto host = MetaKeyUtils::parseHostKey(iter->key()); item.set_hostAddr(std::move(host)); item.set_role(info.role_); @@ -161,7 +161,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillLeaders() { } auto activeHosts = nebula::value(activeHostsRet); - const auto& prefix = MetaServiceUtils::leaderPrefix(); + const auto& prefix = MetaKeyUtils::leaderPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { retCode = nebula::error(iterRet); @@ -181,7 +181,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillLeaders() { nebula::cpp2::ErrorCode code; std::vector removeLeadersKey; for (; iter->valid(); iter->next()) { - auto spaceIdAndPartId = MetaServiceUtils::parseLeaderKeyV3(iter->key()); + auto spaceIdAndPartId = MetaKeyUtils::parseLeaderKeyV3(iter->key()); VLOG(1) << "show hosts: space = " << spaceIdAndPartId.first << ", part = " << spaceIdAndPartId.second; // If the space in the leader key don't exist, remove leader key @@ -192,7 +192,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillLeaders() { continue; } - std::tie(host, term, code) = MetaServiceUtils::parseLeaderValV3(iter->val()); + std::tie(host, term, code) = MetaKeyUtils::parseLeaderValV3(iter->val()); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { continue; } @@ -226,7 +226,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillAllParts() { // get space name by space id const auto& spaceName = spaceIdNameMap_[spaceId]; - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto iterPartRet = doPrefix(partPrefix); if (!nebula::ok(iterPartRet)) { auto retCode = nebula::error(iterPartRet); @@ -238,8 +238,8 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillAllParts() { auto partIter = nebula::value(iterPartRet).get(); std::unordered_map> hostParts; while (partIter->valid()) { - PartitionID partId = MetaServiceUtils::parsePartKeyPartId(partIter->key()); - auto partHosts = MetaServiceUtils::parsePartVal(partIter->val()); + PartitionID partId = MetaKeyUtils::parsePartKeyPartId(partIter->key()); + auto partHosts = MetaKeyUtils::parsePartVal(partIter->val()); for (auto& host : partHosts) { hostParts[host].emplace_back(partId); } @@ -296,7 +296,7 @@ void ListHostsProcessor::removeInvalidLeaders(std::vector&& removeL nebula::cpp2::ErrorCode ListHostsProcessor::getSpaceIdNameMap() { // Get all spaces - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto iterRet = doPrefix(spacePrefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -309,9 +309,9 @@ nebula::cpp2::ErrorCode ListHostsProcessor::getSpaceIdNameMap() { auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); spaceIds_.emplace_back(spaceId); - spaceIdNameMap_.emplace(spaceId, MetaServiceUtils::spaceName(iter->val())); + spaceIdNameMap_.emplace(spaceId, MetaKeyUtils::spaceName(iter->val())); iter->next(); } return nebula::cpp2::ErrorCode::SUCCEEDED; diff --git a/src/meta/processors/parts/ListPartsProcessor.cpp b/src/meta/processors/parts/ListPartsProcessor.cpp index bd2207131dc..ebc25154ee0 100644 --- a/src/meta/processors/parts/ListPartsProcessor.cpp +++ b/src/meta/processors/parts/ListPartsProcessor.cpp @@ -24,7 +24,7 @@ void ListPartsProcessor::process(const cpp2::ListPartsReq& req) { showAllParts_ = false; folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); for (const auto& partId : partIds_) { - auto partKey = MetaServiceUtils::partKey(spaceId_, partId); + auto partKey = MetaKeyUtils::partKey(spaceId_, partId); auto ret = doGet(std::move(partKey)); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -34,7 +34,7 @@ void ListPartsProcessor::process(const cpp2::ListPartsReq& req) { return; } auto hosts = std::move(nebula::value(ret)); - partHostsMap[partId] = MetaServiceUtils::parsePartVal(hosts); + partHostsMap[partId] = MetaKeyUtils::parsePartVal(hosts); } } else { // Show all parts @@ -87,7 +87,7 @@ ListPartsProcessor::getAllParts() { std::unordered_map> partHostsMap; folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); - const auto& prefix = MetaServiceUtils::partPrefix(spaceId_); + const auto& prefix = MetaKeyUtils::partPrefix(spaceId_); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -100,7 +100,7 @@ ListPartsProcessor::getAllParts() { auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - std::vector partHosts = MetaServiceUtils::parsePartVal(iter->val()); + std::vector partHosts = MetaKeyUtils::parsePartVal(iter->val()); partHostsMap.emplace(partId, std::move(partHosts)); iter->next(); } @@ -118,7 +118,7 @@ nebula::cpp2::ErrorCode ListPartsProcessor::getLeaderDist(std::vector leaderKeys; for (auto& partItem : partItems) { - auto key = MetaServiceUtils::leaderKey(spaceId_, partItem.get_part_id()); + auto key = MetaKeyUtils::leaderKey(spaceId_, partItem.get_part_id()); leaderKeys.emplace_back(std::move(key)); } @@ -137,7 +137,7 @@ nebula::cpp2::ErrorCode ListPartsProcessor::getLeaderDist(std::vector spaces; while (iter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(iter->key()); - auto spaceName = MetaServiceUtils::spaceName(iter->val()); + auto spaceId = MetaKeyUtils::spaceId(iter->key()); + auto spaceName = MetaKeyUtils::spaceName(iter->val()); VLOG(3) << "List spaces " << spaceId << ", name " << spaceName; cpp2::IdName space; space.set_id(to(spaceId, EntryType::SPACE)); diff --git a/src/meta/processors/schema/AlterEdgeProcessor.cpp b/src/meta/processors/schema/AlterEdgeProcessor.cpp index 03e1dd72948..d50e708e790 100644 --- a/src/meta/processors/schema/AlterEdgeProcessor.cpp +++ b/src/meta/processors/schema/AlterEdgeProcessor.cpp @@ -30,7 +30,7 @@ void AlterEdgeProcessor::process(const cpp2::AlterEdgeReq& req) { auto edgeType = nebula::value(ret); // Check the edge belongs to the space - auto edgePrefix = MetaServiceUtils::schemaEdgePrefix(spaceId, edgeType); + auto edgePrefix = MetaKeyUtils::schemaEdgePrefix(spaceId, edgeType); auto retPre = doPrefix(edgePrefix); if (!nebula::ok(retPre)) { auto retCode = nebula::error(retPre); @@ -49,8 +49,8 @@ void AlterEdgeProcessor::process(const cpp2::AlterEdgeReq& req) { } // Get lasted version of edge - auto version = MetaServiceUtils::parseEdgeVersion(iter->key()) + 1; - auto schema = MetaServiceUtils::parseSchema(iter->val()); + auto version = MetaKeyUtils::parseEdgeVersion(iter->key()) + 1; + auto schema = MetaKeyUtils::parseSchema(iter->val()); auto columns = schema.get_columns(); auto prop = schema.get_schema_prop(); @@ -146,8 +146,8 @@ void AlterEdgeProcessor::process(const cpp2::AlterEdgeReq& req) { std::vector data; LOG(INFO) << "Alter edge " << edgeName << ", edgeType " << edgeType; - data.emplace_back(MetaServiceUtils::schemaEdgeKey(spaceId, edgeType, version), - MetaServiceUtils::schemaVal(edgeName, schema)); + data.emplace_back(MetaKeyUtils::schemaEdgeKey(spaceId, edgeType, version), + MetaKeyUtils::schemaVal(edgeName, schema)); resp_.set_id(to(edgeType, EntryType::EDGE)); doSyncPutAndUpdate(std::move(data)); } diff --git a/src/meta/processors/schema/AlterTagProcessor.cpp b/src/meta/processors/schema/AlterTagProcessor.cpp index 6f46605308e..987b13912f3 100644 --- a/src/meta/processors/schema/AlterTagProcessor.cpp +++ b/src/meta/processors/schema/AlterTagProcessor.cpp @@ -30,7 +30,7 @@ void AlterTagProcessor::process(const cpp2::AlterTagReq& req) { auto tagId = nebula::value(ret); // Check the tag belongs to the space - auto tagPrefix = MetaServiceUtils::schemaTagPrefix(spaceId, tagId); + auto tagPrefix = MetaKeyUtils::schemaTagPrefix(spaceId, tagId); auto retPre = doPrefix(tagPrefix); if (!nebula::ok(retPre)) { auto retCode = nebula::error(retPre); @@ -49,8 +49,8 @@ void AlterTagProcessor::process(const cpp2::AlterTagReq& req) { } // Get the last version of the tag - auto version = MetaServiceUtils::parseTagVersion(iter->key()) + 1; - auto schema = MetaServiceUtils::parseSchema(iter->val()); + auto version = MetaKeyUtils::parseTagVersion(iter->key()) + 1; + auto schema = MetaKeyUtils::parseSchema(iter->val()); auto columns = schema.get_columns(); auto prop = schema.get_schema_prop(); @@ -145,8 +145,8 @@ void AlterTagProcessor::process(const cpp2::AlterTagReq& req) { std::vector data; LOG(INFO) << "Alter Tag " << tagName << ", tagId " << tagId; - data.emplace_back(MetaServiceUtils::schemaTagKey(spaceId, tagId, version), - MetaServiceUtils::schemaVal(tagName, schema)); + data.emplace_back(MetaKeyUtils::schemaTagKey(spaceId, tagId, version), + MetaKeyUtils::schemaVal(tagName, schema)); resp_.set_id(to(tagId, EntryType::TAG)); doSyncPutAndUpdate(std::move(data)); } diff --git a/src/meta/processors/schema/CreateEdgeProcessor.cpp b/src/meta/processors/schema/CreateEdgeProcessor.cpp index a06a0f46b4d..f42f5c74519 100644 --- a/src/meta/processors/schema/CreateEdgeProcessor.cpp +++ b/src/meta/processors/schema/CreateEdgeProcessor.cpp @@ -83,10 +83,10 @@ void CreateEdgeProcessor::process(const cpp2::CreateEdgeReq& req) { auto edgeType = nebula::value(edgeTypeRet); std::vector data; - data.emplace_back(MetaServiceUtils::indexEdgeKey(spaceId, edgeName), + data.emplace_back(MetaKeyUtils::indexEdgeKey(spaceId, edgeName), std::string(reinterpret_cast(&edgeType), sizeof(EdgeType))); - data.emplace_back(MetaServiceUtils::schemaEdgeKey(spaceId, edgeType, 0), - MetaServiceUtils::schemaVal(edgeName, schema)); + data.emplace_back(MetaKeyUtils::schemaEdgeKey(spaceId, edgeType, 0), + MetaKeyUtils::schemaVal(edgeName, schema)); LOG(INFO) << "Create Edge " << edgeName << ", edgeType " << edgeType; resp_.set_id(to(edgeType, EntryType::EDGE)); diff --git a/src/meta/processors/schema/CreateTagProcessor.cpp b/src/meta/processors/schema/CreateTagProcessor.cpp index 85a3e2a0397..7f0c1cc561c 100644 --- a/src/meta/processors/schema/CreateTagProcessor.cpp +++ b/src/meta/processors/schema/CreateTagProcessor.cpp @@ -83,10 +83,10 @@ void CreateTagProcessor::process(const cpp2::CreateTagReq& req) { auto tagId = nebula::value(tagRet); std::vector data; - data.emplace_back(MetaServiceUtils::indexTagKey(spaceId, tagName), + data.emplace_back(MetaKeyUtils::indexTagKey(spaceId, tagName), std::string(reinterpret_cast(&tagId), sizeof(TagID))); - data.emplace_back(MetaServiceUtils::schemaTagKey(spaceId, tagId, 0), - MetaServiceUtils::schemaVal(tagName, schema)); + data.emplace_back(MetaKeyUtils::schemaTagKey(spaceId, tagId, 0), + MetaKeyUtils::schemaVal(tagName, schema)); LOG(INFO) << "Create Tag " << tagName << ", TagID " << tagId; diff --git a/src/meta/processors/schema/DropEdgeProcessor.cpp b/src/meta/processors/schema/DropEdgeProcessor.cpp index 9807f6eb3a9..f74ac9dffa2 100644 --- a/src/meta/processors/schema/DropEdgeProcessor.cpp +++ b/src/meta/processors/schema/DropEdgeProcessor.cpp @@ -18,7 +18,7 @@ void DropEdgeProcessor::process(const cpp2::DropEdgeReq& req) { auto edgeName = req.get_edge_name(); EdgeType edgeType; - auto indexKey = MetaServiceUtils::indexEdgeKey(spaceId, edgeName); + auto indexKey = MetaKeyUtils::indexEdgeKey(spaceId, edgeName); auto iRet = doGet(indexKey); if (nebula::ok(iRet)) { edgeType = *reinterpret_cast(nebula::value(iRet).c_str()); @@ -85,7 +85,7 @@ void DropEdgeProcessor::process(const cpp2::DropEdgeReq& req) { ErrorOr> DropEdgeProcessor::getEdgeKeys( GraphSpaceID id, EdgeType edgeType) { std::vector keys; - auto key = MetaServiceUtils::schemaEdgePrefix(id, edgeType); + auto key = MetaKeyUtils::schemaEdgePrefix(id, edgeType); auto iterRet = doPrefix(key); if (!nebula::ok(iterRet)) { LOG(ERROR) << "Edge schema prefix failed, edgetype " << edgeType; diff --git a/src/meta/processors/schema/DropTagProcessor.cpp b/src/meta/processors/schema/DropTagProcessor.cpp index b9243a82794..63d03f95f11 100644 --- a/src/meta/processors/schema/DropTagProcessor.cpp +++ b/src/meta/processors/schema/DropTagProcessor.cpp @@ -18,7 +18,7 @@ void DropTagProcessor::process(const cpp2::DropTagReq& req) { auto tagName = req.get_tag_name(); TagID tagId; - auto indexKey = MetaServiceUtils::indexTagKey(spaceId, tagName); + auto indexKey = MetaKeyUtils::indexTagKey(spaceId, tagName); auto iRet = doGet(indexKey); if (nebula::ok(iRet)) { tagId = *reinterpret_cast(nebula::value(iRet).c_str()); @@ -85,7 +85,7 @@ void DropTagProcessor::process(const cpp2::DropTagReq& req) { ErrorOr> DropTagProcessor::getTagKeys( GraphSpaceID id, TagID tagId) { std::vector keys; - auto key = MetaServiceUtils::schemaTagPrefix(id, tagId); + auto key = MetaKeyUtils::schemaTagPrefix(id, tagId); auto iterRet = doPrefix(key); if (!nebula::ok(iterRet)) { LOG(ERROR) << "Tag schema prefix failed, tag id " << tagId; diff --git a/src/meta/processors/schema/GetEdgeProcessor.cpp b/src/meta/processors/schema/GetEdgeProcessor.cpp index 3b750b6cea3..159ef1e1df1 100644 --- a/src/meta/processors/schema/GetEdgeProcessor.cpp +++ b/src/meta/processors/schema/GetEdgeProcessor.cpp @@ -28,7 +28,7 @@ void GetEdgeProcessor::process(const cpp2::GetEdgeReq& req) { std::string schemaValue; // Get the lastest version if (ver < 0) { - auto edgePrefix = MetaServiceUtils::schemaEdgePrefix(spaceId, edgeType); + auto edgePrefix = MetaKeyUtils::schemaEdgePrefix(spaceId, edgeType); auto ret = doPrefix(edgePrefix); if (!nebula::ok(ret)) { LOG(ERROR) << "Get Edge SpaceID: " << spaceId << ", edgeName: " << edgeName @@ -48,7 +48,7 @@ void GetEdgeProcessor::process(const cpp2::GetEdgeReq& req) { } schemaValue = iter->val().str(); } else { - auto edgeKey = MetaServiceUtils::schemaEdgeKey(spaceId, edgeType, ver); + auto edgeKey = MetaKeyUtils::schemaEdgeKey(spaceId, edgeType, ver); auto ret = doGet(edgeKey); if (!nebula::ok(ret)) { LOG(ERROR) << "Get Edge SpaceID: " << spaceId << ", edgeName: " << edgeName << ", version " @@ -62,7 +62,7 @@ void GetEdgeProcessor::process(const cpp2::GetEdgeReq& req) { VLOG(3) << "Get Edge SpaceID: " << spaceId << ", edgeName: " << edgeName << ", version " << ver; handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); - resp_.set_schema(MetaServiceUtils::parseSchema(schemaValue)); + resp_.set_schema(MetaKeyUtils::parseSchema(schemaValue)); onFinished(); } diff --git a/src/meta/processors/schema/GetTagProcessor.cpp b/src/meta/processors/schema/GetTagProcessor.cpp index ef3e2926c0e..ef14df16123 100644 --- a/src/meta/processors/schema/GetTagProcessor.cpp +++ b/src/meta/processors/schema/GetTagProcessor.cpp @@ -28,7 +28,7 @@ void GetTagProcessor::process(const cpp2::GetTagReq& req) { std::string schemaValue; // Get the lastest version if (ver < 0) { - auto tagPrefix = MetaServiceUtils::schemaTagPrefix(spaceId, tagId); + auto tagPrefix = MetaKeyUtils::schemaTagPrefix(spaceId, tagId); auto ret = doPrefix(tagPrefix); if (!nebula::ok(ret)) { LOG(ERROR) << "Get Tag SpaceID: " << spaceId << ", tagName: " << tagName @@ -48,7 +48,7 @@ void GetTagProcessor::process(const cpp2::GetTagReq& req) { } schemaValue = iter->val().str(); } else { - auto tagKey = MetaServiceUtils::schemaTagKey(spaceId, tagId, ver); + auto tagKey = MetaKeyUtils::schemaTagKey(spaceId, tagId, ver); auto ret = doGet(tagKey); if (!nebula::ok(ret)) { LOG(ERROR) << "Get Tag SpaceID: " << spaceId << ", tagName: " << tagName << ", version " @@ -63,7 +63,7 @@ void GetTagProcessor::process(const cpp2::GetTagReq& req) { VLOG(3) << "Get Tag SpaceID: " << spaceId << ", tagName: " << tagName << ", version " << ver; handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); - resp_.set_schema(MetaServiceUtils::parseSchema(schemaValue)); + resp_.set_schema(MetaKeyUtils::parseSchema(schemaValue)); onFinished(); } diff --git a/src/meta/processors/schema/ListEdgesProcessor.cpp b/src/meta/processors/schema/ListEdgesProcessor.cpp index e00c3e7a3e5..af6500555de 100644 --- a/src/meta/processors/schema/ListEdgesProcessor.cpp +++ b/src/meta/processors/schema/ListEdgesProcessor.cpp @@ -14,7 +14,7 @@ void ListEdgesProcessor::process(const cpp2::ListEdgesReq &req) { CHECK_SPACE_ID_AND_RETURN(spaceId); folly::SharedMutex::ReadHolder rHolder(LockUtils::edgeLock()); - auto prefix = MetaServiceUtils::schemaEdgesPrefix(spaceId); + auto prefix = MetaKeyUtils::schemaEdgesPrefix(spaceId); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { LOG(ERROR) << "List Edges failed, SpaceID: " << spaceId; @@ -29,10 +29,10 @@ void ListEdgesProcessor::process(const cpp2::ListEdgesReq &req) { auto key = iter->key(); auto val = iter->val(); auto edgeType = *reinterpret_cast(key.data() + prefix.size()); - auto version = MetaServiceUtils::parseEdgeVersion(key); + auto version = MetaKeyUtils::parseEdgeVersion(key); auto nameLen = *reinterpret_cast(val.data()); auto edgeName = val.subpiece(sizeof(int32_t), nameLen).str(); - auto schema = MetaServiceUtils::parseSchema(val); + auto schema = MetaKeyUtils::parseSchema(val); cpp2::EdgeItem edge; edge.set_edge_type(edgeType); edge.set_edge_name(std::move(edgeName)); diff --git a/src/meta/processors/schema/ListTagsProcessor.cpp b/src/meta/processors/schema/ListTagsProcessor.cpp index 67cd1fb5125..4cd8ed94ead 100644 --- a/src/meta/processors/schema/ListTagsProcessor.cpp +++ b/src/meta/processors/schema/ListTagsProcessor.cpp @@ -14,7 +14,7 @@ void ListTagsProcessor::process(const cpp2::ListTagsReq &req) { CHECK_SPACE_ID_AND_RETURN(spaceId); folly::SharedMutex::ReadHolder rHolder(LockUtils::tagLock()); - auto prefix = MetaServiceUtils::schemaTagsPrefix(spaceId); + auto prefix = MetaKeyUtils::schemaTagsPrefix(spaceId); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { LOG(ERROR) << "List Tags failed, SpaceID: " << spaceId; @@ -29,10 +29,10 @@ void ListTagsProcessor::process(const cpp2::ListTagsReq &req) { auto key = iter->key(); auto val = iter->val(); auto tagID = *reinterpret_cast(key.data() + prefix.size()); - auto version = MetaServiceUtils::parseTagVersion(key); + auto version = MetaKeyUtils::parseTagVersion(key); auto nameLen = *reinterpret_cast(val.data()); auto tagName = val.subpiece(sizeof(int32_t), nameLen).str(); - auto schema = MetaServiceUtils::parseSchema(val); + auto schema = MetaKeyUtils::parseSchema(val); cpp2::TagItem item; item.set_tag_id(tagID); item.set_tag_name(std::move(tagName)); diff --git a/src/meta/processors/session/SessionManagerProcessor.cpp b/src/meta/processors/session/SessionManagerProcessor.cpp index 4058f004ea9..cc157f5a82b 100644 --- a/src/meta/processors/session/SessionManagerProcessor.cpp +++ b/src/meta/processors/session/SessionManagerProcessor.cpp @@ -30,8 +30,8 @@ void CreateSessionProcessor::process(const cpp2::CreateSessionReq& req) { session.set_client_ip(req.get_client_ip()); std::vector data; - data.emplace_back(MetaServiceUtils::sessionKey(session.get_session_id()), - MetaServiceUtils::sessionVal(session)); + data.emplace_back(MetaKeyUtils::sessionKey(session.get_session_id()), + MetaKeyUtils::sessionVal(session)); resp_.set_session(session); ret = doSyncPut(std::move(data)); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -50,7 +50,7 @@ void UpdateSessionsProcessor::process(const cpp2::UpdateSessionsReq& req) { killedQueries; for (auto& session : req.get_sessions()) { auto sessionId = session.get_session_id(); - auto sessionKey = MetaServiceUtils::sessionKey(sessionId); + auto sessionKey = MetaKeyUtils::sessionKey(sessionId); auto ret = doGet(sessionKey); if (!nebula::ok(ret)) { auto errCode = nebula::error(ret); @@ -67,7 +67,7 @@ void UpdateSessionsProcessor::process(const cpp2::UpdateSessionsReq& req) { // client. auto& newQueries = *session.queries_ref(); std::unordered_map killedQueriesInCurrentSession; - auto sessionInMeta = MetaServiceUtils::parseSessionVal(nebula::value(ret)); + auto sessionInMeta = MetaKeyUtils::parseSessionVal(nebula::value(ret)); for (const auto& savedQuery : sessionInMeta.get_queries()) { auto epId = savedQuery.first; auto newQuery = newQueries.find(epId); @@ -91,8 +91,7 @@ void UpdateSessionsProcessor::process(const cpp2::UpdateSessionsReq& req) { continue; } - data.emplace_back(MetaServiceUtils::sessionKey(sessionId), - MetaServiceUtils::sessionVal(session)); + data.emplace_back(MetaKeyUtils::sessionKey(sessionId), MetaKeyUtils::sessionVal(session)); } auto ret = doSyncPut(std::move(data)); @@ -111,7 +110,7 @@ void UpdateSessionsProcessor::process(const cpp2::UpdateSessionsReq& req) { void ListSessionsProcessor::process(const cpp2::ListSessionsReq&) { folly::SharedMutex::ReadHolder rHolder(LockUtils::sessionLock()); - auto& prefix = MetaServiceUtils::sessionPrefix(); + auto& prefix = MetaKeyUtils::sessionPrefix(); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { handleErrorCode(nebula::error(ret)); @@ -122,7 +121,7 @@ void ListSessionsProcessor::process(const cpp2::ListSessionsReq&) { std::vector sessions; auto iter = nebula::value(ret).get(); while (iter->valid()) { - auto session = MetaServiceUtils::parseSessionVal(iter->val()); + auto session = MetaKeyUtils::parseSessionVal(iter->val()); VLOG(3) << "List session: " << session.get_session_id(); sessions.emplace_back(std::move(session)); iter->next(); @@ -138,7 +137,7 @@ void ListSessionsProcessor::process(const cpp2::ListSessionsReq&) { void GetSessionProcessor::process(const cpp2::GetSessionReq& req) { folly::SharedMutex::ReadHolder rHolder(LockUtils::sessionLock()); auto sessionId = req.get_session_id(); - auto sessionKey = MetaServiceUtils::sessionKey(sessionId); + auto sessionKey = MetaKeyUtils::sessionKey(sessionId); auto ret = doGet(sessionKey); if (!nebula::ok(ret)) { auto errCode = nebula::error(ret); @@ -151,7 +150,7 @@ void GetSessionProcessor::process(const cpp2::GetSessionReq& req) { return; } - auto session = MetaServiceUtils::parseSessionVal(nebula::value(ret)); + auto session = MetaKeyUtils::parseSessionVal(nebula::value(ret)); resp_.set_session(std::move(session)); handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); onFinished(); @@ -160,7 +159,7 @@ void GetSessionProcessor::process(const cpp2::GetSessionReq& req) { void RemoveSessionProcessor::process(const cpp2::RemoveSessionReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::sessionLock()); auto sessionId = req.get_session_id(); - auto sessionKey = MetaServiceUtils::sessionKey(sessionId); + auto sessionKey = MetaKeyUtils::sessionKey(sessionId); auto ret = doGet(sessionKey); if (!nebula::ok(ret)) { auto errCode = nebula::error(ret); @@ -184,7 +183,7 @@ void KillQueryProcessor::process(const cpp2::KillQueryReq& req) { std::vector data; for (auto& kv : killQueries) { auto sessionId = kv.first; - auto sessionKey = MetaServiceUtils::sessionKey(sessionId); + auto sessionKey = MetaKeyUtils::sessionKey(sessionId); auto ret = doGet(sessionKey); if (!nebula::ok(ret)) { auto errCode = nebula::error(ret); @@ -197,7 +196,7 @@ void KillQueryProcessor::process(const cpp2::KillQueryReq& req) { return; } - auto session = MetaServiceUtils::parseSessionVal(nebula::value(ret)); + auto session = MetaKeyUtils::parseSessionVal(nebula::value(ret)); for (auto& epId : kv.second) { auto query = session.queries_ref()->find(epId); if (query == session.queries_ref()->end()) { @@ -208,8 +207,7 @@ void KillQueryProcessor::process(const cpp2::KillQueryReq& req) { query->second.set_status(cpp2::QueryStatus::KILLING); } - data.emplace_back(MetaServiceUtils::sessionKey(sessionId), - MetaServiceUtils::sessionVal(session)); + data.emplace_back(MetaKeyUtils::sessionKey(sessionId), MetaKeyUtils::sessionVal(session)); } auto putRet = doSyncPut(std::move(data)); diff --git a/src/meta/processors/user/AuthenticationProcessor.cpp b/src/meta/processors/user/AuthenticationProcessor.cpp index 72acf23e0c0..22c36ec6820 100644 --- a/src/meta/processors/user/AuthenticationProcessor.cpp +++ b/src/meta/processors/user/AuthenticationProcessor.cpp @@ -33,7 +33,7 @@ void CreateUserProcessor::process(const cpp2::CreateUserReq& req) { } std::vector data; - data.emplace_back(MetaServiceUtils::userKey(account), MetaServiceUtils::userVal(password)); + data.emplace_back(MetaKeyUtils::userKey(account), MetaKeyUtils::userVal(password)); doSyncPutAndUpdate(std::move(data)); } @@ -41,8 +41,8 @@ void AlterUserProcessor::process(const cpp2::AlterUserReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::userLock()); const auto& account = req.get_account(); const auto& password = req.get_encoded_pwd(); - auto userKey = MetaServiceUtils::userKey(account); - auto userVal = MetaServiceUtils::userVal(password); + auto userKey = MetaKeyUtils::userKey(account); + auto userVal = MetaKeyUtils::userVal(password); auto iRet = doGet(userKey); if (!nebula::ok(iRet)) { @@ -84,10 +84,10 @@ void DropUserProcessor::process(const cpp2::DropUserReq& req) { } std::vector keys; - keys.emplace_back(MetaServiceUtils::userKey(account)); + keys.emplace_back(MetaKeyUtils::userKey(account)); // Collect related roles by user. - auto prefix = MetaServiceUtils::rolesPrefix(); + auto prefix = MetaKeyUtils::rolesPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { retCode = nebula::error(iterRet); @@ -102,7 +102,7 @@ void DropUserProcessor::process(const cpp2::DropUserReq& req) { auto iter = nebula::value(iterRet).get(); while (iter->valid()) { auto key = iter->key(); - auto user = MetaServiceUtils::parseRoleUser(key); + auto user = MetaKeyUtils::parseRoleUser(key); if (user == account) { keys.emplace_back(key); } @@ -139,8 +139,8 @@ void GrantProcessor::process(const cpp2::GrantRoleReq& req) { } std::vector data; - data.emplace_back(MetaServiceUtils::roleKey(spaceId, account), - MetaServiceUtils::roleVal(roleItem.get_role_type())); + data.emplace_back(MetaKeyUtils::roleKey(spaceId, account), + MetaKeyUtils::roleVal(roleItem.get_role_type())); doSyncPutAndUpdate(std::move(data)); } @@ -161,7 +161,7 @@ void RevokeProcessor::process(const cpp2::RevokeRoleReq& req) { return; } - auto roleKey = MetaServiceUtils::roleKey(spaceId, account); + auto roleKey = MetaKeyUtils::roleKey(spaceId, account); auto result = doGet(roleKey); if (!nebula::ok(result)) { userRet = nebula::error(result); @@ -220,8 +220,8 @@ void ChangePasswordProcessor::process(const cpp2::ChangePasswordReq& req) { } } - auto userKey = MetaServiceUtils::userKey(account); - auto userVal = MetaServiceUtils::userVal(req.get_new_encoded_pwd()); + auto userKey = MetaKeyUtils::userKey(account); + auto userVal = MetaKeyUtils::userVal(req.get_new_encoded_pwd()); std::vector data; data.emplace_back(std::move(userKey), std::move(userVal)); doSyncPutAndUpdate(std::move(data)); @@ -243,8 +243,8 @@ void ListUsersProcessor::process(const cpp2::ListUsersReq& req) { auto iter = nebula::value(ret).get(); std::unordered_map users; while (iter->valid()) { - auto account = MetaServiceUtils::parseUser(iter->key()); - auto password = MetaServiceUtils::parseUserPwd(iter->val()); + auto account = MetaKeyUtils::parseUser(iter->key()); + auto password = MetaKeyUtils::parseUserPwd(iter->val()); users.emplace(std::move(account), std::move(password)); iter->next(); } @@ -258,7 +258,7 @@ void ListRolesProcessor::process(const cpp2::ListRolesReq& req) { CHECK_SPACE_ID_AND_RETURN(spaceId); folly::SharedMutex::ReadHolder rHolder(LockUtils::userLock()); - auto prefix = MetaServiceUtils::roleSpacePrefix(spaceId); + auto prefix = MetaKeyUtils::roleSpacePrefix(spaceId); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -271,7 +271,7 @@ void ListRolesProcessor::process(const cpp2::ListRolesReq& req) { auto iter = nebula::value(ret).get(); std::vector roles; while (iter->valid()) { - auto account = MetaServiceUtils::parseRoleUser(iter->key()); + auto account = MetaKeyUtils::parseRoleUser(iter->key()); auto val = iter->val(); cpp2::RoleItem role; role.set_user_id(std::move(account)); @@ -289,7 +289,7 @@ void GetUserRolesProcessor::process(const cpp2::GetUserRolesReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::userLock()); const auto& act = req.get_account(); - auto prefix = MetaServiceUtils::rolesPrefix(); + auto prefix = MetaKeyUtils::rolesPrefix(); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -302,8 +302,8 @@ void GetUserRolesProcessor::process(const cpp2::GetUserRolesReq& req) { auto iter = nebula::value(ret).get(); std::vector roles; while (iter->valid()) { - auto account = MetaServiceUtils::parseRoleUser(iter->key()); - auto spaceId = MetaServiceUtils::parseRoleSpace(iter->key()); + auto account = MetaKeyUtils::parseRoleUser(iter->key()); + auto spaceId = MetaKeyUtils::parseRoleSpace(iter->key()); if (account == act) { auto val = iter->val(); cpp2::RoleItem role; diff --git a/src/meta/processors/zone/AddGroupProcessor.cpp b/src/meta/processors/zone/AddGroupProcessor.cpp index 54be9621d53..8d81583ade0 100644 --- a/src/meta/processors/zone/AddGroupProcessor.cpp +++ b/src/meta/processors/zone/AddGroupProcessor.cpp @@ -29,7 +29,7 @@ void AddGroupProcessor::process(const cpp2::AddGroupReq& req) { } // check the zone existed - const auto& prefix = MetaServiceUtils::zonePrefix(); + const auto& prefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -42,7 +42,7 @@ void AddGroupProcessor::process(const cpp2::AddGroupReq& req) { auto iter = nebula::value(iterRet).get(); std::vector zones; while (iter->valid()) { - auto zoneName = MetaServiceUtils::parseZoneName(iter->key()); + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); zones.emplace_back(std::move(zoneName)); iter->next(); } @@ -90,16 +90,16 @@ void AddGroupProcessor::process(const cpp2::AddGroupReq& req) { std::vector data; auto groupId = nebula::value(groupIdRet); - data.emplace_back(MetaServiceUtils::indexGroupKey(groupName), + data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaServiceUtils::groupKey(groupName), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); LOG(INFO) << "Create Group: " << groupName; doSyncPutAndUpdate(std::move(data)); } nebula::cpp2::ErrorCode AddGroupProcessor::checkGroupRedundancy(std::vector zones) { - const auto& prefix = MetaServiceUtils::groupPrefix(); + const auto& prefix = MetaKeyUtils::groupPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -110,8 +110,8 @@ nebula::cpp2::ErrorCode AddGroupProcessor::checkGroupRedundancy(std::vectorvalid()) { - auto groupName = MetaServiceUtils::parseGroupName(iter->key()); - auto zoneNames = MetaServiceUtils::parseZoneNames(iter->val()); + auto groupName = MetaKeyUtils::parseGroupName(iter->key()); + auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); std::sort(zoneNames.begin(), zoneNames.end()); if (zones == zoneNames) { LOG(ERROR) << "Group " << groupName diff --git a/src/meta/processors/zone/AddZoneProcessor.cpp b/src/meta/processors/zone/AddZoneProcessor.cpp index 8d55b0a5e08..6de794262e0 100644 --- a/src/meta/processors/zone/AddZoneProcessor.cpp +++ b/src/meta/processors/zone/AddZoneProcessor.cpp @@ -83,16 +83,16 @@ void AddZoneProcessor::process(const cpp2::AddZoneReq& req) { auto zoneId = nebula::value(zoneRet); std::vector data; - data.emplace_back(MetaServiceUtils::indexZoneKey(zoneName), + data.emplace_back(MetaKeyUtils::indexZoneKey(zoneName), std::string(reinterpret_cast(&zoneId), sizeof(ZoneID))); - data.emplace_back(MetaServiceUtils::zoneKey(zoneName), MetaServiceUtils::zoneVal(nodes)); + data.emplace_back(MetaKeyUtils::zoneKey(zoneName), MetaKeyUtils::zoneVal(nodes)); LOG(INFO) << "Create Zone: " << zoneName; doSyncPutAndUpdate(std::move(data)); } nebula::cpp2::ErrorCode AddZoneProcessor::checkHostNotOverlap(const std::vector& nodes) { - const auto& prefix = MetaServiceUtils::zonePrefix(); + const auto& prefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -102,8 +102,8 @@ nebula::cpp2::ErrorCode AddZoneProcessor::checkHostNotOverlap(const std::vector< auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto zoneName = MetaServiceUtils::parseZoneName(iter->key()); - auto hosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); for (auto& node : nodes) { auto hostIter = std::find(hosts.begin(), hosts.end(), node); if (hostIter != hosts.end()) { diff --git a/src/meta/processors/zone/DropGroupProcessor.cpp b/src/meta/processors/zone/DropGroupProcessor.cpp index c578597d69d..67f5179411f 100644 --- a/src/meta/processors/zone/DropGroupProcessor.cpp +++ b/src/meta/processors/zone/DropGroupProcessor.cpp @@ -34,14 +34,14 @@ void DropGroupProcessor::process(const cpp2::DropGroupReq& req) { } std::vector keys; - keys.emplace_back(MetaServiceUtils::indexGroupKey(groupName)); - keys.emplace_back(MetaServiceUtils::groupKey(groupName)); + keys.emplace_back(MetaKeyUtils::indexGroupKey(groupName)); + keys.emplace_back(MetaKeyUtils::groupKey(groupName)); LOG(INFO) << "Drop Group: " << groupName; doSyncMultiRemoveAndUpdate(std::move(keys)); } nebula::cpp2::ErrorCode DropGroupProcessor::checkSpaceDependency(const std::string& groupName) { - const auto& prefix = MetaServiceUtils::spacePrefix(); + const auto& prefix = MetaKeyUtils::spacePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -51,7 +51,7 @@ nebula::cpp2::ErrorCode DropGroupProcessor::checkSpaceDependency(const std::stri auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto properties = MetaServiceUtils::parseSpace(iter->val()); + auto properties = MetaKeyUtils::parseSpace(iter->val()); if (properties.group_name_ref().has_value() && *properties.group_name_ref() == groupName) { LOG(ERROR) << "Space " << properties.get_space_name() << " is bind to the group " << groupName; diff --git a/src/meta/processors/zone/DropZoneProcessor.cpp b/src/meta/processors/zone/DropZoneProcessor.cpp index 09dbe1ab104..fd5de69869f 100644 --- a/src/meta/processors/zone/DropZoneProcessor.cpp +++ b/src/meta/processors/zone/DropZoneProcessor.cpp @@ -34,14 +34,14 @@ void DropZoneProcessor::process(const cpp2::DropZoneReq& req) { } std::vector keys; - keys.emplace_back(MetaServiceUtils::indexZoneKey(zoneName)); - keys.emplace_back(MetaServiceUtils::zoneKey(zoneName)); + keys.emplace_back(MetaKeyUtils::indexZoneKey(zoneName)); + keys.emplace_back(MetaKeyUtils::zoneKey(zoneName)); LOG(INFO) << "Drop Zone " << zoneName; doSyncMultiRemoveAndUpdate(std::move(keys)); } nebula::cpp2::ErrorCode DropZoneProcessor::checkGroupDependency(const std::string& zoneName) { - const auto& prefix = MetaServiceUtils::groupPrefix(); + const auto& prefix = MetaKeyUtils::groupPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -51,10 +51,10 @@ nebula::cpp2::ErrorCode DropZoneProcessor::checkGroupDependency(const std::strin auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto zoneNames = MetaServiceUtils::parseZoneNames(iter->val()); + auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); auto zoneIter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); if (zoneIter != zoneNames.end()) { - auto groupName = MetaServiceUtils::parseGroupName(iter->key()); + auto groupName = MetaKeyUtils::parseGroupName(iter->key()); LOG(ERROR) << "Zone " << zoneName << " is belong to Group " << groupName; return nebula::cpp2::ErrorCode::E_NOT_DROP; } diff --git a/src/meta/processors/zone/GetGroupProcessor.cpp b/src/meta/processors/zone/GetGroupProcessor.cpp index 4e4e3df0032..cc21aa523a4 100644 --- a/src/meta/processors/zone/GetGroupProcessor.cpp +++ b/src/meta/processors/zone/GetGroupProcessor.cpp @@ -25,7 +25,7 @@ void GetGroupProcessor::process(const cpp2::GetGroupReq& req) { return; } - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); auto groupValueRet = doGet(std::move(groupKey)); if (!nebula::ok(groupValueRet)) { auto retCode = nebula::error(groupValueRet); @@ -36,7 +36,7 @@ void GetGroupProcessor::process(const cpp2::GetGroupReq& req) { return; } - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); LOG(INFO) << "Get Group: " << groupName << " zone size: " << zoneNames.size(); resp_.set_zone_names(std::move(zoneNames)); handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); diff --git a/src/meta/processors/zone/GetZoneProcessor.cpp b/src/meta/processors/zone/GetZoneProcessor.cpp index ed8835636e3..93a9c0b9549 100644 --- a/src/meta/processors/zone/GetZoneProcessor.cpp +++ b/src/meta/processors/zone/GetZoneProcessor.cpp @@ -25,7 +25,7 @@ void GetZoneProcessor::process(const cpp2::GetZoneReq& req) { return; } - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); auto zoneValueRet = doGet(std::move(zoneKey)); if (!nebula::ok(zoneValueRet)) { auto retCode = nebula::error(zoneValueRet); @@ -39,7 +39,7 @@ void GetZoneProcessor::process(const cpp2::GetZoneReq& req) { return; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); LOG(INFO) << "Get Zone: " << zoneName << " node size: " << hosts.size(); resp_.set_hosts(std::move(hosts)); handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED); diff --git a/src/meta/processors/zone/ListGroupsProcessor.cpp b/src/meta/processors/zone/ListGroupsProcessor.cpp index f524685d2be..63cc00fd468 100644 --- a/src/meta/processors/zone/ListGroupsProcessor.cpp +++ b/src/meta/processors/zone/ListGroupsProcessor.cpp @@ -11,7 +11,7 @@ namespace meta { void ListGroupsProcessor::process(const cpp2::ListGroupsReq&) { folly::SharedMutex::ReadHolder rHolder(LockUtils::groupLock()); - const auto& prefix = MetaServiceUtils::groupPrefix(); + const auto& prefix = MetaKeyUtils::groupPrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -24,8 +24,8 @@ void ListGroupsProcessor::process(const cpp2::ListGroupsReq&) { std::vector groups; while (iter->valid()) { - auto groupName = MetaServiceUtils::parseGroupName(iter->key()); - auto zoneNames = MetaServiceUtils::parseZoneNames(iter->val()); + auto groupName = MetaKeyUtils::parseGroupName(iter->key()); + auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); cpp2::Group group; group.set_group_name(std::move(groupName)); group.set_zone_names(std::move(zoneNames)); diff --git a/src/meta/processors/zone/ListZonesProcessor.cpp b/src/meta/processors/zone/ListZonesProcessor.cpp index c5e3e17bdd6..fbbff2740a8 100644 --- a/src/meta/processors/zone/ListZonesProcessor.cpp +++ b/src/meta/processors/zone/ListZonesProcessor.cpp @@ -11,7 +11,7 @@ namespace meta { void ListZonesProcessor::process(const cpp2::ListZonesReq&) { folly::SharedMutex::ReadHolder rHolder(LockUtils::zoneLock()); - const auto& prefix = MetaServiceUtils::zonePrefix(); + const auto& prefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -24,8 +24,8 @@ void ListZonesProcessor::process(const cpp2::ListZonesReq&) { std::vector zones; while (iter->valid()) { - auto zoneName = MetaServiceUtils::parseZoneName(iter->key()); - auto hosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); cpp2::Zone zone; zone.set_zone_name(std::move(zoneName)); if (hosts.size() != 0) { diff --git a/src/meta/processors/zone/UpdateGroupProcessor.cpp b/src/meta/processors/zone/UpdateGroupProcessor.cpp index 441d1106a93..d2289c58332 100644 --- a/src/meta/processors/zone/UpdateGroupProcessor.cpp +++ b/src/meta/processors/zone/UpdateGroupProcessor.cpp @@ -22,7 +22,7 @@ void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { return; } - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); auto groupValueRet = doGet(std::move(groupKey)); if (!nebula::ok(groupValueRet)) { auto retCode = nebula::error(groupValueRet); @@ -37,7 +37,7 @@ void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { } auto zoneName = req.get_zone_name(); - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); if (iter != zoneNames.end()) { LOG(ERROR) << "Zone " << zoneName << " already exist in the group " << groupName; @@ -46,7 +46,7 @@ void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { return; } - const auto& zonePrefix = MetaServiceUtils::zonePrefix(); + const auto& zonePrefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(zonePrefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -59,7 +59,7 @@ void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { bool found = false; while (zoneIter->valid()) { - auto name = MetaServiceUtils::parseZoneName(zoneIter->key()); + auto name = MetaKeyUtils::parseZoneName(zoneIter->key()); if (name == zoneName) { found = true; break; @@ -76,7 +76,7 @@ void AddZoneIntoGroupProcessor::process(const cpp2::AddZoneIntoGroupReq& req) { zoneNames.emplace_back(zoneName); std::vector data; - data.emplace_back(std::move(groupKey), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); LOG(INFO) << "Add Zone " << zoneName << " Into Group " << groupName; doSyncPutAndUpdate(std::move(data)); } @@ -94,7 +94,7 @@ void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) return; } - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); auto groupValueRet = doGet(groupKey); if (!nebula::ok(groupValueRet)) { auto retCode = nebula::error(groupValueRet); @@ -109,7 +109,7 @@ void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) } auto zoneName = req.get_zone_name(); - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(nebula::value(groupValueRet))); auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); if (iter == zoneNames.end()) { LOG(ERROR) << "Zone " << zoneName << " not exist in the group " << groupName; @@ -118,7 +118,7 @@ void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) return; } - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto spaceRet = doPrefix(spacePrefix); if (!nebula::ok(spaceRet)) { auto retCode = nebula::error(spaceRet); @@ -131,7 +131,7 @@ void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) nebula::cpp2::ErrorCode spaceCode = nebula::cpp2::ErrorCode::SUCCEEDED; auto spaceIter = nebula::value(spaceRet).get(); while (spaceIter->valid()) { - auto properties = MetaServiceUtils::parseSpace(spaceIter->val()); + auto properties = MetaKeyUtils::parseSpace(spaceIter->val()); if (properties.group_name_ref().has_value() && *properties.group_name_ref() == groupName) { LOG(ERROR) << "Space is bind to the group " << *properties.group_name_ref(); spaceCode = nebula::cpp2::ErrorCode::E_CONFLICT; @@ -147,7 +147,7 @@ void DropZoneFromGroupProcessor::process(const cpp2::DropZoneFromGroupReq& req) zoneNames.erase(iter); std::vector data; - data.emplace_back(std::move(groupKey), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); LOG(INFO) << "Drop Zone " << zoneName << " From Group " << groupName; doSyncPutAndUpdate(std::move(data)); } diff --git a/src/meta/processors/zone/UpdateZoneProcessor.cpp b/src/meta/processors/zone/UpdateZoneProcessor.cpp index e6982a006ac..2ca9bd31e64 100644 --- a/src/meta/processors/zone/UpdateZoneProcessor.cpp +++ b/src/meta/processors/zone/UpdateZoneProcessor.cpp @@ -24,7 +24,7 @@ void AddHostIntoZoneProcessor::process(const cpp2::AddHostIntoZoneReq& req) { return; } - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); auto zoneValueRet = doGet(std::move(zoneKey)); if (!nebula::ok(zoneValueRet)) { auto retCode = nebula::error(zoneValueRet); @@ -38,10 +38,10 @@ void AddHostIntoZoneProcessor::process(const cpp2::AddHostIntoZoneReq& req) { return; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); auto host = req.get_node(); // check this host not exist in all zones - const auto& prefix = MetaServiceUtils::zonePrefix(); + const auto& prefix = MetaKeyUtils::zonePrefix(); auto iterRet = doPrefix(prefix); if (!nebula::ok(iterRet)) { auto retCode = nebula::error(iterRet); @@ -53,8 +53,8 @@ void AddHostIntoZoneProcessor::process(const cpp2::AddHostIntoZoneReq& req) { auto iter = nebula::value(iterRet).get(); while (iter->valid()) { - auto name = MetaServiceUtils::parseZoneName(iter->key()); - auto zoneHosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto name = MetaKeyUtils::parseZoneName(iter->key()); + auto zoneHosts = MetaKeyUtils::parseZoneHosts(iter->val()); auto hostIter = std::find(zoneHosts.begin(), zoneHosts.end(), host); if (hostIter != zoneHosts.end()) { LOG(ERROR) << "Host overlap found in zone " << name; @@ -85,7 +85,7 @@ void AddHostIntoZoneProcessor::process(const cpp2::AddHostIntoZoneReq& req) { hosts.emplace_back(host); std::vector data; - data.emplace_back(std::move(zoneKey), MetaServiceUtils::zoneVal(std::move(hosts))); + data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(hosts))); LOG(INFO) << "Add Host " << host << " Into Zone " << zoneName; doSyncPutAndUpdate(std::move(data)); } @@ -103,7 +103,7 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { return; } - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); auto zoneValueRet = doGet(std::move(zoneKey)); if (!nebula::ok(zoneValueRet)) { auto retCode = nebula::error(zoneValueRet); @@ -117,12 +117,12 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { return; } - const auto& spacePrefix = MetaServiceUtils::spacePrefix(); + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto spaceIterRet = doPrefix(spacePrefix); auto spaceIter = nebula::value(spaceIterRet).get(); while (spaceIter->valid()) { - auto spaceId = MetaServiceUtils::spaceId(spaceIter->key()); - auto spaceKey = MetaServiceUtils::spaceKey(spaceId); + auto spaceId = MetaKeyUtils::spaceId(spaceIter->key()); + auto spaceKey = MetaKeyUtils::spaceKey(spaceId); auto ret = doGet(spaceKey); if (!nebula::ok(ret)) { auto retCode = nebula::error(ret); @@ -133,17 +133,17 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { return; } - auto properties = MetaServiceUtils::parseSpace(nebula::value(ret)); + auto properties = MetaKeyUtils::parseSpace(nebula::value(ret)); if (!properties.group_name_ref().has_value()) { spaceIter->next(); continue; } - const auto& partPrefix = MetaServiceUtils::partPrefix(spaceId); + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto partIterRet = doPrefix(partPrefix); auto partIter = nebula::value(partIterRet).get(); while (partIter->valid()) { - auto partHosts = MetaServiceUtils::parsePartVal(partIter->val()); + auto partHosts = MetaKeyUtils::parsePartVal(partIter->val()); for (auto& h : partHosts) { if (h == req.get_node()) { LOG(ERROR) << h << " is related with partition"; @@ -157,7 +157,7 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { spaceIter->next(); } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); auto host = req.get_node(); auto iter = std::find(hosts.begin(), hosts.end(), host); if (iter == hosts.end()) { @@ -169,7 +169,7 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { hosts.erase(iter); std::vector data; - data.emplace_back(std::move(zoneKey), MetaServiceUtils::zoneVal(std::move(hosts))); + data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(hosts))); LOG(INFO) << "Drop Host " << host << " From Zone " << zoneName; doSyncPutAndUpdate(std::move(data)); } diff --git a/src/meta/test/ActiveHostsManTest.cpp b/src/meta/test/ActiveHostsManTest.cpp index 01f92fd20f9..1514fd56c23 100644 --- a/src/meta/test/ActiveHostsManTest.cpp +++ b/src/meta/test/ActiveHostsManTest.cpp @@ -69,13 +69,13 @@ TEST(ActiveHostsManTest, NormalTest) { ASSERT_EQ(3, nebula::value(hostsRet).size()); { - const auto& prefix = MetaServiceUtils::hostPrefix(); + const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); int i = 0; while (iter->valid()) { - auto host = MetaServiceUtils::parseHostKey(iter->key()); + auto host = MetaKeyUtils::parseHostKey(iter->key()); HostInfo info = HostInfo::decode(iter->val()); ASSERT_EQ(HostAddr("0", i), HostAddr(host.host, host.port)); if (i != 0) { @@ -148,14 +148,14 @@ TEST(ActiveHostsManTest, LeaderTest) { nebula::cpp2::ErrorCode code; std::map results; { - const auto& prefix = MetaServiceUtils::leaderPrefix(); + const auto& prefix = MetaKeyUtils::leaderPrefix(); std::unique_ptr iter; auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); int i = 0; while (iter->valid()) { - auto spaceAndPart = MetaServiceUtils::parseLeaderKeyV3(iter->key()); - std::tie(host, term, code) = MetaServiceUtils::parseLeaderValV3(iter->val()); + auto spaceAndPart = MetaKeyUtils::parseLeaderKeyV3(iter->key()); + std::tie(host, term, code) = MetaKeyUtils::parseLeaderValV3(iter->val()); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(INFO) << "error: " << apache::thrift::util::enumNameSafe(code); continue; @@ -194,7 +194,7 @@ TEST(LastUpdateTimeManTest, NormalTest) { LastUpdateTimeMan::update(kv.get(), now - 100); { - auto key = MetaServiceUtils::lastUpdateTimeKey(); + auto key = MetaKeyUtils::lastUpdateTimeKey(); std::string val; auto ret = kv->get(kDefaultSpaceId, kDefaultPartId, key, &val); ASSERT_EQ(ret, nebula::cpp2::ErrorCode::SUCCEEDED); diff --git a/src/meta/test/AdminClientTest.cpp b/src/meta/test/AdminClientTest.cpp index d7dfa7423a7..0625d86079d 100644 --- a/src/meta/test/AdminClientTest.cpp +++ b/src/meta/test/AdminClientTest.cpp @@ -216,7 +216,7 @@ TEST(AdminClientTest, RetryTest) { thriftPeers.emplace_back(Utils::getStoreAddrFromAdminAddr({localIp, rpcServer1->port_})); std::vector data; - data.emplace_back(MetaServiceUtils::partKey(0, 1), MetaServiceUtils::partVal(thriftPeers)); + data.emplace_back(MetaKeyUtils::partKey(0, 1), MetaKeyUtils::partVal(thriftPeers)); folly::Baton baton; kv->asyncMultiPut( kDefaultSpaceId, kDefaultPartId, std::move(data), [&baton](nebula::cpp2::ErrorCode code) { diff --git a/src/meta/test/AuthProcessorTest.cpp b/src/meta/test/AuthProcessorTest.cpp index e2262ee6f21..c7a65164106 100644 --- a/src/meta/test/AuthProcessorTest.cpp +++ b/src/meta/test/AuthProcessorTest.cpp @@ -34,7 +34,7 @@ TEST(AuthProcessorTest, CreateUserTest) { std::string userVal; std::unique_ptr iter; auto retCode = - kv->get(kDefaultSpaceId, kDefaultPartId, MetaServiceUtils::userKey("user1"), &userVal); + kv->get(kDefaultSpaceId, kDefaultPartId, MetaKeyUtils::userKey("user1"), &userVal); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); } // Test user exists and param 'if_not_exists' == false; @@ -154,7 +154,7 @@ TEST(AuthProcessorTest, DropUserTest) { std::string userVal; std::unique_ptr iter; auto retCode = - kv->get(kDefaultSpaceId, kDefaultPartId, MetaServiceUtils::userKey("user1"), &userVal); + kv->get(kDefaultSpaceId, kDefaultPartId, MetaKeyUtils::userKey("user1"), &userVal); ASSERT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, retCode); } } @@ -523,7 +523,7 @@ TEST(AuthProcessorTest, GrantRevokeTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); // check role deleted after drop space - auto rolePrefix = MetaServiceUtils::roleSpacePrefix(space1); + auto rolePrefix = MetaKeyUtils::roleSpacePrefix(space1); std::unique_ptr roleIter; auto roleRet = kv->prefix(kDefaultSpaceId, kDefaultPartId, rolePrefix, &roleIter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, roleRet); diff --git a/src/meta/test/BalancerTest.cpp b/src/meta/test/BalancerTest.cpp index 7f6b5fbece2..593e2339a17 100644 --- a/src/meta/test/BalancerTest.cpp +++ b/src/meta/test/BalancerTest.cpp @@ -89,7 +89,7 @@ TEST(BalanceTest, BalanceTaskTest) { } void showHostLoading(kvstore::KVStore* kv, GraphSpaceID spaceId) { - auto prefix = MetaServiceUtils::partPrefix(spaceId); + auto prefix = MetaKeyUtils::partPrefix(spaceId); std::unique_ptr iter; auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); @@ -98,7 +98,7 @@ void showHostLoading(kvstore::KVStore* kv, GraphSpaceID spaceId) { auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - auto hs = MetaServiceUtils::parsePartVal(iter->val()); + auto hs = MetaKeyUtils::parsePartVal(iter->val()); for (auto h : hs) { hostPart[h].emplace_back(partId); } @@ -115,7 +115,7 @@ void showHostLoading(kvstore::KVStore* kv, GraphSpaceID spaceId) { } HostParts assignHostParts(kvstore::KVStore* kv, GraphSpaceID spaceId) { - auto prefix = MetaServiceUtils::partPrefix(spaceId); + auto prefix = MetaKeyUtils::partPrefix(spaceId); std::unique_ptr iter; kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); HostParts hostPart; @@ -123,7 +123,7 @@ HostParts assignHostParts(kvstore::KVStore* kv, GraphSpaceID spaceId) { auto key = iter->key(); PartitionID partId; memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - auto hs = MetaServiceUtils::parsePartVal(iter->val()); + auto hs = MetaKeyUtils::parsePartVal(iter->val()); for (auto h : hs) { hostPart[h].emplace_back(partId); } @@ -893,14 +893,14 @@ TEST(BalanceTest, BalancePlanTest) { } int32_t verifyBalancePlan(kvstore::KVStore* kv, BalanceID balanceId, BalanceStatus balanceStatus) { - const auto& prefix = MetaServiceUtils::balancePlanPrefix(); + const auto& prefix = MetaKeyUtils::balancePlanPrefix(); std::unique_ptr iter; auto retcode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); EXPECT_EQ(retcode, nebula::cpp2::ErrorCode::SUCCEEDED); int32_t num = 0; while (iter->valid()) { - auto id = MetaServiceUtils::parseBalanceID(iter->key()); - auto status = MetaServiceUtils::parseBalanceStatus(iter->val()); + auto id = MetaKeyUtils::parseBalanceID(iter->key()); + auto status = MetaKeyUtils::parseBalanceStatus(iter->val()); EXPECT_EQ(balanceId, id); EXPECT_EQ(balanceStatus, status); num++; @@ -915,18 +915,18 @@ void verifyBalanceTask(kvstore::KVStore* kv, BalanceTaskResult result, std::unordered_map& partCount, int32_t exceptNumber = 0) { - const auto& prefix = MetaServiceUtils::balanceTaskPrefix(balanceId); + const auto& prefix = MetaKeyUtils::balanceTaskPrefix(balanceId); std::unique_ptr iter; auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(code, nebula::cpp2::ErrorCode::SUCCEEDED); int32_t num = 0; while (iter->valid()) { - auto keyTuple = MetaServiceUtils::parseBalanceTaskKey(iter->key()); + auto keyTuple = MetaKeyUtils::parseBalanceTaskKey(iter->key()); ASSERT_EQ(balanceId, std::get<0>(keyTuple)); ASSERT_EQ(1, std::get<1>(keyTuple)); partCount[std::get<3>(keyTuple)]--; partCount[std::get<4>(keyTuple)]++; - auto valueTuple = MetaServiceUtils::parseBalanceTaskVal(iter->val()); + auto valueTuple = MetaKeyUtils::parseBalanceTaskVal(iter->val()); ASSERT_EQ(status, std::get<0>(valueTuple)); ASSERT_EQ(result, std::get<1>(valueTuple)); ASSERT_LT(0, std::get<2>(valueTuple)); @@ -1252,7 +1252,7 @@ TEST(BalanceTest, StopPlanTest) { // wait until the only IN_PROGRESS task finished; sleep(3); { - const auto& prefix = MetaServiceUtils::balanceTaskPrefix(balanceId); + const auto& prefix = MetaKeyUtils::balanceTaskPrefix(balanceId); std::unique_ptr iter; auto retcode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(retcode, nebula::cpp2::ErrorCode::SUCCEEDED); @@ -1263,7 +1263,7 @@ TEST(BalanceTest, StopPlanTest) { // PartitionID partId = // std::get<2>(BalanceTask::MetaServiceUtils(iter->key())); { - auto tup = MetaServiceUtils::parseBalanceTaskVal(iter->val()); + auto tup = MetaKeyUtils::parseBalanceTaskVal(iter->val()); task.status_ = std::get<0>(tup); task.ret_ = std::get<1>(tup); task.startTimeMs_ = std::get<2>(tup); @@ -1325,7 +1325,7 @@ TEST(BalanceTest, CleanLastInvalidBalancePlanTest) { ASSERT_EQ(value(cleanRet), balanceId); { - const auto& prefix = MetaServiceUtils::balancePlanPrefix(); + const auto& prefix = MetaKeyUtils::balancePlanPrefix(); std::unique_ptr iter; auto retcode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(retcode, nebula::cpp2::ErrorCode::SUCCEEDED); diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index bcedc7f01d2..7d4f83988c6 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -3,31 +3,6 @@ # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -nebula_add_test( - NAME - meta_utils_test - SOURCES - MetaServiceUtilsTest.cpp - ../MetaServiceUtils.cpp - OBJECTS - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - LIBRARIES - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - gtest -) - nebula_add_test( NAME processor_test diff --git a/src/meta/test/CreateBackupProcessorTest.cpp b/src/meta/test/CreateBackupProcessorTest.cpp index adc97905693..accf48ee4db 100644 --- a/src/meta/test/CreateBackupProcessorTest.cpp +++ b/src/meta/test/CreateBackupProcessorTest.cpp @@ -106,21 +106,21 @@ TEST(ProcessorTest, CreateBackupTest) { properties.set_space_name("test_space"); properties.set_partition_num(1); properties.set_replica_factor(1); - auto spaceVal = MetaServiceUtils::spaceVal(properties); + auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space"), std::string(reinterpret_cast(&id), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id), MetaServiceUtils::spaceVal(properties)); + data.emplace_back(MetaKeyUtils::spaceKey(id), MetaKeyUtils::spaceVal(properties)); cpp2::SpaceDesc properties2; GraphSpaceID id2 = 2; properties2.set_space_name("test_space2"); properties2.set_partition_num(1); properties2.set_replica_factor(1); - spaceVal = MetaServiceUtils::spaceVal(properties2); - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space2"), + spaceVal = MetaKeyUtils::spaceVal(properties2); + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space2"), std::string(reinterpret_cast(&id), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id2), MetaServiceUtils::spaceVal(properties2)); + data.emplace_back(MetaKeyUtils::spaceKey(id2), MetaKeyUtils::spaceVal(properties2)); std::string indexName = "test_space_index"; int32_t tagIndex = 2; @@ -134,9 +134,9 @@ TEST(ProcessorTest, CreateBackupTest) { schemaID.set_tag_id(tagID); item.set_schema_id(schemaID); item.set_schema_name(tagName); - data.emplace_back(MetaServiceUtils::indexIndexKey(id, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(id, indexName), std::string(reinterpret_cast(&tagIndex), sizeof(IndexID))); - data.emplace_back(MetaServiceUtils::indexKey(id, tagIndex), MetaServiceUtils::indexVal(item)); + data.emplace_back(MetaKeyUtils::indexKey(id, tagIndex), MetaKeyUtils::indexVal(item)); std::vector allHosts; allHosts.emplace_back(storageHost); @@ -147,8 +147,8 @@ TEST(ProcessorTest, CreateBackupTest) { for (int32_t i = 0; i < 1; i++, idx++) { hosts2.emplace_back(allHosts[idx % 1]); } - data.emplace_back(MetaServiceUtils::partKey(id, partId), MetaServiceUtils::partVal(hosts2)); - data.emplace_back(MetaServiceUtils::partKey(id2, partId), MetaServiceUtils::partVal(hosts2)); + data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts2)); + data.emplace_back(MetaKeyUtils::partKey(id2, partId), MetaKeyUtils::partVal(hosts2)); } folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { diff --git a/src/meta/test/GetStatsTest.cpp b/src/meta/test/GetStatsTest.cpp index d0565de00be..c25088e2f09 100644 --- a/src/meta/test/GetStatsTest.cpp +++ b/src/meta/test/GetStatsTest.cpp @@ -29,7 +29,7 @@ using ::testing::Return; using ::testing::SetArgPointee; std::string toTempKey(int32_t space, int32_t jobId) { - std::string key = MetaServiceUtils::statsKey(space); + std::string key = MetaKeyUtils::statsKey(space); return key.append(reinterpret_cast(&jobId), sizeof(int32_t)); } @@ -48,7 +48,7 @@ void copyData(kvstore::KVStore* kv, } void genTempData(int32_t spaceId, int jobId, kvstore::KVStore* kv) { - auto statsKey = MetaServiceUtils::statsKey(spaceId); + auto statsKey = MetaKeyUtils::statsKey(spaceId); auto tempKey = toTempKey(spaceId, jobId); copyData(kv, 0, 0, statsKey, tempKey); } @@ -148,7 +148,7 @@ TEST_F(GetStatsTest, StatsJob) { ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); // Directly find statis data in kvstore, statis data does not exist. - auto key = MetaServiceUtils::statsKey(spaceId); + auto key = MetaKeyUtils::statsKey(spaceId); std::string val; auto ret = kv_->get(kDefaultSpaceId, kDefaultPartId, key, &val); ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, ret); @@ -170,7 +170,7 @@ TEST_F(GetStatsTest, StatsJob) { statisJob.setStatus(cpp2::JobStatus::FINISHED); jobMgr->save(statisJob.jobKey(), statisJob.jobVal()); auto jobId = statisJob.getJobId(); - auto statsKey = MetaServiceUtils::statsKey(spaceId); + auto statsKey = MetaKeyUtils::statsKey(spaceId); auto tempKey = toTempKey(spaceId, jobId); copyData(kv_.get(), 0, 0, statsKey, tempKey); @@ -201,12 +201,12 @@ TEST_F(GetStatsTest, StatsJob) { ASSERT_EQ(0, statsItem.get_space_edges()); // Directly find statis data in kvstore, statis data exists. - auto key = MetaServiceUtils::statsKey(spaceId); + auto key = MetaKeyUtils::statsKey(spaceId); std::string val; auto ret = kv_->get(kDefaultSpaceId, kDefaultPartId, key, &val); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); - auto statsItem1 = MetaServiceUtils::parseStatsVal(val); + auto statsItem1 = MetaKeyUtils::parseStatsVal(val); ASSERT_EQ(cpp2::JobStatus::FINISHED, statsItem1.get_status()); ASSERT_EQ(0, statsItem1.get_tag_vertices().size()); ASSERT_EQ(0, statsItem1.get_edges().size()); @@ -246,12 +246,12 @@ TEST_F(GetStatsTest, StatsJob) { ASSERT_EQ(0, statsItem.get_space_edges()); // Directly find statis data in kvstore, statis data exists. - auto key = MetaServiceUtils::statsKey(spaceId); + auto key = MetaKeyUtils::statsKey(spaceId); std::string val; auto ret = kv_->get(kDefaultSpaceId, kDefaultPartId, key, &val); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); - auto statsItem1 = MetaServiceUtils::parseStatsVal(val); + auto statsItem1 = MetaKeyUtils::parseStatsVal(val); ASSERT_EQ(cpp2::JobStatus::FINISHED, statsItem1.get_status()); ASSERT_EQ(0, statsItem1.get_tag_vertices().size()); ASSERT_EQ(0, statsItem1.get_edges().size()); @@ -266,7 +266,7 @@ TEST_F(GetStatsTest, StatsJob) { // Remove statis data. { - auto key = MetaServiceUtils::statsKey(spaceId); + auto key = MetaKeyUtils::statsKey(spaceId); folly::Baton baton; auto retCode = nebula::cpp2::ErrorCode::SUCCEEDED; kv_->asyncRemove(kDefaultSpaceId, kDefaultPartId, key, [&](nebula::cpp2::ErrorCode code) { @@ -300,7 +300,7 @@ TEST_F(GetStatsTest, StatsJob) { auto result2 = jobMgr->runJobInternal(statisJob2); auto jobId2 = statisJob2.getJobId(); - auto statsKey2 = MetaServiceUtils::statsKey(spaceId); + auto statsKey2 = MetaKeyUtils::statsKey(spaceId); auto tempKey2 = toTempKey(spaceId, jobId2); copyData(kv_.get(), 0, 0, statsKey2, tempKey2); @@ -335,12 +335,12 @@ TEST_F(GetStatsTest, StatsJob) { ASSERT_EQ(0, statsItem.get_space_edges()); // Directly find statis data in kvstore, statis data exists. - auto key = MetaServiceUtils::statsKey(spaceId); + auto key = MetaKeyUtils::statsKey(spaceId); std::string val; auto ret = kv_->get(kDefaultSpaceId, kDefaultPartId, key, &val); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); - auto statsItem1 = MetaServiceUtils::parseStatsVal(val); + auto statsItem1 = MetaKeyUtils::parseStatsVal(val); ASSERT_EQ(cpp2::JobStatus::FINISHED, statsItem1.get_status()); ASSERT_EQ(0, statsItem1.get_tag_vertices().size()); ASSERT_EQ(0, statsItem1.get_edges().size()); diff --git a/src/meta/test/IndexProcessorTest.cpp b/src/meta/test/IndexProcessorTest.cpp index 655ad2db3c6..2289b136f3a 100644 --- a/src/meta/test/IndexProcessorTest.cpp +++ b/src/meta/test/IndexProcessorTest.cpp @@ -1577,14 +1577,14 @@ void mockSchemas(kvstore::KVStore* kv) { (*srcsch.columns_ref()).emplace_back(std::move(col)); } auto tagIdVal = std::string(reinterpret_cast(&tagId), sizeof(tagId)); - schemas.emplace_back(MetaServiceUtils::indexTagKey(1, "test_tag"), tagIdVal); - schemas.emplace_back(MetaServiceUtils::schemaTagKey(1, tagId, ver), - MetaServiceUtils::schemaVal("test_tag", srcsch)); + schemas.emplace_back(MetaKeyUtils::indexTagKey(1, "test_tag"), tagIdVal); + schemas.emplace_back(MetaKeyUtils::schemaTagKey(1, tagId, ver), + MetaKeyUtils::schemaVal("test_tag", srcsch)); auto edgeTypeVal = std::string(reinterpret_cast(&edgeType), sizeof(edgeType)); - schemas.emplace_back(MetaServiceUtils::indexEdgeKey(1, "test_edge"), edgeTypeVal); - schemas.emplace_back(MetaServiceUtils::schemaEdgeKey(1, edgeType, ver), - MetaServiceUtils::schemaVal("test_edge", srcsch)); + schemas.emplace_back(MetaKeyUtils::indexEdgeKey(1, "test_edge"), edgeTypeVal); + schemas.emplace_back(MetaKeyUtils::schemaEdgeKey(1, edgeType, ver), + MetaKeyUtils::schemaVal("test_edge", srcsch)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(schemas), [&](nebula::cpp2::ErrorCode code) { @@ -2083,7 +2083,7 @@ TEST(ProcessorTest, IndexIdInSpaceRangeTest) { // check tag and edge count int count = 0; - auto tagprefix = MetaServiceUtils::schemaTagsPrefix(1); + auto tagprefix = MetaKeyUtils::schemaTagsPrefix(1); std::unique_ptr iter; auto retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, tagprefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); @@ -2093,7 +2093,7 @@ TEST(ProcessorTest, IndexIdInSpaceRangeTest) { } ASSERT_EQ(1, count); - auto edgeprefix = MetaServiceUtils::schemaEdgesPrefix(1); + auto edgeprefix = MetaKeyUtils::schemaEdgesPrefix(1); retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, edgeprefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); while (iter->valid()) { @@ -2102,7 +2102,7 @@ TEST(ProcessorTest, IndexIdInSpaceRangeTest) { } ASSERT_EQ(2, count); - auto indexPrefix = MetaServiceUtils::indexPrefix(1); + auto indexPrefix = MetaKeyUtils::indexPrefix(1); retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, indexPrefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); while (iter->valid()) { diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index f49609a89c0..af328e63359 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -16,8 +16,8 @@ #include "common/meta/GflagsManager.h" #include "common/meta/ServerBasedSchemaManager.h" #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/common_constants.h" -#include "meta/MetaServiceUtils.h" #include "meta/test/TestUtils.h" #include "mock/MockCluster.h" diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 6e7bd60dd03..7536b78a9a2 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -1482,10 +1482,8 @@ TEST(ProcessorTest, DropTagTest) { { std::string tagVal; std::unique_ptr iter; - auto ret = kv.get()->get(kDefaultSpaceId, - kDefaultPartId, - std::move(MetaServiceUtils::indexTagKey(1, "tag_0")), - &tagVal); + auto ret = kv.get()->get( + kDefaultSpaceId, kDefaultPartId, std::move(MetaKeyUtils::indexTagKey(1, "tag_0")), &tagVal); ASSERT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, ret); std::string tagPrefix = "__tags__"; ret = kv.get()->prefix(kDefaultSpaceId, kDefaultPartId, tagPrefix, &iter); @@ -1573,7 +1571,7 @@ TEST(ProcessorTest, DropEdgeTest) { std::unique_ptr iter; auto ret = kv.get()->get(kDefaultSpaceId, kDefaultPartId, - std::move(MetaServiceUtils::indexEdgeKey(1, "edge_0")), + std::move(MetaKeyUtils::indexEdgeKey(1, "edge_0")), &edgeVal); ASSERT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, ret); std::string edgePrefix = "__edges__"; @@ -2791,7 +2789,7 @@ TEST(ProcessorTest, TagIdAndEdgeTypeInSpaceRangeTest) { // check tag and edge count int count = 0; - auto tagprefix = MetaServiceUtils::schemaTagsPrefix(1); + auto tagprefix = MetaKeyUtils::schemaTagsPrefix(1); std::unique_ptr iter; auto retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, tagprefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); @@ -2801,7 +2799,7 @@ TEST(ProcessorTest, TagIdAndEdgeTypeInSpaceRangeTest) { } ASSERT_EQ(10, count); - auto edgeprefix = MetaServiceUtils::schemaEdgesPrefix(1); + auto edgeprefix = MetaKeyUtils::schemaEdgesPrefix(1); retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, edgeprefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, retCode); while (iter->valid()) { diff --git a/src/meta/test/RestoreProcessorTest.cpp b/src/meta/test/RestoreProcessorTest.cpp index 91c418f7382..eb2b9cc8a4d 100644 --- a/src/meta/test/RestoreProcessorTest.cpp +++ b/src/meta/test/RestoreProcessorTest.cpp @@ -52,15 +52,15 @@ TEST(RestoreProcessorTest, RestoreTest) { properties2.set_replica_factor(3); properties2.set_group_name(groupName); - auto spaceVal = MetaServiceUtils::spaceVal(properties); + auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space"), std::string(reinterpret_cast(&id), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id), MetaServiceUtils::spaceVal(properties)); + data.emplace_back(MetaKeyUtils::spaceKey(id), MetaKeyUtils::spaceVal(properties)); - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space2"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space2"), std::string(reinterpret_cast(&id2), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id2), MetaServiceUtils::spaceVal(properties2)); + data.emplace_back(MetaKeyUtils::spaceKey(id2), MetaKeyUtils::spaceVal(properties2)); std::unordered_map> partInfo; @@ -72,30 +72,30 @@ TEST(RestoreProcessorTest, RestoreTest) { hosts4.emplace_back(h); partInfo[h].emplace_back(partId); } - data.emplace_back(MetaServiceUtils::partKey(id, partId), MetaServiceUtils::partVal(hosts4)); + data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts4)); } auto groupId = 1; std::string zoneName = "test_zone"; std::vector zoneNames = {zoneName}; - data.emplace_back(MetaServiceUtils::indexGroupKey(groupName), + data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaServiceUtils::groupKey(groupName), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); - data.emplace_back(MetaServiceUtils::userKey("root"), MetaServiceUtils::userVal("root")); + data.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("root")); auto zoneId = 1; - data.emplace_back(MetaServiceUtils::indexZoneKey(zoneName), + data.emplace_back(MetaKeyUtils::indexZoneKey(zoneName), std::string(reinterpret_cast(&zoneId), sizeof(ZoneID))); - data.emplace_back(MetaServiceUtils::zoneKey(zoneName), MetaServiceUtils::zoneVal(hosts)); + data.emplace_back(MetaKeyUtils::zoneKey(zoneName), MetaKeyUtils::zoneVal(hosts)); int32_t autoId = 666; - data.emplace_back(MetaServiceUtils::idKey(), + data.emplace_back(MetaKeyUtils::idKey(), std::string(reinterpret_cast(&autoId), sizeof(autoId))); auto lastUpdateTime = time::WallClock::fastNowInMilliSec(); - data.emplace_back(MetaServiceUtils::lastUpdateTimeKey(), - MetaServiceUtils::lastUpdateTimeVal(lastUpdateTime)); + data.emplace_back(MetaKeyUtils::lastUpdateTimeKey(), + MetaKeyUtils::lastUpdateTimeVal(lastUpdateTime)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { @@ -105,7 +105,7 @@ TEST(RestoreProcessorTest, RestoreTest) { baton.wait(); std::unordered_set spaces = {id}; - auto backupName = folly::format("BACKUP_{}", MetaServiceUtils::genTimestampStr()).str(); + auto backupName = folly::format("BACKUP_{}", MetaKeyUtils::genTimestampStr()).str(); auto spaceNames = std::make_unique>(); spaceNames->emplace_back("test_space"); auto backupFiles = MetaServiceUtils::backupSpaces(kv.get(), spaces, backupName, spaceNames.get()); @@ -116,7 +116,7 @@ TEST(RestoreProcessorTest, RestoreTest) { auto it = std::find_if(files.cbegin(), files.cend(), [](auto& f) { auto const pos = f.find_last_of("/"); auto name = f.substr(pos + 1); - if (name == MetaServiceUtils::groupPrefix() || name == MetaServiceUtils::zonePrefix()) { + if (name == MetaKeyUtils::groupPrefix() || name == MetaKeyUtils::zonePrefix()) { return true; } return false; @@ -138,8 +138,7 @@ TEST(RestoreProcessorTest, RestoreTest) { fs::TempDir restoreTootPath("/tmp/RestoreTest.XXXXXX"); std::unique_ptr kvRestore(MockCluster::initMetaKV(restoreTootPath.path())); std::vector restoreData; - restoreData.emplace_back(MetaServiceUtils::userKey("root"), - MetaServiceUtils::userVal("password")); + restoreData.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("password")); folly::Baton restoreBaton; kvRestore->asyncMultiPut(0, 0, std::move(restoreData), [&](nebula::cpp2::ErrorCode code) { @@ -157,7 +156,7 @@ TEST(RestoreProcessorTest, RestoreTest) { nebula::cpp2::ErrorCode result; std::unique_ptr iter; - const auto& partPrefix = MetaServiceUtils::partPrefix(id); + const auto& partPrefix = MetaKeyUtils::partPrefix(id); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, partPrefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); @@ -165,8 +164,8 @@ TEST(RestoreProcessorTest, RestoreTest) { while (iter->valid()) { auto key = iter->key(); - auto partId = MetaServiceUtils::parsePartKeyPartId(key); - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); + auto partId = MetaKeyUtils::parsePartKeyPartId(key); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& host : partHosts) { LOG(INFO) << "partHost: " << host.toString(); toPartInfo[host].emplace_back(partId); @@ -179,15 +178,15 @@ TEST(RestoreProcessorTest, RestoreTest) { ASSERT_TRUE(std::equal(parts.cbegin(), parts.cend(), pi.second.cbegin())); } - auto prefix = MetaServiceUtils::zonePrefix(); + auto prefix = MetaKeyUtils::zonePrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); std::vector zones; std::vector restoredHosts = {host4, host5, host6}; while (iter->valid()) { - auto zn = MetaServiceUtils::parseZoneName(iter->key()); - auto zoneHosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto zn = MetaKeyUtils::parseZoneName(iter->key()); + auto zoneHosts = MetaKeyUtils::parseZoneHosts(iter->val()); cpp2::Zone zone; ASSERT_EQ(zoneName, zn); ASSERT_EQ(zoneHosts.size(), restoredHosts.size()); @@ -197,32 +196,32 @@ TEST(RestoreProcessorTest, RestoreTest) { iter->next(); } - prefix = MetaServiceUtils::groupPrefix(); + prefix = MetaKeyUtils::groupPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); // ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, result); ASSERT_FALSE(iter->valid()); - prefix = MetaServiceUtils::userPrefix(); + prefix = MetaKeyUtils::userPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); while (iter->valid()) { - auto user = MetaServiceUtils::parseUser(iter->key()); - auto password = MetaServiceUtils::parseUserPwd(iter->val()); + auto user = MetaKeyUtils::parseUser(iter->key()); + auto password = MetaKeyUtils::parseUserPwd(iter->val()); ASSERT_EQ(user, std::string("root")); ASSERT_EQ(password, std::string("password")); iter->next(); } - prefix = MetaServiceUtils::indexPrefix(id); + prefix = MetaKeyUtils::indexPrefix(id); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); while (iter->valid()) { - auto sid = MetaServiceUtils::parseIndexesKeySpaceID(iter->key()); + auto sid = MetaKeyUtils::parseIndexesKeySpaceID(iter->key()); ASSERT_EQ(sid, id); iter->next(); } - prefix = MetaServiceUtils::indexPrefix(id2); + prefix = MetaKeyUtils::indexPrefix(id2); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); ASSERT_FALSE(iter->valid()); @@ -278,15 +277,15 @@ TEST(RestoreProcessorTest, RestoreFullTest) { properties2.set_replica_factor(3); properties2.set_group_name(groupName); - auto spaceVal = MetaServiceUtils::spaceVal(properties); + auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space"), std::string(reinterpret_cast(&id), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id), MetaServiceUtils::spaceVal(properties)); + data.emplace_back(MetaKeyUtils::spaceKey(id), MetaKeyUtils::spaceVal(properties)); - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space2"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space2"), std::string(reinterpret_cast(&id2), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id2), MetaServiceUtils::spaceVal(properties2)); + data.emplace_back(MetaKeyUtils::spaceKey(id2), MetaKeyUtils::spaceVal(properties2)); std::unordered_map> partInfo; @@ -298,22 +297,22 @@ TEST(RestoreProcessorTest, RestoreFullTest) { hosts4.emplace_back(h); partInfo[h].emplace_back(partId); } - data.emplace_back(MetaServiceUtils::partKey(id, partId), MetaServiceUtils::partVal(hosts4)); + data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts4)); } auto groupId = 1; std::string zoneName = "test_zone"; std::vector zoneNames = {zoneName}; - data.emplace_back(MetaServiceUtils::indexGroupKey(groupName), + data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaServiceUtils::groupKey(groupName), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); - data.emplace_back(MetaServiceUtils::userKey("root"), MetaServiceUtils::userVal("root")); + data.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("root")); auto zoneId = 1; - data.emplace_back(MetaServiceUtils::indexZoneKey(zoneName), + data.emplace_back(MetaKeyUtils::indexZoneKey(zoneName), std::string(reinterpret_cast(&zoneId), sizeof(ZoneID))); - data.emplace_back(MetaServiceUtils::zoneKey(zoneName), MetaServiceUtils::zoneVal(hosts)); + data.emplace_back(MetaKeyUtils::zoneKey(zoneName), MetaKeyUtils::zoneVal(hosts)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { @@ -323,7 +322,7 @@ TEST(RestoreProcessorTest, RestoreFullTest) { baton.wait(); std::unordered_set spaces = {id}; - auto backupName = folly::format("BACKUP_{}", MetaServiceUtils::genTimestampStr()).str(); + auto backupName = folly::format("BACKUP_{}", MetaKeyUtils::genTimestampStr()).str(); auto backupFiles = MetaServiceUtils::backupSpaces(kv.get(), spaces, backupName, nullptr); DCHECK(nebula::hasValue(backupFiles)); { @@ -332,7 +331,7 @@ TEST(RestoreProcessorTest, RestoreFullTest) { auto it = std::find_if(files.cbegin(), files.cend(), [](auto& f) { auto const pos = f.find_last_of("/"); auto name = f.substr(pos + 1); - if (name == MetaServiceUtils::groupPrefix() || name == MetaServiceUtils::zonePrefix()) { + if (name == MetaKeyUtils::groupPrefix() || name == MetaKeyUtils::zonePrefix()) { return true; } return false; @@ -354,8 +353,7 @@ TEST(RestoreProcessorTest, RestoreFullTest) { fs::TempDir restoreTootPath("/tmp/RestoreFullTest.XXXXXX"); std::unique_ptr kvRestore(MockCluster::initMetaKV(restoreTootPath.path())); std::vector restoreData; - restoreData.emplace_back(MetaServiceUtils::userKey("root"), - MetaServiceUtils::userVal("password")); + restoreData.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("password")); folly::Baton restoreBaton; kvRestore->asyncMultiPut(0, 0, std::move(restoreData), [&](nebula::cpp2::ErrorCode code) { @@ -373,7 +371,7 @@ TEST(RestoreProcessorTest, RestoreFullTest) { nebula::cpp2::ErrorCode result; std::unique_ptr iter; - const auto& partPrefix = MetaServiceUtils::partPrefix(id); + const auto& partPrefix = MetaKeyUtils::partPrefix(id); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, partPrefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); @@ -381,8 +379,8 @@ TEST(RestoreProcessorTest, RestoreFullTest) { while (iter->valid()) { auto key = iter->key(); - auto partId = MetaServiceUtils::parsePartKeyPartId(key); - auto partHosts = MetaServiceUtils::parsePartVal(iter->val()); + auto partId = MetaKeyUtils::parsePartKeyPartId(key); + auto partHosts = MetaKeyUtils::parsePartVal(iter->val()); for (auto& host : partHosts) { LOG(INFO) << "partHost: " << host.toString(); toPartInfo[host].emplace_back(partId); @@ -395,15 +393,15 @@ TEST(RestoreProcessorTest, RestoreFullTest) { ASSERT_TRUE(std::equal(parts.cbegin(), parts.cend(), pi.second.cbegin())); } - auto prefix = MetaServiceUtils::zonePrefix(); + auto prefix = MetaKeyUtils::zonePrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); std::vector zones; std::vector restoredHosts = {host4, host5, host6}; while (iter->valid()) { - auto zn = MetaServiceUtils::parseZoneName(iter->key()); - auto zoneHosts = MetaServiceUtils::parseZoneHosts(iter->val()); + auto zn = MetaKeyUtils::parseZoneName(iter->key()); + auto zoneHosts = MetaKeyUtils::parseZoneHosts(iter->val()); cpp2::Zone zone; ASSERT_EQ(zoneName, zn); ASSERT_EQ(zoneHosts.size(), restoredHosts.size()); @@ -413,36 +411,36 @@ TEST(RestoreProcessorTest, RestoreFullTest) { iter->next(); } - prefix = MetaServiceUtils::groupPrefix(); + prefix = MetaKeyUtils::groupPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); // ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, result); ASSERT_TRUE(iter->valid()); - prefix = MetaServiceUtils::userPrefix(); + prefix = MetaKeyUtils::userPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); while (iter->valid()) { - auto user = MetaServiceUtils::parseUser(iter->key()); - auto password = MetaServiceUtils::parseUserPwd(iter->val()); + auto user = MetaKeyUtils::parseUser(iter->key()); + auto password = MetaKeyUtils::parseUserPwd(iter->val()); ASSERT_EQ(user, std::string("root")); ASSERT_EQ(password, std::string("root")); iter->next(); } - prefix = MetaServiceUtils::indexPrefix(id); + prefix = MetaKeyUtils::indexPrefix(id); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); while (iter->valid()) { - auto sid = MetaServiceUtils::parseIndexesKeySpaceID(iter->key()); + auto sid = MetaKeyUtils::parseIndexesKeySpaceID(iter->key()); ASSERT_EQ(sid, id); iter->next(); } - prefix = MetaServiceUtils::indexPrefix(id2); + prefix = MetaKeyUtils::indexPrefix(id2); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); while (iter->valid()) { - auto sid = MetaServiceUtils::parseIndexesKeySpaceID(iter->key()); + auto sid = MetaKeyUtils::parseIndexesKeySpaceID(iter->key()); ASSERT_EQ(sid, id2); iter->next(); } diff --git a/src/meta/test/TestUtils.h b/src/meta/test/TestUtils.h index 5fe506a2f33..a9e2f28a715 100644 --- a/src/meta/test/TestUtils.h +++ b/src/meta/test/TestUtils.h @@ -109,19 +109,17 @@ class TestUtils { std::vector data; int32_t id = 10; for (auto iter = zoneInfo.begin(); iter != zoneInfo.end(); iter++) { - data.emplace_back(MetaServiceUtils::indexZoneKey(iter->first), + data.emplace_back(MetaKeyUtils::indexZoneKey(iter->first), std::string(reinterpret_cast(&id), sizeof(ZoneID))); - data.emplace_back(MetaServiceUtils::zoneKey(iter->first), - MetaServiceUtils::zoneVal(iter->second)); + data.emplace_back(MetaKeyUtils::zoneKey(iter->first), MetaKeyUtils::zoneVal(iter->second)); id += 1; } id = 100; for (auto iter = groupInfo.begin(); iter != groupInfo.end(); iter++) { - data.emplace_back(MetaServiceUtils::indexGroupKey(iter->first), + data.emplace_back(MetaKeyUtils::indexGroupKey(iter->first), std::string(reinterpret_cast(&id), sizeof(GroupID))); - data.emplace_back(MetaServiceUtils::groupKey(iter->first), - MetaServiceUtils::groupVal(iter->second)); + data.emplace_back(MetaKeyUtils::groupKey(iter->first), MetaKeyUtils::groupVal(iter->second)); id += 1; } @@ -137,7 +135,7 @@ class TestUtils { const std::string& zoneName, HostAddr host, bool isAdd) { - auto zoneKey = MetaServiceUtils::zoneKey(zoneName); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); std::string zoneValue; auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -145,7 +143,7 @@ class TestUtils { return false; } - auto hosts = MetaServiceUtils::parseZoneHosts(std::move(zoneValue)); + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(zoneValue)); auto iter = std::find(hosts.begin(), hosts.end(), host); if (isAdd) { if (iter != hosts.end()) { @@ -162,7 +160,7 @@ class TestUtils { } std::vector data; - data.emplace_back(zoneKey, MetaServiceUtils::zoneVal(std::move(hosts))); + data.emplace_back(zoneKey, MetaKeyUtils::zoneVal(std::move(hosts))); bool ret = false; folly::Baton baton; @@ -178,7 +176,7 @@ class TestUtils { const std::string& groupName, const std::string& zoneName, bool isAdd) { - auto groupKey = MetaServiceUtils::groupKey(groupName); + auto groupKey = MetaKeyUtils::groupKey(groupName); std::string groupValue; auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -186,7 +184,7 @@ class TestUtils { return false; } - auto zoneNames = MetaServiceUtils::parseZoneNames(std::move(groupValue)); + auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); if (isAdd) { if (iter != zoneNames.end()) { @@ -203,7 +201,7 @@ class TestUtils { } std::vector data; - data.emplace_back(std::move(groupKey), MetaServiceUtils::groupVal(zoneNames)); + data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { @@ -224,11 +222,11 @@ class TestUtils { properties.set_space_name("test_space"); properties.set_partition_num(partitionNum); properties.set_replica_factor(replica); - auto spaceVal = MetaServiceUtils::spaceVal(properties); + auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; - data.emplace_back(MetaServiceUtils::indexSpaceKey("test_space"), + data.emplace_back(MetaKeyUtils::indexSpaceKey("test_space"), std::string(reinterpret_cast(&id), sizeof(GraphSpaceID))); - data.emplace_back(MetaServiceUtils::spaceKey(id), MetaServiceUtils::spaceVal(properties)); + data.emplace_back(MetaKeyUtils::spaceKey(id), MetaKeyUtils::spaceVal(properties)); std::vector allHosts; for (int i = 0; i < totalHost; i++) { @@ -241,7 +239,7 @@ class TestUtils { for (int32_t i = 0; i < replica; i++, idx++) { hosts.emplace_back(allHosts[idx % totalHost]); } - data.emplace_back(MetaServiceUtils::partKey(id, partId), MetaServiceUtils::partVal(hosts)); + data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts)); } folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { @@ -278,9 +276,9 @@ class TestUtils { } auto tagName = folly::stringPrintf("tag_%d", tagId); auto tagIdVal = std::string(reinterpret_cast(&tagId), sizeof(tagId)); - tags.emplace_back(MetaServiceUtils::indexTagKey(spaceId, tagName), tagIdVal); - tags.emplace_back(MetaServiceUtils::schemaTagKey(spaceId, tagId, ver++), - MetaServiceUtils::schemaVal(tagName, srcsch)); + tags.emplace_back(MetaKeyUtils::indexTagKey(spaceId, tagName), tagIdVal); + tags.emplace_back(MetaKeyUtils::schemaTagKey(spaceId, tagId, ver++), + MetaKeyUtils::schemaVal(tagName, srcsch)); } folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(tags), [&](nebula::cpp2::ErrorCode code) { @@ -307,9 +305,9 @@ class TestUtils { item.set_fields(std::move(columns)); std::vector data; - data.emplace_back(MetaServiceUtils::indexIndexKey(space, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(space, indexName), std::string(reinterpret_cast(&indexID), sizeof(IndexID))); - data.emplace_back(MetaServiceUtils::indexKey(space, indexID), MetaServiceUtils::indexVal(item)); + data.emplace_back(MetaKeyUtils::indexKey(space, indexID), MetaKeyUtils::indexVal(item)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); @@ -335,9 +333,9 @@ class TestUtils { item.set_fields(std::move(columns)); std::vector data; - data.emplace_back(MetaServiceUtils::indexIndexKey(space, indexName), + data.emplace_back(MetaKeyUtils::indexIndexKey(space, indexName), std::string(reinterpret_cast(&indexID), sizeof(IndexID))); - data.emplace_back(MetaServiceUtils::indexKey(space, indexID), MetaServiceUtils::indexVal(item)); + data.emplace_back(MetaKeyUtils::indexKey(space, indexID), MetaKeyUtils::indexVal(item)); folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); @@ -373,9 +371,9 @@ class TestUtils { } auto edgeName = folly::stringPrintf("edge_%d", edgeType); auto edgeTypeVal = std::string(reinterpret_cast(&edgeType), sizeof(edgeType)); - edges.emplace_back(MetaServiceUtils::indexEdgeKey(spaceId, edgeName), edgeTypeVal); - edges.emplace_back(MetaServiceUtils::schemaEdgeKey(spaceId, edgeType, ver++), - MetaServiceUtils::schemaVal(edgeName, srcsch)); + edges.emplace_back(MetaKeyUtils::indexEdgeKey(spaceId, edgeName), edgeTypeVal); + edges.emplace_back(MetaKeyUtils::schemaEdgeKey(spaceId, edgeType, ver++), + MetaKeyUtils::schemaVal(edgeName, srcsch)); } folly::Baton baton; diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index 8b3c870fcff..3640fa0d7f3 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -14,6 +14,7 @@ #include "common/datatypes/Map.h" #include "common/datatypes/Value.h" #include "common/expression/ConstantExpression.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/meta_types.h" #include "kvstore/Common.h" #include "meta/ActiveHostsMan.h" @@ -31,7 +32,7 @@ Status MetaDataUpgrade::rewriteHosts(const folly::StringPiece &key, const folly: auto info = HostInfo::decodeV1(val); auto newVal = HostInfo::encodeV2(info); auto newKey = - MetaServiceUtils::hostKeyV2(network::NetworkUtils::intToIPv4(host.get_ip()), host.get_port()); + MetaKeyUtils::hostKeyV2(network::NetworkUtils::intToIPv4(host.get_ip()), host.get_port()); NG_LOG_AND_RETURN_IF_ERROR(put(newKey, newVal)); NG_LOG_AND_RETURN_IF_ERROR(remove(key)); return Status::OK(); @@ -41,7 +42,7 @@ Status MetaDataUpgrade::rewriteLeaders(const folly::StringPiece &key, const folly::StringPiece &val) { auto host = meta::v1::MetaServiceUtilsV1::parseLeaderKey(key); auto newKey = - MetaServiceUtils::leaderKey(network::NetworkUtils::intToIPv4(host.get_ip()), host.get_port()); + MetaKeyUtils::leaderKey(network::NetworkUtils::intToIPv4(host.get_ip()), host.get_port()); NG_LOG_AND_RETURN_IF_ERROR(put(newKey, val)); NG_LOG_AND_RETURN_IF_ERROR(remove(key)); return Status::OK(); @@ -58,7 +59,7 @@ Status MetaDataUpgrade::rewriteSpaces(const folly::StringPiece &key, spaceDesc.set_collate_name(oldProps.get_collate_name()); (*spaceDesc.vid_type_ref()).set_type_length(8); (*spaceDesc.vid_type_ref()).set_type(cpp2::PropertyType::INT64); - NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaServiceUtils::spaceVal(spaceDesc))); + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::spaceVal(spaceDesc))); return Status::OK(); } @@ -71,7 +72,7 @@ Status MetaDataUpgrade::rewriteParts(const folly::StringPiece &key, const folly: hostAddr.port = host.get_port(); newHosts.emplace_back(std::move(hostAddr)); } - NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaServiceUtils::partVal(newHosts))); + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::partVal(newHosts))); return Status::OK(); } @@ -93,7 +94,7 @@ Status MetaDataUpgrade::rewriteSchemas(const folly::StringPiece &key, auto nameLen = *reinterpret_cast(val.data()); auto schemaName = val.subpiece(sizeof(int32_t), nameLen).str(); - auto encodeVal = MetaServiceUtils::schemaVal(schemaName, newSchema); + auto encodeVal = MetaKeyUtils::schemaVal(schemaName, newSchema); NG_LOG_AND_RETURN_IF_ERROR(put(key, encodeVal)); return Status::OK(); } @@ -113,7 +114,7 @@ Status MetaDataUpgrade::rewriteIndexes(const folly::StringPiece &key, newItem.set_schema_id(schemaId); NG_LOG_AND_RETURN_IF_ERROR( convertToNewIndexColumns((*oldItem.fields_ref()), (*newItem.fields_ref()))); - NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaServiceUtils::indexVal(newItem))); + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::indexVal(newItem))); return Status::OK(); } @@ -160,7 +161,7 @@ Status MetaDataUpgrade::rewriteConfigs(const folly::StringPiece &key, } } auto newVal = - MetaServiceUtils::configValue(static_cast(item.get_mode()), configVal); + MetaKeyUtils::configValue(static_cast(item.get_mode()), configVal); NG_LOG_AND_RETURN_IF_ERROR(put(key, newVal)); return Status::OK(); } diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index 3ef242494dd..839015df554 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -10,6 +10,7 @@ #include "common/base/Base.h" #include "common/base/Status.h" +#include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/meta_types.h" #include "kvstore/KVStore.h" #include "meta/processors/Common.h" diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 9ad3b3e7529..d976ab2b209 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -17,6 +17,7 @@ set(storage_test_deps $ $ $ + $ $ $ $ diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 893bd53e494..b04c86360cc 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -15,6 +15,7 @@ set(tools_test_deps $ $ $ + $ $ $ $ diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 4aa3f68be00..584d03acf2d 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -23,6 +23,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 46c508f3b1f..8ba15751e2b 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -20,6 +20,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ diff --git a/src/tools/meta-dump/MetaDumpTool.cpp b/src/tools/meta-dump/MetaDumpTool.cpp index 4f85276d71e..e918ba8613f 100644 --- a/src/tools/meta-dump/MetaDumpTool.cpp +++ b/src/tools/meta-dump/MetaDumpTool.cpp @@ -9,8 +9,8 @@ #include "common/fs/FileUtils.h" #include "common/time/TimeUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/ActiveHostsMan.h" -#include "meta/MetaServiceUtils.h" DEFINE_string(path, "", "rocksdb instance path"); @@ -40,8 +40,8 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto spaceId = MetaServiceUtils::spaceId(key); - auto desc = MetaServiceUtils::parseSpace(val); + auto spaceId = MetaKeyUtils::spaceId(key); + auto desc = MetaKeyUtils::parseSpace(val); LOG(INFO) << folly::sformat( "space id: {}, space name: {}, partition num: {}, replica_factor: " "{}", @@ -59,9 +59,9 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto spaceId = MetaServiceUtils::parsePartKeySpaceId(key); - auto partId = MetaServiceUtils::parsePartKeyPartId(key); - auto hosts = MetaServiceUtils::parsePartVal(val); + auto spaceId = MetaKeyUtils::parsePartKeySpaceId(key); + auto partId = MetaKeyUtils::parsePartKeyPartId(key); + auto hosts = MetaKeyUtils::parsePartVal(val); std::stringstream ss; for (const auto& host : hosts) { ss << host << " "; @@ -78,7 +78,7 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto host = MetaServiceUtils::parseHostKey(key); + auto host = MetaKeyUtils::parseHostKey(key); auto info = HostInfo::decode(val); auto role = apache::thrift::util::enumNameSafe(info.role_); auto time = time::TimeConversion::unixSecondsToDateTime(info.lastHBTimeInMilliSec_ / 1000); @@ -93,9 +93,9 @@ class MetaDumper { iter->Seek(rocksdb::Slice(prefix)); while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); - auto spaceId = MetaServiceUtils::parseTagsKeySpaceID(key); - auto tagId = MetaServiceUtils::parseTagId(key); - auto ver = MetaServiceUtils::parseTagVersion(key); + auto spaceId = MetaKeyUtils::parseTagsKeySpaceID(key); + auto tagId = MetaKeyUtils::parseTagId(key); + auto ver = MetaKeyUtils::parseTagVersion(key); LOG(INFO) << folly::sformat("space id: {}, tag id: {}, ver: {}", spaceId, tagId, ver); iter->Next(); } @@ -106,9 +106,9 @@ class MetaDumper { iter->Seek(rocksdb::Slice(prefix)); while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); - auto spaceId = MetaServiceUtils::parseEdgesKeySpaceID(key); - auto type = MetaServiceUtils::parseEdgeType(key); - auto ver = MetaServiceUtils::parseEdgeVersion(key); + auto spaceId = MetaKeyUtils::parseEdgesKeySpaceID(key); + auto type = MetaKeyUtils::parseEdgeType(key); + auto ver = MetaKeyUtils::parseEdgeVersion(key); LOG(INFO) << folly::sformat("space id: {}, edge type: {}, ver: {}", spaceId, type, ver); iter->Next(); } @@ -119,8 +119,8 @@ class MetaDumper { iter->Seek(rocksdb::Slice(prefix)); while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); - auto spaceId = MetaServiceUtils::parseIndexesKeySpaceID(key); - auto indexId = MetaServiceUtils::parseIndexesKeyIndexID(key); + auto spaceId = MetaKeyUtils::parseIndexesKeySpaceID(key); + auto indexId = MetaKeyUtils::parseIndexesKeyIndexID(key); LOG(INFO) << folly::sformat("space id: {}, index: {}", spaceId, indexId); iter->Next(); } @@ -135,9 +135,9 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto spaceIdAndPartId = MetaServiceUtils::parseLeaderKeyV3(key); + auto spaceIdAndPartId = MetaKeyUtils::parseLeaderKeyV3(key); - std::tie(host, term, code) = MetaServiceUtils::parseLeaderValV3(val); + std::tie(host, term, code) = MetaKeyUtils::parseLeaderValV3(val); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << folly::sformat("leader space id: {}, part id: {} illegal.", spaceIdAndPartId.first, @@ -159,8 +159,8 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto group = MetaServiceUtils::parseGroupName(key); - auto zones = MetaServiceUtils::parseZoneNames(val); + auto group = MetaKeyUtils::parseGroupName(key); + auto zones = MetaKeyUtils::parseZoneNames(val); std::stringstream ss; for (const auto& zone : zones) { ss << zone << " "; @@ -176,8 +176,8 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { auto key = folly::StringPiece(iter->key().data(), iter->key().size()); auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto zone = MetaServiceUtils::parseZoneName(key); - auto hosts = MetaServiceUtils::parseZoneHosts(val); + auto zone = MetaKeyUtils::parseZoneName(key); + auto hosts = MetaKeyUtils::parseZoneHosts(val); std::stringstream ss; for (const auto& host : hosts) { ss << host << " "; diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 17a2592fe13..32e8eafa5ee 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -17,6 +17,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index e278053ecb2..ea4d1dfab25 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -13,6 +13,7 @@ set(perf_test_deps $ $ $ + $ $ $ $ From 17c04dd13f105621d6937acb9a78a0eb84063585 Mon Sep 17 00:00:00 2001 From: Sophie-Xie <84560950+Sophie-Xie@users.noreply.github.com> Date: Fri, 22 Oct 2021 10:02:16 +0800 Subject: [PATCH 2/4] Create pull_request_template.md (#3157) * Create pull_request_template.md * Update pull_request_template.md * Update pull_request_template.md * Update pull_request_template.md Co-authored-by: Yee <2520865+yixinglu@users.noreply.github.com> --- .github/pull_request_template.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..469ab40adc0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,29 @@ +#### What type of PR is this? +- [ ] bug +- [ ] feature + +#### Which issue(s) this PR fixes: +close #xxx +（If it is requirement, issue(s) number must be listed.） + +#### What this PR does / why we need it? + + +#### Special notes for your reviewer, ex. impact of this fix, etc: + + +#### Additional context: + + +#### Checklist： +- [ ] Documentation affected （If need to modify document, please label it.) +- [ ] Incompatible （If it is incompatile, please describle it and label it.） +- [ ] Need to cherry pick （If need to cherry pick to some branchs, please label the destination version(s).) +- [ ] Performance regression: Consumes more CPU +- [ ] Performance regression: Consumes more Memory + + + +#### Release notes： +Please confirm whether to reflect in release notes and how to describe: +> ` From db6da983909e5ffb807f12afa41093205931f5d5 Mon Sep 17 00:00:00 2001 From: "jie.wang" <38901892+jievince@users.noreply.github.com> Date: Fri, 22 Oct 2021 10:40:15 +0800 Subject: [PATCH 3/4] fix geo predicate rule and geography isvalid (#3180) --- src/common/datatypes/Geography.cpp | 6 ++++++ src/common/function/FunctionManager.cpp | 16 +++++++++----- src/common/geo/GeoFunction.cpp | 4 ++++ src/common/geo/io/wkb/test/WKBTest.cpp | 2 +- src/common/geo/test/GeoFunctionTest.cpp | 21 +++++++++++++++++++ src/common/utils/IndexKeyUtils.h | 2 +- .../rule/GeoPredicateIndexScanBaseRule.cpp | 6 +++--- src/graph/validator/LookupValidator.cpp | 2 ++ tests/tck/features/geo/GeoBase.feature | 18 ++++++++++++++++ 9 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/common/datatypes/Geography.cpp b/src/common/datatypes/Geography.cpp index aef9a457599..d26d43e6a66 100644 --- a/src/common/datatypes/Geography.cpp +++ b/src/common/datatypes/Geography.cpp @@ -50,6 +50,9 @@ bool LineString::isValid() const { if (coordList.size() < 2) { return false; } + for (const auto& coord : coordList) { + if (!coord.isValid()) return false; + } auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); CHECK_NOTNULL(s2Region); return static_cast(s2Region.get())->IsValid(); @@ -71,6 +74,9 @@ bool Polygon::isValid() const { if (coordList.front() != coordList.back()) { return false; } + for (const auto& coord : coordList) { + if (!coord.isValid()) return false; + } } auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); CHECK_NOTNULL(s2Region); diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index 07b402e74bb..68fc09dfd68 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -368,11 +368,16 @@ std::unordered_map> FunctionManager::typ { TypeSignature({Value::Type::GEOGRAPHY, Value::Type::GEOGRAPHY, Value::Type::FLOAT}, Value::Type::BOOL), + TypeSignature({Value::Type::GEOGRAPHY, Value::Type::GEOGRAPHY, Value::Type::INT}, + Value::Type::BOOL), TypeSignature({Value::Type::GEOGRAPHY, Value::Type::GEOGRAPHY, Value::Type::FLOAT, Value::Type::BOOL}, Value::Type::BOOL), + TypeSignature( + {Value::Type::GEOGRAPHY, Value::Type::GEOGRAPHY, Value::Type::INT, Value::Type::BOOL}, + Value::Type::BOOL), }}, // geo measures {"st_distance", @@ -2497,7 +2502,7 @@ FunctionManager::FunctionManager() { attr.isPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography() || - !args[2].get().isFloat()) { + !args[2].get().isNumeric()) { return Value::kNullBadType; } bool exclusive = false; @@ -2507,10 +2512,11 @@ FunctionManager::FunctionManager() { } exclusive = args[3].get().getBool(); } - return geo::GeoFunction::dWithin(args[0].get().getGeography(), - args[1].get().getGeography(), - args[2].get().getFloat(), - exclusive); + return geo::GeoFunction::dWithin( + args[0].get().getGeography(), + args[1].get().getGeography(), + args[2].get().isFloat() ? args[2].get().getFloat() : args[2].get().getInt(), + exclusive); }; } // geo measures diff --git a/src/common/geo/GeoFunction.cpp b/src/common/geo/GeoFunction.cpp index 5ceedb47b54..cbf8311721f 100644 --- a/src/common/geo/GeoFunction.cpp +++ b/src/common/geo/GeoFunction.cpp @@ -436,6 +436,10 @@ std::vector GeoFunction::s2CoveringCellIds( opts.set_max_cells(maxCells); if (bufferInMeters == 0.0) { + if (a.shape() == GeoShape::POINT) { + const S2Point& gPoint = static_cast(aRegion.get())->point(); + return {S2CellId(gPoint).id()}; + } return coveringCellIds(*aRegion, opts); } diff --git a/src/common/geo/io/wkb/test/WKBTest.cpp b/src/common/geo/io/wkb/test/WKBTest.cpp index e93c3ef1e2b..1d232a88e68 100644 --- a/src/common/geo/io/wkb/test/WKBTest.cpp +++ b/src/common/geo/io/wkb/test/WKBTest.cpp @@ -82,7 +82,7 @@ TEST_F(WKBTest, TestWKB) { LineString v(std::vector{Coordinate(26.4, 78.9), Coordinate(138.725, 91.0)}); auto result = read(v); ASSERT_TRUE(result.ok()) << result.status(); - EXPECT_EQ(true, v.isValid()); + EXPECT_EQ(false, v.isValid()); } { LineString v(std::vector{Coordinate(0, 1), Coordinate(2, 3), Coordinate(0, 1)}); diff --git a/src/common/geo/test/GeoFunctionTest.cpp b/src/common/geo/test/GeoFunctionTest.cpp index 5bfc5ba489d..9cddf7105fb 100644 --- a/src/common/geo/test/GeoFunctionTest.cpp +++ b/src/common/geo/test/GeoFunctionTest.cpp @@ -1096,6 +1096,16 @@ TEST(isValid, lineString) { bool b = line.isValid(); EXPECT_EQ(false, b); } + { + auto line = Geography::fromWKT("LINESTRING(1.0 1.0, 181.0 2.0)").value(); + bool b = line.isValid(); + EXPECT_EQ(false, b); + } + { + auto line = Geography::fromWKT("LINESTRING(1.0 1.0, 1.0 90.001)").value(); + bool b = line.isValid(); + EXPECT_EQ(false, b); + } } TEST(isValid, polygon) { @@ -1153,6 +1163,17 @@ TEST(isValid, polygon) { // bool b = polygon.isValid(); // EXPECT_EQ(false, b); // Expect false, got true // } + { + auto polygon = + Geography::fromWKT("POLYGON((1.0 1.0, -180.0001 2.0, 0.0 2.0, 1.0 1.0))").value(); + bool b = polygon.isValid(); + EXPECT_EQ(false, b); + } + { + auto polygon = Geography::fromWKT("POLYGON((1.0 1.0, 2.0 2.0, 0.0 -90.001, 1.0 1.0))").value(); + bool b = polygon.isValid(); + EXPECT_EQ(false, b); + } } } // namespace geo diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index 049fabebebb..90522db8757 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -437,7 +437,7 @@ class IndexKeyUtils final { break; } case Value::Type::GEOGRAPHY: { - // LOG(FATAL) << "unable to get geography value from index key" + // NOTE: The data read from index key is S2CellId which type is uint64, not wkb len = sizeof(uint64_t); break; } diff --git a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp index 5f7eacd82d9..2b10d1fb602 100644 --- a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp +++ b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp @@ -81,7 +81,7 @@ StatusOr GeoPredicateIndexScanBaseRule::transform( first->kind() == Expression::Kind::kEdgeProperty); DCHECK(second->kind() == Expression::Kind::kConstant); const auto& secondVal = static_cast(second)->value(); - DCHECK(secondVal.type() == Value::Type::GEOGRAPHY); + DCHECK(secondVal.isGeography()); const auto& geog = secondVal.getGeography(); // TODO(jie): Get index params from meta to construct RegionCoverParams @@ -100,8 +100,8 @@ StatusOr GeoPredicateIndexScanBaseRule::transform( auto* third = geoPredicate->args()->args()[2]; DCHECK_EQ(third->kind(), Expression::Kind::kConstant); const auto& thirdVal = static_cast(third)->value(); - DCHECK_EQ(thirdVal.type(), Value::Type::FLOAT); - double distanceInMeters = thirdVal.getFloat(); + DCHECK(thirdVal.isNumeric()); + double distanceInMeters = thirdVal.isFloat() ? thirdVal.getFloat() : thirdVal.getInt(); scanRanges = geoIndex.dWithin(geog, distanceInMeters); } std::vector idxCtxs; diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index a17f83a9807..615354ac0b3 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -224,6 +224,8 @@ Status LookupValidator::validateFilter() { auto ret = checkFilter(filter); NG_RETURN_IF_ERROR(ret); lookupCtx_->filter = std::move(ret).value(); + // Make sure the type of the rewritted filter expr is right + NG_RETURN_IF_ERROR(deduceExprType(lookupCtx_->filter)); } NG_RETURN_IF_ERROR(deduceProps(lookupCtx_->filter, exprProps_)); return Status::OK(); diff --git a/tests/tck/features/geo/GeoBase.feature b/tests/tck/features/geo/GeoBase.feature index c390f20a6ed..1110c17c9f1 100644 --- a/tests/tck/features/geo/GeoBase.feature +++ b/tests/tck/features/geo/GeoBase.feature @@ -420,6 +420,16 @@ Feature: Geo base LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, any_shape.geo) YIELD ST_ASText(any_shape.geo); """ Then a SemanticError should be raised at runtime: Expression ST_Intersects(any_shape.geo,any_shape.geo) not supported yet + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8.4)) < true YIELD ST_ASText(any_shape.geo); + """ + Then a SemanticError should be raised at runtime: + When executing query: + """ + LOOKUP ON any_shape WHERE ST_DWithin(any_shape.geo, ST_Point(3, 8.4), true) YIELD ST_ASText(any_shape.geo); + """ + Then a SemanticError should be raised at runtime: # Match with geo predicate When executing query: """ @@ -523,6 +533,14 @@ Feature: Geo base | VertexID | ST_ASText(any_shape.geo) | | "101" | "POINT(3 8)" | | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_DWithin(any_shape.geo, ST_Point(3, 8), 100) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | # ST_Covers When executing query: """ From b8dc6acb4aaa025fee88ab3e43d5c93bf84b003b Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Fri, 22 Oct 2021 01:36:45 -0500 Subject: [PATCH 4/4] fix crash when drop space (#3185) --- src/kvstore/RocksEngine.cpp | 2 +- src/kvstore/test/RocksEngineTest.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index 73bdb14e104..12db0cbf110 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -124,8 +124,8 @@ RocksEngine::RocksEngine(GraphSpaceID spaceId, } CHECK(status.ok()) << status.ToString(); db_.reset(db); - partsNum_ = allParts().size(); extractorLen_ = sizeof(PartitionID) + vIdLen; + partsNum_ = allParts().size(); LOG(INFO) << "open rocksdb on " << path; backup(); diff --git a/src/kvstore/test/RocksEngineTest.cpp b/src/kvstore/test/RocksEngineTest.cpp index 66eddedff2c..09903a4f6de 100644 --- a/src/kvstore/test/RocksEngineTest.cpp +++ b/src/kvstore/test/RocksEngineTest.cpp @@ -734,6 +734,8 @@ TEST(RebuildPrefixBloomFilter, RebuildPrefixBloomFilter) { EXPECT_EQ("123", value); }; + auto checkSystemPart = [&]() { EXPECT_EQ(10, engine->allParts().size()); }; + checkVertexPrefix(1, "1"); checkVertexPrefix(1, "2"); checkVertexPrefix(2, "3"); @@ -748,11 +750,16 @@ TEST(RebuildPrefixBloomFilter, RebuildPrefixBloomFilter) { checkEdgePartPrefix(2); checkRangeWithPartPrefix(1); checkRangeWithPartPrefix(2); + checkSystemPart(); checkSystemCommit(1); checkSystemCommit(2); }; auto writeData = [&engine] { + for (PartitionID partId = 1; partId <= 10; partId++) { + engine->addPart(partId); + } + LOG(INFO) << "Write some data"; std::vector data; for (TagID tagId = 0; tagId < 10; tagId++) {