diff --git a/proto/BUILD b/proto/BUILD index a18a262a09..565d46d1e4 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -68,8 +68,7 @@ proto_library( "chunkserver.proto", "curve_storage.proto", ]), - deps = [":common_proto", - ":scan_proto"], + deps = [":common_proto", ":scan_proto"], visibility = ["//visibility:public"], ) @@ -98,10 +97,10 @@ proto_library( cc_proto_library( name = "scan_cc_proto", visibility = ["//visibility:public"], - deps = [":scan_proto"], + deps = [":scan_proto"], ) proto_library( name = "scan_proto", - srcs = ["scan.proto"], + srcs = ["scan.proto"], ) diff --git a/proto/common.proto b/proto/common.proto index 21f759f246..65254e176e 100644 --- a/proto/common.proto +++ b/proto/common.proto @@ -32,4 +32,8 @@ message Peer { message CopysetInfo { required uint32 logicalPoolId = 1; required uint32 copysetId = 2; + optional bool scaning = 3; + optional uint64 lastScanSec = 4; + optional bool lastScanConsistent = 5; } + diff --git a/proto/heartbeat.proto b/proto/heartbeat.proto index 19c17b8cd6..aa92baa1a0 100644 --- a/proto/heartbeat.proto +++ b/proto/heartbeat.proto @@ -44,7 +44,7 @@ message CopySetInfo { optional bool scaning = 8; // timestamp for last success scan (seconds) optional uint64 lastScanSec = 9; - // failed crc check scanmap + // the detail information for inconsistent copyset repeated chunkserver.ScanMap scanMap = 10; }; @@ -118,7 +118,7 @@ enum ConfigChangeType { NONE = 4; // 配置变更命令：change复制组一个成员 CHANGE_PEER = 5; - // start scan on the peer + // start scan on the peer START_SCAN_PEER = 6; // cancel scan on the peer CANCEL_SCAN_PEER = 7; diff --git a/proto/topology.proto b/proto/topology.proto index 7d19919f0b..678a209aa1 100644 --- a/proto/topology.proto +++ b/proto/topology.proto @@ -121,6 +121,7 @@ message CopysetData { repeated uint32 chunkserverIds = 4; optional bool availFlag = 5; optional uint64 lastScanSec = 6; + optional bool lastScanConsistent = 7; } // rpc struct defination @@ -455,6 +456,8 @@ message GetClusterInfoResponse { } message GetCopySetsInClusterRequest { + // Filter copysets which in scaning + optional bool filterScaning = 1; } message GetCopySetsInClusterResponse { @@ -462,6 +465,16 @@ message GetCopySetsInClusterResponse { repeated common.CopysetInfo copysetInfos = 2; } +message GetCopysetRequest { + required uint32 logicalPoolId = 1; + required uint32 copysetId = 2; +} + +message GetCopysetResponse { + required sint32 statusCode = 1; + optional common.CopysetInfo copysetInfo = 2; +} + message SetCopysetsAvailFlagRequest { required bool availFlag = 1; repeated common.CopysetInfo copysets = 2; @@ -508,11 +521,12 @@ service TopologyService { rpc GetLogicalPool(GetLogicalPoolRequest) returns (GetLogicalPoolResponse); rpc ListLogicalPool(ListLogicalPoolRequest) returns (ListLogicalPoolResponse); rpc SetLogicalPool(SetLogicalPoolRequest) returns(SetLogicalPoolResponse); - rpc SetLogicalPoolScanState(SetLogicalPoolScanStateRequest) returns(SetLogicalPoolScanStateResponse); + rpc SetLogicalPoolScanState(SetLogicalPoolScanStateRequest) returns (SetLogicalPoolScanStateResponse); rpc GetChunkServerListInCopySets(GetChunkServerListInCopySetsRequest) returns (GetChunkServerListInCopySetsResponse); rpc GetCopySetsInChunkServer(GetCopySetsInChunkServerRequest) returns (GetCopySetsInChunkServerResponse); rpc GetCopySetsInCluster(GetCopySetsInClusterRequest) returns (GetCopySetsInClusterResponse); + rpc GetCopyset(GetCopysetRequest) returns (GetCopysetResponse); rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse); rpc SetCopysetsAvailFlag(SetCopysetsAvailFlagRequest) returns (SetCopysetsAvailFlagResponse); rpc ListUnAvailCopySets(ListUnAvailCopySetsRequest) returns (ListUnAvailCopySetsResponse); diff --git a/src/mds/heartbeat/heartbeat_manager.cpp b/src/mds/heartbeat/heartbeat_manager.cpp index 75fbc2db67..dc20b050cd 100644 --- a/src/mds/heartbeat/heartbeat_manager.cpp +++ b/src/mds/heartbeat/heartbeat_manager.cpp @@ -354,9 +354,10 @@ bool HeartbeatManager::FromHeartbeatCopySetInfoToTopologyOne( topoCopysetInfo.SetScaning(info.scaning()); } - // set last scan + // set last scan and last scan consistent if (info.has_lastscansec()) { topoCopysetInfo.SetLastScanSec(info.lastscansec()); + topoCopysetInfo.SetLastScanConsistent(info.scanmap_size() == 0); } *out = topoCopysetInfo; diff --git a/src/mds/heartbeat/topo_updater.cpp b/src/mds/heartbeat/topo_updater.cpp index 8e73292ebc..154bbcb67e 100644 --- a/src/mds/heartbeat/topo_updater.cpp +++ b/src/mds/heartbeat/topo_updater.cpp @@ -136,7 +136,9 @@ void TopoUpdater::UpdateTopo(const CopySetInfo &reportCopySetInfo) { << reportCopySetInfo.GetLogicalPoolId() << "," << reportCopySetInfo.GetId() << "): scaning=" << reportCopySetInfo.GetScaning() - << ", lastScanSec=" << reportCopySetInfo.GetLastScanSec(); + << ", lastScanSec=" << reportCopySetInfo.GetLastScanSec() + << ", lastScanConsistent=" + << reportCopySetInfo.GetLastScanConsistent(); } } else if (recordCopySetInfo.GetEpoch() > reportCopySetInfo.GetEpoch()) { if (recordCopySetInfo.GetLeader() != reportCopySetInfo.GetLeader()) { diff --git a/src/mds/topology/topology.cpp b/src/mds/topology/topology.cpp index ada9d25613..595bcfd364 100644 --- a/src/mds/topology/topology.cpp +++ b/src/mds/topology/topology.cpp @@ -1106,6 +1106,7 @@ int TopologyImpl::UpdateCopySetTopo(const CopySetInfo &data) { if (data.IsLatestLastScanSec(it->second.GetLastScanSec())) { it->second.SetLastScanSec(data.GetLastScanSec()); + it->second.SetLastScanConsistent(data.GetLastScanConsistent()); } it->second.SetDirtyFlag(true); diff --git a/src/mds/topology/topology_item.cpp b/src/mds/topology/topology_item.cpp index 2200f5a3b9..951ca03d23 100644 --- a/src/mds/topology/topology_item.cpp +++ b/src/mds/topology/topology_item.cpp @@ -335,6 +335,7 @@ bool CopySetInfo::SerializeToString(std::string *value) const { } data.set_availflag(available_); data.set_lastscansec(lastScanSec_); + data.set_lastscanconsistent(lastScanConsistent_); return data.SerializeToString(value); } @@ -354,6 +355,8 @@ bool CopySetInfo::ParseFromString(const std::string &value) { peers_.insert(data.chunkserverids(i)); } lastScanSec_ = data.has_lastscansec() ? data.lastscansec() : 0; + lastScanConsistent_ = data.has_lastscanconsistent() ? + data.lastscanconsistent() : true; return ret; } diff --git a/src/mds/topology/topology_item.h b/src/mds/topology/topology_item.h index 049e9f8ab1..85406ab896 100644 --- a/src/mds/topology/topology_item.h +++ b/src/mds/topology/topology_item.h @@ -687,6 +687,7 @@ class CopySetInfo { hasScaning_(false), lastScanSec_(0), hasLastScanSec_(false), + lastScanConsistent_(true), dirty_(false), available_(true) {} @@ -702,6 +703,7 @@ class CopySetInfo { hasScaning_(false), lastScanSec_(0), hasLastScanSec_(false), + lastScanConsistent_(true), dirty_(false), available_(true) {} @@ -717,6 +719,7 @@ class CopySetInfo { hasScaning_(v.hasScaning_), lastScanSec_(v.lastScanSec_), hasLastScanSec_(v.hasLastScanSec_), + lastScanConsistent_(v.lastScanConsistent_), dirty_(v.dirty_), available_(v.available_) {} @@ -735,6 +738,7 @@ class CopySetInfo { hasScaning_ = v.hasScaning_; lastScanSec_ = v.lastScanSec_; hasLastScanSec_ = v.hasLastScanSec_; + lastScanConsistent_ = v.lastScanConsistent_; dirty_ = v.dirty_; available_ = v.available_; return *this; @@ -827,6 +831,14 @@ class CopySetInfo { return lastScanSec_; } + bool SetLastScanConsistent(bool lastScanConsistent) { + lastScanConsistent_ = lastScanConsistent; + } + + bool GetLastScanConsistent() const { + return lastScanConsistent_; + } + void ClearCandidate() { hasCandidate_ = false; } @@ -876,6 +888,9 @@ class CopySetInfo { // whether the lastScanSec_ has been set bool hasLastScanSec_; + // whether the data of copies is consistent for last scan + bool lastScanConsistent_; + /** * @brief to mark whether data is dirty, for writing to storage regularly */ diff --git a/src/mds/topology/topology_service.cpp b/src/mds/topology/topology_service.cpp index 1e99724511..938ace00f9 100644 --- a/src/mds/topology/topology_service.cpp +++ b/src/mds/topology/topology_service.cpp @@ -839,31 +839,60 @@ void TopologyServiceImpl::GetCopySetsInChunkServer( } void TopologyServiceImpl::GetCopySetsInCluster( - google::protobuf::RpcController* cntl_base, - const GetCopySetsInClusterRequest* request, - GetCopySetsInClusterResponse* response, - google::protobuf::Closure* done) { + google::protobuf::RpcController* cntl_base, + const GetCopySetsInClusterRequest* request, + GetCopySetsInClusterResponse* response, + google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); - brpc::Controller* cntl = - static_cast(cntl_base); + brpc::Controller* cntl = static_cast(cntl_base); + + auto localAddr = cntl->local_side(); + auto remoteAddr = cntl->remote_side(); + LOG(INFO) + << "Received request[log_id=" << cntl->log_id() + << "] from " << remoteAddr << " to " << localAddr + << ". [GetCopySetsInCluster] " << request->DebugString(); - LOG(INFO) << "Received request[log_id=" << cntl->log_id() - << "] from " << cntl->remote_side() - << " to " << cntl->local_side() - << ". [GetCopySetsInClusterRequest]"; topology_->GetCopySetsInCluster(request, response); - if (kTopoErrCodeSuccess != response->statuscode()) { - LOG(ERROR) << "Send response[log_id=" << cntl->log_id() - << "] from " << cntl->local_side() - << " to " << cntl->remote_side() - << ". [GetCopySetsInClusterResponse] " - << response->DebugString(); + + std::ostringstream errMsg; + errMsg << "Send response[log_id=" << cntl->log_id() + << "] from " << localAddr << " to " << remoteAddr + << ". [GetCopySetsInCluster] " << response->DebugString(); + + if (response->statuscode() == kTopoErrCodeSuccess) { + LOG(INFO) << errMsg.str(); } else { - LOG(INFO) << "Send response[log_id=" << cntl->log_id() - << "] from " << cntl->local_side() - << " to " << cntl->remote_side() - << ". [GetCopySetsInClusterResponse] copyset num: " - << response->copysetinfos_size(); + LOG(ERROR) << errMsg.str(); + } +} + +void TopologyServiceImpl::GetCopyset( + google::protobuf::RpcController* cntl_base, + const GetCopysetRequest* request, + GetCopysetResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + brpc::Controller* cntl = static_cast(cntl_base); + + auto localAddr = cntl->local_side(); + auto remoteAddr = cntl->remote_side(); + LOG(INFO) + << "Received request[log_id=" << cntl->log_id() + << "] from " << remoteAddr << " to " << localAddr + << ". [GetCopyset] " << request->DebugString(); + + topology_->GetCopyset(request, response); + + std::ostringstream errMsg; + errMsg << "Send response[log_id=" << cntl->log_id() + << "] from " << localAddr << " to " << remoteAddr + << ". [GetCopyset] " << response->DebugString(); + + if (response->statuscode() == kTopoErrCodeSuccess) { + LOG(INFO) << errMsg.str(); + } else { + LOG(ERROR) << errMsg.str(); } } diff --git a/src/mds/topology/topology_service.h b/src/mds/topology/topology_service.h index 6c2b00582a..971edb296b 100644 --- a/src/mds/topology/topology_service.h +++ b/src/mds/topology/topology_service.h @@ -174,10 +174,15 @@ class TopologyServiceImpl : public TopologyService { google::protobuf::Closure* done); virtual void GetCopySetsInCluster( - google::protobuf::RpcController* cntl_base, - const GetCopySetsInClusterRequest* request, - GetCopySetsInClusterResponse* response, - google::protobuf::Closure* done); + google::protobuf::RpcController* cntl_base, + const GetCopySetsInClusterRequest* request, + GetCopySetsInClusterResponse* response, + google::protobuf::Closure* done); + + virtual void GetCopyset(google::protobuf::RpcController* cntl_base, + const GetCopysetRequest* request, + GetCopysetResponse* response, + google::protobuf::Closure* done); virtual void GetClusterInfo( google::protobuf::RpcController* cntl_base, diff --git a/src/mds/topology/topology_service_manager.cpp b/src/mds/topology/topology_service_manager.cpp index 04b4e876fd..e71953c80d 100644 --- a/src/mds/topology/topology_service_manager.cpp +++ b/src/mds/topology/topology_service_manager.cpp @@ -31,6 +31,7 @@ #include #include //NOLINT #include //NOLINT +#include #include "brpc/channel.h" #include "brpc/controller.h" @@ -1399,16 +1400,45 @@ void TopologyServiceManager::GetCopySetsInChunkServer( } void TopologyServiceManager::GetCopySetsInCluster( - const GetCopySetsInClusterRequest* request, - GetCopySetsInClusterResponse* response) { + const GetCopySetsInClusterRequest* request, + GetCopySetsInClusterResponse* response) { + auto filter = [&](const CopySetInfo& copysetInfo) { + if (request->has_filterscaning() && !copysetInfo.GetScaning()) { + return false; + } + + return true; + }; + + auto logicalPoolIds = topology_->GetLogicalPoolInCluster(); + std::sort(logicalPoolIds.begin(), logicalPoolIds.end()); + for (const auto& lpid : logicalPoolIds) { + auto copysetInfos = + topology_-> GetCopySetInfosInLogicalPool(lpid, filter); + for (auto& copysetInfo : copysetInfos) { + CopysetInfo* info = response->add_copysetinfos(); + ConvertCopyset(copysetInfo, info); + } + } + response->set_statuscode(kTopoErrCodeSuccess); - std::vector copysets = - topology_->GetCopySetsInCluster(); - for (const CopySetKey& copyset : copysets) { - CopysetInfo *info = response->add_copysetinfos(); - info->set_logicalpoolid(copyset.first); - info->set_copysetid(copyset.second); +} + +void TopologyServiceManager::GetCopyset(const GetCopysetRequest* request, + GetCopysetResponse* response) { + CopySetInfo copysetInfo; + auto lpid = request->logicalpoolid(); + auto copysetId = request->copysetid(); + CopySetKey copysetKey(lpid, copysetId); + + if (!topology_->GetCopySet(copysetKey, ©setInfo)) { + response->set_statuscode(kTopoErrCodeCopySetNotFound); + return; } + + response->set_statuscode(kTopoErrCodeSuccess); + auto info = response->mutable_copysetinfo(); + ConvertCopyset(copysetInfo, info); } void TopologyServiceManager::GetClusterInfo( @@ -1458,6 +1488,15 @@ void TopologyServiceManager::ListUnAvailCopySets( response->set_statuscode(kTopoErrCodeSuccess); } +void TopologyServiceManager::ConvertCopyset(const CopySetInfo& in, + ::curve::common::CopysetInfo* out) { + out->set_logicalpoolid(in.GetLogicalPoolId()); + out->set_copysetid(in.GetId()); + out->set_scaning(in.GetScaning()); + out->set_lastscansec(in.GetLastScanSec()); + out->set_lastscanconsistent(in.GetLastScanConsistent()); +} + } // namespace topology } // namespace mds } // namespace curve diff --git a/src/mds/topology/topology_service_manager.h b/src/mds/topology/topology_service_manager.h index efda035c3a..2ee4c5e628 100644 --- a/src/mds/topology/topology_service_manager.h +++ b/src/mds/topology/topology_service_manager.h @@ -132,8 +132,11 @@ class TopologyServiceManager { GetCopySetsInChunkServerResponse* response); virtual void GetCopySetsInCluster( - const GetCopySetsInClusterRequest* request, - GetCopySetsInClusterResponse* response); + const GetCopySetsInClusterRequest* request, + GetCopySetsInClusterResponse* response); + + virtual void GetCopyset(const GetCopysetRequest* request, + GetCopysetResponse* response); virtual void GetClusterInfo( const GetClusterInfoRequest* request, @@ -212,6 +215,14 @@ class TopologyServiceManager { int RemoveErrLogicalPoolAndCopyset(const LogicalPool &pool, const std::vector *copysetInfos); + /** + * @brief Convert topology CopySetInfo to proto CopysetInfo + * @param[in] in the topology CopySetInfo + * @param[in] out the proto CopysetInfo + */ + void ConvertCopyset(const CopySetInfo& in, + ::curve::common::CopysetInfo* out); + private: /** * @brief topology module diff --git a/src/tools/copyset_check_core.cpp b/src/tools/copyset_check_core.cpp index 269c2eb73c..0a76c74571 100644 --- a/src/tools/copyset_check_core.cpp +++ b/src/tools/copyset_check_core.cpp @@ -433,6 +433,7 @@ int CopysetCheckCore::CheckCopysetsWithMds() { std::cout << "GetCopySetsInCluster fail!" << std::endl; return -1; } + CopysetStatistics statistics = GetCopysetStatistics(); if (copysetsInMds.size() != copysets_[kTotal].size()) { std::cout << "Copyset numbers in chunkservers not consistent" @@ -476,9 +477,36 @@ int CopysetCheckCore::CheckCopysetsWithMds() { std::cout << std::endl; ret = -1; } + + // Check scan status for inconsistent copyset + auto nInconsistent = CheckScanStatus(copysetsInMds); + if (nInconsistent > 0) { + std::cout << "There are " << nInconsistent << " inconsistent copyset" + << std::endl; + ret = -1; + } + return ret; } +int CopysetCheckCore::CheckScanStatus( + const std::vector& copysetInfos) { + int count = 0; + for (auto& copysetInfo : copysetInfos) { + if (!copysetInfo.has_lastscanconsistent() || + copysetInfo.lastscanconsistent()) { + continue; + } + + auto groupId = ToGroupId(copysetInfo.logicalpoolid(), + copysetInfo.copysetid()); + copysets_[kThreeCopiesInconsistent].emplace(groupId); + count++; + } + + return count; +} + int CopysetCheckCore::CheckOperator(const std::string& opName, uint64_t checkTimeSec) { uint64_t startTime = curve::common::TimeUtility::GetTimeofDaySec(); diff --git a/src/tools/copyset_check_core.h b/src/tools/copyset_check_core.h index ad6e1b0e5d..1da63ec0cd 100644 --- a/src/tools/copyset_check_core.h +++ b/src/tools/copyset_check_core.h @@ -101,6 +101,7 @@ const char kLogIndexGapTooBig[] = "index gap too big"; const char kPeersNoSufficient[] = "peers not sufficient"; const char kMinorityPeerNotOnline[] = "minority peer not online"; const char kMajorityPeerNotOnline[] = "majority peer not online"; +const char kThreeCopiesInconsistent[] = "Three copies inconsistent"; class CopysetCheckCore { public: @@ -407,6 +408,8 @@ class CopysetCheckCore { int CheckCopysetsWithMds(); + int CheckScanStatus(const std::vector& copysetInfos); + private: // 向mds发送RPC的client std::shared_ptr mdsClient_; diff --git a/src/tools/curve_tool_define.h b/src/tools/curve_tool_define.h index 5bd9984857..ac7d474f9b 100644 --- a/src/tools/curve_tool_define.h +++ b/src/tools/curve_tool_define.h @@ -53,6 +53,7 @@ const char kClientStatusCmd[] = "client-status"; const char kClientListCmd[] = "client-list"; const char kSnapshotCloneStatusCmd[] = "snapshot-clone-status"; const char kClusterStatusCmd[] = "cluster-status"; +const char kScanStatusCmd[] = "scan-status"; // NamesPaceTool相关命令 const char kGetCmd[] = "get"; diff --git a/src/tools/curve_tool_main.cpp b/src/tools/curve_tool_main.cpp index 0a1eea6307..0578078de9 100644 --- a/src/tools/curve_tool_main.cpp +++ b/src/tools/curve_tool_main.cpp @@ -60,7 +60,8 @@ const char* kHelpStr = "Usage: curve_ops_tool [Command] [OPTIONS...]\n" "set-copyset-availflag: set copysets available flags\n" "update-throttle: update file throttle params\n" "rapid-leader-schedule: rapid leader schedule in cluster in logicalpool\n" //NOLINT - "set-scan-state: set scan state for specify logical pool\n\n" + "set-scan-state: set scan state for specify logical pool\n" + "scan-status: show scan status\n\n" "You can specify the config path by -confPath to avoid typing too many options\n"; //NOLINT diff --git a/src/tools/mds_client.cpp b/src/tools/mds_client.cpp index 527b235b02..9fc1bc5539 100644 --- a/src/tools/mds_client.cpp +++ b/src/tools/mds_client.cpp @@ -736,12 +736,17 @@ int MDSClient::GetCopySetsInChunkServer( return -1; } -int MDSClient::GetCopySetsInCluster(std::vector* copysets) { +int MDSClient::GetCopySetsInCluster(std::vector* copysets, + bool filterScaning) { assert(copysets != nullptr); curve::mds::topology::GetCopySetsInClusterRequest request; curve::mds::topology::GetCopySetsInClusterResponse response; curve::mds::topology::TopologyService_Stub stub(&channel_); + if (filterScaning) { + request.set_filterscaning(true); + } + auto fp = &curve::mds::topology::TopologyService_Stub::GetCopySetsInCluster; if (SendRpcToMds(&request, &response, &stub, fp) != 0) { std::cout << "GetCopySetsInCluster from all mds fail!" @@ -761,6 +766,32 @@ int MDSClient::GetCopySetsInCluster(std::vector* copysets) { return -1; } + +int MDSClient::GetCopyset(PoolIdType lpid, + CopySetIdType copysetId, + CopysetInfo* copysetInfo) { + curve::mds::topology::GetCopysetRequest request; + curve::mds::topology::GetCopysetResponse response; + curve::mds::topology::TopologyService_Stub stub(&channel_); + + request.set_logicalpoolid(lpid); + request.set_copysetid(copysetId); + auto fn = &curve::mds::topology::TopologyService_Stub::GetCopyset; + if (SendRpcToMds(&request, &response, &stub, fn) != 0) { + std::cout << "GetCopyset from all mds fail!" << std::endl; + return -1; + } + + auto retCode = response.statuscode(); + if (retCode == kTopoErrCodeSuccess) { + *copysetInfo = response.copysetinfo(); + return 0; + } + + std::cout << "GetCopyset fail with retCode: " << retCode << std::endl; + return -1; +} + int MDSClient::ListServersInCluster(std::vector* servers) { assert(servers != nullptr); // list physicalpools diff --git a/src/tools/mds_client.h b/src/tools/mds_client.h index dbd2001f10..7e9074b122 100644 --- a/src/tools/mds_client.h +++ b/src/tools/mds_client.h @@ -313,11 +313,24 @@ class MDSClient { std::vector* copysets); /** - * @brief 获取集群中的所有copyset - * @param[out] copysets 集群中copyset的列表 - * @return 成功返回0，失败返回-1 + * @brief Get all copysets in cluster + * @param[out] the copyset list + * @param[in] filterScaning whether need to filter copyset which in scaning + * @return 0 if success, else return -1 + */ + virtual int GetCopySetsInCluster(std::vector* copysetInfos, + bool filterScaning = false); + + /** + * @brief Get specify copyset + * @param[in] lpid logical pool id + * @param[in] copysetId copyset id + * @param[out] copysetInfo the copyset + * @return 0 if success, else return -1 */ - virtual int GetCopySetsInCluster(std::vector* copysets); + virtual int GetCopyset(PoolIdType lpid, + CopySetIdType copysetId, + CopysetInfo* copysetInfo); /** * @brief 列出集群中的所有server diff --git a/src/tools/status_tool.cpp b/src/tools/status_tool.cpp index a29d795d9d..31d1aca86b 100644 --- a/src/tools/status_tool.cpp +++ b/src/tools/status_tool.cpp @@ -114,7 +114,8 @@ bool StatusTool::SupportCommand(const std::string& command) { || command == kSnapshotCloneStatusCmd || command == kClusterStatusCmd || command == kServerListCmd - || command == kLogicalPoolList); + || command == kLogicalPoolList + || command == kScanStatusCmd); } void StatusTool::PrintHelp(const std::string& cmd) { @@ -132,18 +133,20 @@ void StatusTool::PrintHelp(const std::string& cmd) { std::cout << " [-snapshotCloneAddr=127.0.0.1:5555]" << " [-confPath=/etc/curve/tools.conf]"; } + if (cmd == kChunkserverListCmd) { std::cout << " [-offline] [-unhealthy] [-checkHealth=false]" << " [-confPath=/etc/curve/tools.conf]" << " [-checkCSAlive]"; - } - if (cmd == kClientStatusCmd) { + } else if (cmd == kClientStatusCmd) { std::cout << " [-detail] [-confPath=/etc/curve/tools.conf]"; - } - if (cmd == kClientListCmd) { + } else if (cmd == kClientListCmd) { std::cout << " [-listClientInRepo=false]" << " [-confPath=/etc/curve/tools.conf]"; + } else if (cmd == kScanStatusCmd) { + std::cout << " [-logicalPoolId=1] [-copysetId=1]" << std::endl; } + std::cout << std::endl; } @@ -696,6 +699,49 @@ int StatusTool::ClientListCmd() { return 0; } +int StatusTool::ScanStatusCmd() { + if (FLAGS_logicalPoolId != 0 && FLAGS_copysetId != 0) { + CopysetInfo copysetInfo; + auto lpid = FLAGS_logicalPoolId; + auto copysetId = FLAGS_copysetId; + if (mdsClient_->GetCopyset(lpid, copysetId, ©setInfo) != 0) { + std::cout << "GetCopyset fail!" << std::endl; + return -1; + } + + std::cout + << "Scan status for copyset(" + << lpid << "," << copysetId << "):" << std::endl + << " scaning=" << copysetInfo.scaning() + << " lastScanSec=" << copysetInfo.lastscansec() + << " lastScanConsistent=" << copysetInfo.lastscanconsistent() + << std::endl; + + return 0; + } + + std::vector copysetInfos; + if (mdsClient_->GetCopySetsInCluster(©setInfos, true) != 0) { + std::cout << "GetCopySetsInCluster fail!" << std::endl; + return -1; + } + + int count = 0; + std::cout << "Scaning copysets: " << copysetInfos.size(); + for (auto& copysetInfo : copysetInfos) { + if (count % 5 == 0) { + std::cout << std::endl; + } + std::cout << " (" << copysetInfo.logicalpoolid() + << "," << copysetInfo.copysetid() << ")"; + count++; + } + + std::cout << std::endl; + + return 0; +} + int CheckUseWalPool(const std::map> &poolChunkservers, bool *useWalPool, @@ -1051,6 +1097,8 @@ int StatusTool::RunCommand(const std::string &cmd) { return PrintClusterStatus(); } else if (cmd == kClientListCmd) { return ClientListCmd(); + } else if (cmd == kScanStatusCmd) { + return ScanStatusCmd(); } else { std::cout << "Command not supported!" << std::endl; return -1; diff --git a/src/tools/status_tool.h b/src/tools/status_tool.h index 292c5abf9a..1cfdf0c115 100644 --- a/src/tools/status_tool.h +++ b/src/tools/status_tool.h @@ -139,6 +139,7 @@ class StatusTool : public CurveTool { int PrintChunkserverStatus(bool checkLeftSize = true); int PrintClientStatus(); int ClientListCmd(); + int ScanStatusCmd(); void PrintCsLeftSizeStatistics(const std::string& name, const std::map>& poolLeftSize); diff --git a/test/mds/topology/mock_topology.h b/test/mds/topology/mock_topology.h index cd85063c80..74bef7cc92 100644 --- a/test/mds/topology/mock_topology.h +++ b/test/mds/topology/mock_topology.h @@ -262,9 +262,12 @@ class MockTopologyServiceManager : public TopologyServiceManager { const GetCopySetsInChunkServerRequest *request, GetCopySetsInChunkServerResponse *response)); - MOCK_METHOD2(GetCopySetsInCluster, void( - const GetCopySetsInClusterRequest *request, - GetCopySetsInClusterResponse *response)); + MOCK_METHOD2(GetCopySetsInCluster, + void(const GetCopySetsInClusterRequest* request, + GetCopySetsInClusterResponse* response)); + + MOCK_METHOD2(GetCopyset, void(const GetCopysetRequest* request, + GetCopysetResponse* response)); MOCK_METHOD2(GetClusterInfo, void(const GetClusterInfoRequest* request, diff --git a/test/mds/topology/test_topology_service.cpp b/test/mds/topology/test_topology_service.cpp index 10d1d9d922..eef943e393 100644 --- a/test/mds/topology/test_topology_service.cpp +++ b/test/mds/topology/test_topology_service.cpp @@ -1417,7 +1417,7 @@ TEST_F(TestTopologyService, TestSetLogicalPoolScanState) { SetLogicalPoolScanStateResponse resp; resp.set_statuscode(kTopoErrCodeSuccess); EXPECT_CALL(*manager_, SetLogicalPoolScanState(_, _)) - .WillRepeatedly(SetArgPointee<1>(resp)); + .WillOnce(SetArgPointee<1>(resp)); request.set_logicalpoolid(1); request.set_scanenable(true); @@ -1432,7 +1432,7 @@ TEST_F(TestTopologyService, TestSetLogicalPoolScanState) { SetLogicalPoolScanStateResponse resp; resp.set_statuscode(kTopoErrCodeLogicalPoolNotFound); EXPECT_CALL(*manager_, SetLogicalPoolScanState(_, _)) - .WillRepeatedly(SetArgPointee<1>(resp)); + .WillOnce(SetArgPointee<1>(resp)); request.set_logicalpoolid(1); request.set_scanenable(true); @@ -1614,6 +1614,45 @@ TEST_F(TestTopologyService, test_GetCopySetsInCluster_fail) { ASSERT_EQ(kTopoErrCodeInvalidParam, response.statuscode()); } +TEST_F(TestTopologyService, TestGetCopyset) { + brpc::Channel channel; + brpc::Controller cntl; + GetCopysetRequest request; + GetCopysetResponse response; + + ASSERT_EQ(channel.Init(listenAddr_, NULL), 0); + TopologyService_Stub stub(&channel); + + // CASE 1: Get copyset success + { + GetCopysetResponse resp; + resp.set_statuscode(kTopoErrCodeSuccess); + EXPECT_CALL(*manager_, GetCopyset(_, _)) + .WillOnce(SetArgPointee<1>(resp)); + + request.set_logicalpoolid(1); + request.set_copysetid(1); + stub.GetCopyset(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(response.statuscode(), kTopoErrCodeSuccess); + } + + // CASE 2: Get copyset fail with copyset not found + { + cntl.Reset(); + GetCopysetResponse resp; + resp.set_statuscode(kTopoErrCodeCopySetNotFound); + EXPECT_CALL(*manager_, GetCopyset(_, _)) + .WillOnce(SetArgPointee<1>(resp)); + + request.set_logicalpoolid(1); + request.set_copysetid(1); + stub.GetCopyset(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(response.statuscode(), kTopoErrCodeCopySetNotFound); + } +} + TEST_F(TestTopologyService, test_SetCopysetsAvailFlag_success) { brpc::Channel channel; if (channel.Init(listenAddr_, NULL) != 0) { diff --git a/test/mds/topology/test_topology_service_manager.cpp b/test/mds/topology/test_topology_service_manager.cpp index dfc7e760a4..97c343f169 100644 --- a/test/mds/topology/test_topology_service_manager.cpp +++ b/test/mds/topology/test_topology_service_manager.cpp @@ -196,15 +196,20 @@ class TestTopologyServiceManager : public ::testing::Test { } void PrepareAddCopySet(CopySetIdType copysetId, - PoolIdType logicalPoolId, - const std::set &members) { - CopySetInfo cs(logicalPoolId, - copysetId); - cs.SetCopySetMembers(members); + PoolIdType logicalPoolId, + const std::set &members, + bool scaning = false, + LastScanSecType lastScanSec = 0, + bool lastScanConsistent = true) { + CopySetInfo copysetInfo(logicalPoolId, copysetId); + copysetInfo.SetCopySetMembers(members); + copysetInfo.SetScaning(scaning); + copysetInfo.SetLastScanSec(lastScanSec); + copysetInfo.SetLastScanConsistent(lastScanConsistent); + EXPECT_CALL(*storage_, StorageCopySet(_)) .WillOnce(Return(true)); - int ret = topology_->AddCopySet(cs); - ASSERT_EQ(kTopoErrCodeSuccess, ret) + ASSERT_EQ(topology_->AddCopySet(copysetInfo), kTopoErrCodeSuccess) << "should have PrepareAddLogicalPool()"; } @@ -2582,7 +2587,6 @@ TEST_F(TestTopologyServiceManager, TestSetLogicalPoolScanState) { SetLogicalPoolScanStateRequest request; SetLogicalPoolScanStateResponse response; - // CASE 1: logical pool not found request.set_logicalpoolid(lpid + 1); request.set_scanenable(true); @@ -2844,44 +2848,96 @@ TEST_F(TestTopologyServiceManager, ASSERT_EQ(0, response.copysetinfos_size()); } -TEST_F(TestTopologyServiceManager, test_GetCopySetsInCluster) { - PoolIdType logicalPoolId1 = 0x1; - PoolIdType physicalPoolId1 = 0x11; - PrepareAddPhysicalPool(physicalPoolId1); - PrepareAddLogicalPool(logicalPoolId1, "logicalPool1", physicalPoolId1); - PoolIdType logicalPoolId2 = 0x2; - PoolIdType physicalPoolId2 = 0x12; - PrepareAddPhysicalPool(physicalPoolId2); - PrepareAddLogicalPool(logicalPoolId2, "logicalPool2", physicalPoolId2); +TEST_F(TestTopologyServiceManager, TestGetCopySetsInCluster) { + PoolIdType ppid1 = 1, ppid2 = 2; // physical pool id + PoolIdType lpid1 = 1, lpid2 = 2; // logical pool id + PrepareAddPhysicalPool(ppid1); + PrepareAddPhysicalPool(ppid2); + PrepareAddLogicalPool(lpid1, "logicalPool", ppid1); + PrepareAddLogicalPool(lpid2, "logicalPool", ppid2); - std::set members = {1, 2, 3}; - for (int i = 1; i <= 10; ++i) { - PrepareAddCopySet(i, logicalPoolId1, members); - } - for (int i = 11; i <= 20; ++i) { - PrepareAddCopySet(i, logicalPoolId2, members); + std::set peers = { 1, 2, 3 }; + for (auto i = 1; i <= 20; i++) { + auto lpid = (i <= 10 ? lpid1 : lpid2); + // copysetId, logicalPoolid, chunkservers, scaning + PrepareAddCopySet(i, lpid, peers, i % 2 == 0); } - GetCopySetsInClusterRequest request; - GetCopySetsInClusterResponse response; - serviceManager_->GetCopySetsInCluster(&request, &response); + // CASE 1: GetCopySetsInCluster without filter + { + GetCopySetsInClusterRequest request; + GetCopySetsInClusterResponse response; + serviceManager_->GetCopySetsInCluster(&request, &response); + + ASSERT_EQ(response.statuscode(), kTopoErrCodeSuccess); + ASSERT_EQ(response.copysetinfos_size(), 20); + for (auto i = 1; i <= 20; i++) { + auto copysetInfo = response.copysetinfos(i - 1); + ASSERT_EQ(copysetInfo.logicalpoolid(), i <= 10 ? lpid1 : lpid2); + ASSERT_EQ(copysetInfo.copysetid(), i); + ASSERT_EQ(copysetInfo.scaning(), i % 2 == 0); + } + } - ASSERT_EQ(kTopoErrCodeSuccess, response.statuscode()); - ASSERT_EQ(20, response.copysetinfos_size()); - for (int i = 0; i < 20; i++) { - if (i < 10) { - ASSERT_EQ(1, response.copysetinfos(i).logicalpoolid()); - } else { - ASSERT_EQ(2, response.copysetinfos(i).logicalpoolid()); + // CASE 2: GetCopySetsInCluster with filter scaning copysets + { + GetCopySetsInClusterRequest request; + GetCopySetsInClusterResponse response; + request.set_filterscaning(true); + serviceManager_->GetCopySetsInCluster(&request, &response); + + ASSERT_EQ(response.statuscode(), kTopoErrCodeSuccess); + ASSERT_EQ(response.copysetinfos_size(), 10); + for (auto i = 1; i <= 10; i++) { + auto copysetInfo = response.copysetinfos(i - 1); + ASSERT_EQ(copysetInfo.logicalpoolid(), i * 2 <= 10 ? lpid1 : lpid2); + ASSERT_EQ(copysetInfo.copysetid(), i * 2); + ASSERT_TRUE(copysetInfo.scaning()); } - ASSERT_EQ(i + 1, response.copysetinfos(i).copysetid()); } - GetCopySetsInClusterResponse response2; - serviceManager_->GetCopySetsInCluster(&request, &response2); +} - ASSERT_EQ(kTopoErrCodeSuccess, response2.statuscode()); - ASSERT_EQ(20, response2.copysetinfos_size()); - ASSERT_EQ(1, response2.copysetinfos(0).copysetid()); +TEST_F(TestTopologyServiceManager, TestGetCopyset) { + PoolIdType ppid = 1; // physical pool id + PoolIdType lpid = 1; // logical pool id + PrepareAddPhysicalPool(ppid); + PrepareAddLogicalPool(lpid, "logicalPool", ppid); + + std::set peers = { 1, 2, 3 }; + // copysetId, logicalPoolid, chunkservers, + // scaning, lastScanSec, lastScanConsistent + PrepareAddCopySet(1, lpid, peers, false, 123456789, false); + + // CASE 1: Get copyset success + { + GetCopysetRequest request; + GetCopysetResponse response; + request.set_logicalpoolid(1); + request.set_copysetid(1); + serviceManager_->GetCopyset(&request, &response); + + ASSERT_EQ(response.statuscode(), kTopoErrCodeSuccess); + ASSERT_EQ(response.copysetinfo().logicalpoolid(), 1); + ASSERT_EQ(response.copysetinfo().copysetid(), 1); + ASSERT_EQ(response.copysetinfo().scaning(), false); + ASSERT_EQ(response.copysetinfo().lastscansec(), 123456789); + ASSERT_EQ(response.copysetinfo().lastscanconsistent(), false); + } + + // CASE 2: Get copyset fail with copyset not found + { + GetCopysetRequest request; + GetCopysetResponse response; + request.set_logicalpoolid(1); + request.set_copysetid(2); + serviceManager_->GetCopyset(&request, &response); + ASSERT_EQ(response.statuscode(), kTopoErrCodeCopySetNotFound); + + request.set_logicalpoolid(2); + request.set_copysetid(1); + serviceManager_->GetCopyset(&request, &response); + ASSERT_EQ(response.statuscode(), kTopoErrCodeCopySetNotFound); + } } TEST_F(TestTopologyServiceManager, test_SetCopysetsAvailFlag) { diff --git a/test/tools/copyset_check_core_test.cpp b/test/tools/copyset_check_core_test.cpp index fad978e61b..9ef6de55ce 100644 --- a/test/tools/copyset_check_core_test.cpp +++ b/test/tools/copyset_check_core_test.cpp @@ -805,7 +805,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterNormal) { copyset.set_logicalpoolid(1); copyset.set_copysetid(100); copysetsInMds.emplace_back(copyset); - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(1) .WillRepeatedly(DoAll(SetArgPointee<0>(copysetsInMds), Return(0))); @@ -844,7 +844,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) { .Times(1) .WillOnce(Return(-1)); std::vector copysetsInMds; - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(1) .WillRepeatedly(DoAll(SetArgPointee<0>(copysetsInMds), Return(0))); @@ -880,7 +880,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) { copyset.set_logicalpoolid(1); copyset.set_copysetid(100); copysetsInMds.emplace_back(copyset); - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(2) .WillRepeatedly(DoAll(SetArgPointee<0>(copysetsInMds), Return(0))); @@ -912,7 +912,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) { .WillRepeatedly(DoAll(SetArgPointee<0>(iobuf), Return(0))); // 从获取copyset失败 - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(1) .WillRepeatedly(Return(-1)); ASSERT_EQ(-1, copysetCheck4.CheckCopysetsInCluster()); @@ -924,7 +924,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) { copysetsInMds.emplace_back(copyset); copyset.set_copysetid(100); copysetsInMds.emplace_back(copyset); - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(1) .WillRepeatedly(DoAll(SetArgPointee<0>(copysetsInMds), Return(0))); @@ -932,7 +932,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) { ASSERT_EQ(0, copysetCheck4.GetCopysetStatistics().unhealthyRatio); // copyset数量一致，但是内容不一致 copysetsInMds.pop_back(); - EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_)) + EXPECT_CALL(*mdsClient_, GetCopySetsInCluster(_, _)) .Times(1) .WillRepeatedly(DoAll(SetArgPointee<0>(copysetsInMds), Return(0))); diff --git a/test/tools/curve_tool_factory_test.cpp b/test/tools/curve_tool_factory_test.cpp index 6d8dd4775a..99913083e9 100644 --- a/test/tools/curve_tool_factory_test.cpp +++ b/test/tools/curve_tool_factory_test.cpp @@ -51,6 +51,8 @@ TEST(CurveToolFactoryTest, GetStatusTool) { ASSERT_TRUE(dynamic_cast(curveTool.get()) != nullptr); curveTool = CurveToolFactory::GenerateCurveTool("cluster-status"); ASSERT_TRUE(dynamic_cast(curveTool.get()) != nullptr); + curveTool = CurveToolFactory::GenerateCurveTool("scan-status"); + ASSERT_TRUE(dynamic_cast(curveTool.get()) != nullptr); curveTool = CurveToolFactory::GenerateCurveTool("nothing"); ASSERT_TRUE(curveTool.get() == nullptr); } diff --git a/test/tools/mds_client_test.cpp b/test/tools/mds_client_test.cpp index 72645088b3..4543bf717c 100644 --- a/test/tools/mds_client_test.cpp +++ b/test/tools/mds_client_test.cpp @@ -42,6 +42,8 @@ using curve::mds::topology::GetCopySetsInChunkServerRequest; using curve::mds::topology::GetCopySetsInChunkServerResponse; using curve::mds::topology::GetCopySetsInClusterRequest; using curve::mds::topology::GetCopySetsInClusterResponse; +using curve::mds::topology::GetCopysetRequest; +using curve::mds::topology::GetCopysetResponse; using curve::mds::topology::SetCopysetsAvailFlagRequest; using curve::mds::topology::SetCopysetsAvailFlagResponse; using curve::mds::topology::ListUnAvailCopySetsRequest; @@ -1083,6 +1085,64 @@ TEST_F(ToolMDSClientTest, GetCopySetsInCluster) { } } +TEST_F(ToolMDSClientTest, GetCopyset) { + auto succCallback = callback; + auto failCallback = [](RpcController* controller, + const GetCopysetRequest* request, + GetCopysetResponse* response, + Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller *cntl = + dynamic_cast(controller); + cntl->SetFailed("fail"); + }; + + GetCopysetResponse succResp, failResp; + + succResp.set_statuscode(curve::mds::topology::kTopoErrCodeSuccess); + auto copysetInfo = succResp.mutable_copysetinfo(); + copysetInfo->set_logicalpoolid(1); + copysetInfo->set_copysetid(1); + copysetInfo->set_scaning(true); + copysetInfo->set_lastscansec(123456789); + copysetInfo->set_lastscanconsistent(false); + + failResp.set_statuscode(curve::mds::topology::kTopoErrCodeCopySetNotFound); + + // CASE 1: Send rpc failed + { + CopysetInfo copysetInfo; + EXPECT_CALL(*topoService, GetCopyset(_, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(succResp), + Invoke(failCallback))); + ASSERT_EQ(mdsClient.GetCopyset(1, 1, ©setInfo), -1); + } + + // CASE 2: copyset not found -> GetCopyset fail + { + CopysetInfo copysetInfo; + EXPECT_CALL(*topoService, GetCopyset(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(failResp), + Invoke(succCallback))); + ASSERT_EQ(mdsClient.GetCopyset(1, 1, ©setInfo), -1); + } + + // CASE 3: GetCopyset success + { + CopysetInfo copysetInfo; + EXPECT_CALL(*topoService, GetCopyset(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(succResp), + Invoke(succCallback))); + ASSERT_EQ(mdsClient.GetCopyset(1, 1, ©setInfo), 0); + ASSERT_EQ(copysetInfo.logicalpoolid(), 1); + ASSERT_EQ(copysetInfo.copysetid(), 1); + ASSERT_EQ(copysetInfo.scaning(), true); + ASSERT_EQ(copysetInfo.lastscansec(), 123456789); + ASSERT_EQ(copysetInfo.lastscanconsistent(), false); + } +} + TEST_F(ToolMDSClientTest, RapidLeaderSchedule) { // 发送rpc失败 EXPECT_CALL(*scheduleService, RapidLeaderSchedule(_, _, _, _)) diff --git a/test/tools/mock/mock_mds_client.h b/test/tools/mock/mock_mds_client.h index 126bf28369..fb94ed7662 100644 --- a/test/tools/mock/mock_mds_client.h +++ b/test/tools/mock/mock_mds_client.h @@ -74,7 +74,9 @@ class MockMDSClient : public MDSClient { std::vector*)); MOCK_METHOD2(GetCopySetsInChunkServer, int(const std::string&, std::vector*)); - MOCK_METHOD1(GetCopySetsInCluster, int(std::vector*)); + + MOCK_METHOD2(GetCopySetsInCluster, int(std::vector*, bool)); + MOCK_METHOD1(ListServersInCluster, int(std::vector*)); MOCK_METHOD1(ListChunkServersInCluster, int(std::vector*)); diff --git a/test/tools/mock/mock_topology_service.h b/test/tools/mock/mock_topology_service.h index 56cb7093b5..c8e95dd5d5 100644 --- a/test/tools/mock/mock_topology_service.h +++ b/test/tools/mock/mock_topology_service.h @@ -148,6 +148,12 @@ class MockTopologyService : public TopologyService { const GetCopySetsInClusterRequest *request, GetCopySetsInClusterResponse *response, Closure *done)); + + MOCK_METHOD4(GetCopyset, void(RpcController* controller, + const GetCopysetRequest* request, + GetCopysetResponse* response, + Closure* done)); + MOCK_METHOD4(GetClusterInfo, void(RpcController *controller, const GetClusterInfoRequest *request,