diff --git a/conf/client.conf b/conf/client.conf index 962c2d6bef..971e90fca9 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -156,3 +156,11 @@ global.turnOffHealthCheck=true ### throttle config # throttle.enable=false + +##### discard configurations ##### +# enable/disable discard +discard.enable=true +# discard granularity +discard.granularity=4096 +# discard cleanup task delay times in millisecond +discard.taskDelayMs=60000 diff --git a/conf/cs_client.conf b/conf/cs_client.conf index 663d822757..98e00e77f9 100644 --- a/conf/cs_client.conf +++ b/conf/cs_client.conf @@ -151,3 +151,11 @@ global.metricDummyServerStartPort=9000 # session map文件，存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +##### discard configurations ##### +# enable/disable discard +discard.enable=false +# discard granularity +discard.granularity=4096 +# discard cleanup task delay times in millisecond +discard.taskDelayMs=60000 diff --git a/conf/mds.conf b/conf/mds.conf index 165af73b7d..48130459af 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -26,6 +26,8 @@ mds.segment.alloc.periodic.persistInterMs=10000 # 出错情况下的重试间隔,单位ms mds.segment.alloc.retryInterMs=1000 +mds.segment.discard.scanIntevalMs=5000 + # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s) # 该值和etcd集群election timeout相关. diff --git a/conf/py_client.conf b/conf/py_client.conf index 601a4e1069..0a681aa0b1 100644 --- a/conf/py_client.conf +++ b/conf/py_client.conf @@ -145,3 +145,11 @@ global.metricDummyServerStartPort=10000 # session map文件，存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +##### discard configurations ##### +# enable/disable discard +discard.enable=false +# discard granularity +discard.granularity=4096 +# discard cleanup task delay times in millisecond +discard.taskDelayMs=60000 diff --git a/conf/snap_client.conf b/conf/snap_client.conf index e1386901ac..4e11946a57 100644 --- a/conf/snap_client.conf +++ b/conf/snap_client.conf @@ -151,3 +151,11 @@ global.metricDummyServerStartPort=9000 # session map文件，存储打开文件的filename到path的映射 # global.sessionMapPath=./session_map.json + +##### discard configurations ##### +# enable/disable discard +discard.enable=false +# discard granularity +discard.granularity=4096 +# discard cleanup task delay times in millisecond +discard.taskDelayMs=60000 diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml index c54c06dce7..68fc8d7c81 100644 --- a/curve-ansible/roles/generate_config/defaults/main.yml +++ b/curve-ansible/roles/generate_config/defaults/main.yml @@ -30,6 +30,7 @@ mds_etcd_operation_timeout_ms: 5000 mds_etcd_retry_times: 3 mds_segment_alloc_periodic_persist_inter_ms: 10000 mds_segment_alloc_retry_inter_ms: 1000 +mds_segment_discard_scan_interval_ms: 5000 mds_leader_session_inter_sec: 5 mds_leader_election_timeout_ms: 0 mds_enable_copyset_scheduler: true @@ -220,6 +221,9 @@ client_session_map_path: ./session_map.json client_closefd_timeout_sec: 300 client_closefd_time_interval_sec: 600 client_throttle_enable: false +client_discard_enable: true +client_discard_granularity: 4096 +client_discard_task_delay_ms: 60000 # nebd默认配置 client_config_path: /etc/curve/client.conf diff --git a/curve-ansible/roles/generate_config/templates/client.conf.j2 b/curve-ansible/roles/generate_config/templates/client.conf.j2 index db285a0d89..08d4413780 100644 --- a/curve-ansible/roles/generate_config/templates/client.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/client.conf.j2 @@ -167,3 +167,11 @@ global.sessionMapPath={{ client_session_map_path }} ### throttle config # throttle.enable={{ client_throttle_enable }} + +##### discard configurations ##### +# enable/disable discard +discard.enable={{ client_discard_enable }} +# discard granularity +discard.granularity={{ client_discard_granularity }} +# discard cleanup task delay times in millisecond +discard.taskDelayMs={{ client_discard_task_delay_ms }} diff --git a/curve-ansible/roles/generate_config/templates/mds.conf.j2 b/curve-ansible/roles/generate_config/templates/mds.conf.j2 index 2eb7a85cac..f0dc602a8f 100644 --- a/curve-ansible/roles/generate_config/templates/mds.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/mds.conf.j2 @@ -34,6 +34,8 @@ mds.segment.alloc.periodic.persistInterMs={{ mds_segment_alloc_periodic_persist_ # 出错情况下的重试间隔,单位ms mds.segment.alloc.retryInterMs={{ mds_segment_alloc_retry_inter_ms }} +mds.segment.discard.scanIntevalMs={{ mds_segment_discard_scan_interval_ms }} + # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s) # 该值和etcd集群election timeout相关. diff --git a/include/client/libcurve.h b/include/client/libcurve.h index a9ae59a1b2..88202e1f11 100644 --- a/include/client/libcurve.h +++ b/include/client/libcurve.h @@ -113,6 +113,7 @@ const char* ErrorNum2ErrorName(LIBCURVE_ERROR err); typedef enum LIBCURVE_OP { LIBCURVE_OP_READ, LIBCURVE_OP_WRITE, + LIBCURVE_OP_DISCARD, LIBCURVE_OP_MAX, } LIBCURVE_OP; @@ -233,6 +234,16 @@ int Read(int fd, char* buf, off_t offset, size_t length); */ int Write(int fd, const char* buf, off_t offset, size_t length); +/** + * @brief Synchronous discard operation + * @param fd file descriptor + * @param offset discard offset + * @param length discard length + * @return On success, return 0. + * On error, returns a negative value. + */ +int Discard(int fd, off_t offset, size_t length); + /** * 异步模式读 * @param: fd为当前open返回的文件描述符 @@ -249,6 +260,14 @@ int AioRead(int fd, CurveAioContext* aioctx); */ int AioWrite(int fd, CurveAioContext* aioctx); +/** + * @brief Asynchronous discard operation + * @param fd file descriptor + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ +int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 重命名文件 * @param: userinfo是用户信息 @@ -510,6 +529,14 @@ class CurveClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType); + /** + * @brief Async Discard + * @param fd file descriptor + * @param aioctx async request context + * @return return error code, 0(LIBCURVE_ERROR::OK) means success + */ + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 测试使用，设置fileclient * @param client 需要设置的fileclient diff --git a/nebd/src/part2/request_executor_curve.cpp b/nebd/src/part2/request_executor_curve.cpp index 7c3a08082c..24dd02c557 100644 --- a/nebd/src/part2/request_executor_curve.cpp +++ b/nebd/src/part2/request_executor_curve.cpp @@ -152,13 +152,33 @@ int CurveRequestExecutor::GetInfo( return 0; } -int CurveRequestExecutor::Discard( - NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::Discard(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { + int curveFd = GetCurveFdFromNebdFileInstance(fd); + if (curveFd < 0) { + LOG(ERROR) << "Parse curve fd failed"; + return -1; + } - aioctx->ret = 0; - aioctx->cb(aioctx); + CurveAioCombineContext* curveCombineCtx = new CurveAioCombineContext(); + curveCombineCtx->nebdCtx = aioctx; + int ret = FromNebdCtxToCurveCtx(aioctx, &curveCombineCtx->curveCtx); + if (ret < 0) { + LOG(ERROR) << "Convert nebd aio context to curve aio context failed, " + "curve fd: " + << curveFd; + delete curveCombineCtx; + return -1; + } - return 0; + ret = client_->AioDiscard(curveFd, &curveCombineCtx->curveCtx); + if (ret == LIBCURVE_ERROR::OK) { + return 0; + } + + LOG(ERROR) << "Curve client return failed, curve fd: " << curveFd; + delete curveCombineCtx; + return -1; } int CurveRequestExecutor::AioRead( @@ -271,7 +291,9 @@ int CurveRequestExecutor::FromNebdOpToCurveOp(LIBAIO_OP op, LIBCURVE_OP *out) { case LIBAIO_OP::LIBAIO_OP_WRITE: *out = LIBCURVE_OP_WRITE; return 0; - + case LIBAIO_OP::LIBAIO_OP_DISCARD: + *out = LIBCURVE_OP_DISCARD; + return 0; default: return -1; } diff --git a/nebd/test/part2/mock_curve_client.h b/nebd/test/part2/mock_curve_client.h index ce12097c98..515b0fb991 100644 --- a/nebd/test/part2/mock_curve_client.h +++ b/nebd/test/part2/mock_curve_client.h @@ -45,6 +45,7 @@ class MockCurveClient : public ::curve::client::CurveClient { int(int, CurveAioContext*, curve::client::UserDataType)); MOCK_METHOD3(AioWrite, int(int, CurveAioContext*, curve::client::UserDataType)); + MOCK_METHOD2(AioDiscard, int(int, CurveAioContext*)); }; } // namespace server diff --git a/nebd/test/part2/test_request_executor_curve.cpp b/nebd/test/part2/test_request_executor_curve.cpp index 0f85914f65..c6a1264ae1 100644 --- a/nebd/test/part2/test_request_executor_curve.cpp +++ b/nebd/test/part2/test_request_executor_curve.cpp @@ -358,20 +358,58 @@ TEST_F(TestReuqestExecutorCurve, test_AioWrite) { TEST_F(TestReuqestExecutorCurve, test_Discard) { auto executor = CurveRequestExecutor::GetInstance(); + NebdServerAioContext aioctx; + aioctx.cb = NebdUnitTestCallback; std::string curveFilename("/cinder/volume-1234_cinder_"); - std::unique_ptr curveFileIns(new CurveFileInstance()); - NebdServerAioContext* aioctx = new NebdServerAioContext(); - nebd::client::DiscardResponse response; - TestReuqestExecutorCurveClosure done; - aioctx->op = LIBAIO_OP::LIBAIO_OP_DISCARD; - aioctx->cb = NebdFileServiceCallback; - aioctx->response = &response; - aioctx->done = &done; + // 1. not an curve volume + { + std::unique_ptr nebdFileIns(new NebdFileInstance()); + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .Times(0); + ASSERT_EQ(-1, executor.Discard(nebdFileIns.get(), &aioctx)); + } - ASSERT_EQ(0, executor.Discard(curveFileIns.get(), aioctx)); - ASSERT_TRUE(done.IsRunned()); - ASSERT_EQ(response.retcode(), nebd::client::RetCode::kOK); + // 2. fd is invalid + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + curveFileIns->fd = -1; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .Times(0); + ASSERT_EQ(-1, executor.Discard(curveFileIns.get(), &aioctx)); + } + + // 3. curve client return failed + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + aioctx.size = 1; + aioctx.offset = 0; + aioctx.op = LIBAIO_OP::LIBAIO_OP_DISCARD; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.Discard(curveFileIns.get(), &aioctx)); + } + + // 4. ok + { + std::unique_ptr curveFileIns( + new CurveFileInstance()); + aioctx.size = 1; + aioctx.offset = 0; + aioctx.op = LIBAIO_OP::LIBAIO_OP_DISCARD; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + CurveAioContext* curveCtx; + EXPECT_CALL(*curveClient_, AioDiscard(_, _)) + .WillOnce(DoAll(SaveArg<1>(&curveCtx), + Return(LIBCURVE_ERROR::OK))); + ASSERT_EQ(0, executor.Discard(curveFileIns.get(), &aioctx)); + curveCtx->cb(curveCtx); + } } TEST_F(TestReuqestExecutorCurve, test_Flush) { diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto index be18b9e395..9c8aeda20d 100644 --- a/proto/nameserver2.proto +++ b/proto/nameserver2.proto @@ -184,6 +184,11 @@ message PageFileSegment { repeated PageFileChunkInfo chunks = 5; } +message DiscardSegmentInfo { + required FileInfo fileInfo = 1; + required PageFileSegment pageFileSegment = 2; +} + message CreateFileRequest { required string fileName = 1; required FileType fileType = 3; @@ -254,6 +259,18 @@ message GetOrAllocateSegmentResponse { optional PageFileSegment pageFileSegment = 2; } +message DeAllocateSegmentRequest { + required string fileName = 1; + required string owner = 2; + required uint64 offset = 3; + optional string signature = 4; + required uint64 date = 5; +} + +message DeAllocateSegmentResponse { + required StatusCode statusCode = 1; +} + message RenameFileRequest { required string oldFileName = 1; required string newFileName = 2; @@ -569,6 +586,7 @@ service CurveFSService { rpc GetFileInfo(GetFileInfoRequest) returns (GetFileInfoResponse); rpc GetOrAllocateSegment(GetOrAllocateSegmentRequest) returns (GetOrAllocateSegmentResponse); + rpc DeAllocateSegment(DeAllocateSegmentRequest) returns (DeAllocateSegmentResponse); rpc RenameFile(RenameFileRequest) returns (RenameFileResponse); rpc ExtendFile(ExtendFileRequest) returns (ExtendFileResponse); rpc ChangeOwner(ChangeOwnerRequest) returns (ChangeOwnerResponse); diff --git a/src/client/client_common.h b/src/client/client_common.h index 77125ca845..ff2ee874c8 100644 --- a/src/client/client_common.h +++ b/src/client/client_common.h @@ -43,12 +43,17 @@ using CopysetID = uint32_t; using LogicPoolID = uint32_t; using ChunkServerID = uint32_t; using ChunkIndex = uint32_t; +using SegmentIndex = uint32_t; using EndPoint = butil::EndPoint; using Status = butil::Status; using IOManagerID = uint64_t; +constexpr uint64_t KiB = 1024; +constexpr uint64_t MiB = 1024 * KiB; +constexpr uint64_t GiB = 1024 * MiB; + // 操作类型 enum class OpType { READ = 0, @@ -58,6 +63,7 @@ enum class OpType { CREATE_CLONE, RECOVER_CHUNK, GET_CHUNK_INFO, + DISCARD, UNKNOWN }; @@ -245,6 +251,8 @@ inline const char* OpTypeToString(OpType optype) { return "RecoverChunk"; case OpType::GET_CHUNK_INFO: return "GetChunkInfo"; + case OpType::DISCARD: + return "Discard"; case OpType::UNKNOWN: default: return "Unknown"; diff --git a/src/client/client_config.cpp b/src/client/client_config.cpp index b2f1412ae0..e30d88dabe 100644 --- a/src/client/client_config.cpp +++ b/src/client/client_config.cpp @@ -39,6 +39,9 @@ namespace curve { namespace client { int ClientConfig::Init(const char* configpath) { conf_.SetConfigPath(configpath); + + LOG(INFO) << "Init config from " << configpath; + if (!conf_.LoadConfig()) { LOG(ERROR) << "Load config failed, config path = " << configpath; return -1; @@ -251,6 +254,23 @@ int ClientConfig::Init(const char* configpath) { << "config no throttle.enable info, using default value " << fileServiceOption_.ioOpt.throttleOption.enable; + ret = conf_.GetBoolValue("discard.enable", + &fileServiceOption_.ioOpt.discardOption.enable); + LOG_IF(ERROR, ret == false) << "config no discard.enable info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value( + "discard.granularity", + &fileServiceOption_.ioOpt.metaCacheOpt.discardGranularity); + LOG_IF(ERROR, ret == false) << "config no discard.granularity info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value( + "discard.taskDelayMs", + &fileServiceOption_.ioOpt.discardOption.taskDelayMs); + LOG_IF(ERROR, ret == false) << "config no discard.taskDelayMs info"; + RETURN_IF_FALSE(ret); + return 0; } diff --git a/src/client/client_metric.h b/src/client/client_metric.h index f1ba5d1405..876f83fd33 100644 --- a/src/client/client_metric.h +++ b/src/client/client_metric.h @@ -40,10 +40,6 @@ static void GetStringValue(std::ostream& os, void* arg) { os << *static_cast(arg); } -static uint64_t GetUnInt64Value(void* arg) { - return *static_cast(arg); -} - // 悬挂IO统计，文件级别统计，方便定位 struct IOSuspendMetric { // 当前persecond计数总数 @@ -92,6 +88,19 @@ struct InterfaceMetric { latency(prefix, name + "_lat") {} }; +struct DiscardMetric { + explicit DiscardMetric(const std::string& prefix) + : totalSuccess(prefix, "discard_total_success"), + totalError(prefix, "discard_total_error"), + totalCanceled(prefix, "discard_total_canceled"), + pending(prefix, "discard_pending") {} + + bvar::Adder totalSuccess; + bvar::Adder totalError; + bvar::Adder totalCanceled; + bvar::Adder pending; +}; + // 文件级别metric信息统计 struct FileMetric { const std::string prefix = "curve_client"; @@ -105,6 +114,7 @@ struct FileMetric { // 当前文件请求的最大请求字节数，这种统计方式可以很方便的看到最大值，分位值 bvar::LatencyRecorder readSizeRecorder; bvar::LatencyRecorder writeSizeRecorder; + bvar::LatencyRecorder discardSizeRecorder; // libcurve最底层read rpc接口统计信息metric统计 InterfaceMetric readRPC; @@ -114,23 +124,31 @@ struct FileMetric { InterfaceMetric userRead; // 用户写请求qps、eps、rps InterfaceMetric userWrite; + // user's discard request + InterfaceMetric userDiscard; + // get leader失败重试qps PerSecondMetric getLeaderRetryQPS; // 当前文件上的悬挂IO数量 IOSuspendMetric suspendRPCMetric; + DiscardMetric discardMetric; + explicit FileMetric(const std::string& name) : filename(name), inflightRPCNum(prefix, filename + "_inflight_rpc_num"), readSizeRecorder(prefix, filename + "_read_request_size_recoder"), writeSizeRecorder(prefix, filename + "_write_request_size_recoder"), + discardSizeRecorder(prefix, filename + "_discard_request_size_recoder"), // NOLINT readRPC(prefix, filename + "_read_rpc"), writeRPC(prefix, filename + "_write_rpc"), userRead(prefix, filename + "_read"), userWrite(prefix, filename + "_write"), + userDiscard(prefix, filename + "_discard"), getLeaderRetryQPS(prefix, filename + "_get_leader_retry_rpc"), - suspendRPCMetric(prefix, filename + "_suspend_io_num") {} + suspendRPCMetric(prefix, filename + "_suspend_io_num"), + discardMetric(prefix + filename) {} }; // 用于全局mds接口统计信息调用信息统计 @@ -155,6 +173,8 @@ struct MDSClientMetric { InterfaceMetric getServerList; // GetOrAllocateSegment接口统计信息 InterfaceMetric getOrAllocateSegment; + // DeAllocateSegment接口统计信息 + InterfaceMetric deAllocateSegment; // RenameFile接口统计信息 InterfaceMetric renameFile; // Extend接口统计信息 @@ -191,6 +211,7 @@ struct MDSClientMetric { refreshSession(prefix, "refreshSession"), getServerList(prefix, "getServerList"), getOrAllocateSegment(prefix, "getOrAllocateSegment"), + deAllocateSegment(prefix, "deAllocateSegment"), renameFile(prefix, "renameFile"), extendFile(prefix, "extendFile"), deleteFile(prefix, "deleteFile"), @@ -250,6 +271,10 @@ class MetricHelper { fm->userWrite.bps.count << length; fm->writeSizeRecorder << length; break; + case OpType::DISCARD: + fm->userDiscard.qps.count << 1; + fm->userDiscard.bps.count << length; + fm->discardSizeRecorder << length; default: break; } @@ -270,6 +295,8 @@ class MetricHelper { case OpType::WRITE: fm->userWrite.eps.count << 1; break; + case OpType::DISCARD: + fm->userDiscard.eps.count << 1; default: break; } @@ -294,6 +321,8 @@ class MetricHelper { case OpType::WRITE: fm->userWrite.rps.count << 1; break; + case OpType::DISCARD: + fm->userDiscard.rps.count << 1; default: break; } @@ -435,6 +464,8 @@ class MetricHelper { case OpType::WRITE: fm->userWrite.latency << duration; break; + case OpType::DISCARD: + fm->userDiscard.latency << duration; default: break; } diff --git a/src/client/config_info.h b/src/client/config_info.h index ee468407f2..8f8c4e6161 100644 --- a/src/client/config_info.h +++ b/src/client/config_info.h @@ -198,6 +198,7 @@ struct MetaCacheOption { uint32_t metacacheRPCRetryIntervalUS = 500; uint32_t metacacheGetLeaderRPCTimeOutMS = 1000; uint32_t metacacheGetLeaderBackupRequestMS = 100; + uint32_t discardGranularity = 4096; std::string metacacheGetLeaderBackupRequestLbName = "rr"; ChunkServerUnstableOption chunkserverUnstableOption; }; @@ -223,6 +224,12 @@ struct TaskThreadOption { uint32_t isolationTaskThreadPoolSize = 1; }; +// for discard +struct DiscardOption { + bool enable = false; + uint32_t taskDelayMs = 1000 * 60; // 1 min +}; + /** * timed close fd thread in SourceReader config * @fdTimeout: sourcereader fd timeout @@ -248,6 +255,7 @@ struct IOOption { RequestScheduleOption reqSchdulerOpt; CloseFdThreadOption closeFdThreadOption; ThrottleOption throttleOption; + DiscardOption discardOption; }; /** diff --git a/src/client/discard_task.cpp b/src/client/discard_task.cpp new file mode 100644 index 0000000000..6ae6c09db7 --- /dev/null +++ b/src/client/discard_task.cpp @@ -0,0 +1,149 @@ + +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#include "src/client/discard_task.h" + +#include + +#include +#include +#include + +namespace curve { +namespace client { + +void DiscardTask::Run() { + const FInfo* fileInfo = metaCache_->GetFileInfo(); + uint64_t offset = + static_cast(segmentIndex_) * fileInfo->segmentsize; + FileSegment* fileSegment = metaCache_->GetFileSegment(segmentIndex_); + + fileSegment->AcquireWriteLock(); + metric_->pending << -1; + + if (!fileSegment->IsAllBitSet()) { + LOG(WARNING) << "DiscardTask find bitmap was cleared, cancel task, " + "filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << timerId_; + metric_->totalCanceled << 1; + fileSegment->ReleaseLock(); + taskManager_->OnTaskFinish(timerId_); + return; + } + + LIBCURVE_ERROR errCode = mdsClient_->DeAllocateSegment(fileInfo, offset); + if (errCode == LIBCURVE_ERROR::OK) { + metric_->totalSuccess << 1; + fileSegment->ClearBitmap(); + metaCache_->CleanChunksInSegment(segmentIndex_); + LOG(INFO) << "DiscardTask success, filename = " + << fileInfo->fullPathName << ", offset = " << offset + << ", taskid = " << timerId_; + } else if (errCode == LIBCURVE_ERROR::UNDER_SNAPSHOT) { + metric_->totalError << 1; + LOG(WARNING) << "DiscardTask failed, " << fileInfo->fullPathName + << " has snapshot, offset = " << offset + << ", taskid = " << timerId_; + } else { + metric_->totalError << 1; + LOG(ERROR) << "DiscardTask failed, mds return error = " << errCode + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset << ", taskid = " << timerId_; + } + + fileSegment->ReleaseLock(); + taskManager_->OnTaskFinish(timerId_); +} + +DiscardTaskManager::DiscardTaskManager(DiscardMetric* metric) + : mtx_(), cond_(), unfinishedTasks_(), metric_(metric) {} + +static void RunDiscardTask(void* arg) { + DiscardTask* task = static_cast(arg); + task->Run(); +} + +void DiscardTaskManager::OnTaskFinish(bthread_timer_t timerId) { + std::lock_guard lk(mtx_); + unfinishedTasks_.erase(timerId); + cond_.notify_one(); +} + +bool DiscardTaskManager::ScheduleTask(SegmentIndex segmentIndex, + MetaCache* metaCache, + MDSClient* mdsclient, timespec abstime) { + bthread_timer_t timerId; + std::unique_ptr task( + new DiscardTask(this, segmentIndex, metaCache, mdsclient, metric_)); + + int ret = bthread_timer_add(&timerId, abstime, RunDiscardTask, task.get()); + if (ret == 0) { + task->SetId(timerId); + LOG(INFO) << "Schedule discard task success, taskid = " << task->Id(); + std::lock_guard lk(mtx_); + unfinishedTasks_.emplace(timerId, std::move(task)); + metric_->pending << 1; + return true; + } + + LOG(ERROR) << "bthread_timer_add failed, ret = " << ret + << ", errno = " << errno << ", run task directly"; + return false; +} + +void DiscardTaskManager::Stop() { + std::unordered_set currentTasks; + + { + std::lock_guard lk(mtx_); + for (auto& kv : unfinishedTasks_) { + currentTasks.emplace(kv.first); + } + } + + for (const auto& timerId : currentTasks) { + int ret = bthread_timer_del(timerId); + + if (ret == 0) { + LOG(INFO) << "Cancenl discard task success, taskid = " << timerId; + } else if (ret == 1) { + LOG(WARNING) << "Task is running, taskid = " << timerId; + continue; + } else if (ret == EINVAL) { + LOG(WARNING) + << "Task has been completed or taskid is invalid, taskid = " + << timerId; + } + + OnTaskFinish(timerId); + } + + std::unique_lock lk(mtx_); + while (!unfinishedTasks_.empty()) { + cond_.wait(lk); + } +} + +} // namespace client +} // namespace curve diff --git a/src/client/discard_task.h b/src/client/discard_task.h new file mode 100644 index 0000000000..1afc57b1eb --- /dev/null +++ b/src/client/discard_task.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Thu Dec 17 11:05:38 CST 2020 + * Author: wuhanqing + */ + +#ifndef SRC_CLIENT_DISCARD_TASK_H_ +#define SRC_CLIENT_DISCARD_TASK_H_ + +#include +#include +#include +#include + +#include + +#include "src/client/client_metric.h" +#include "src/client/metacache.h" +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +class DiscardTaskManager; + + +/** + * DiscardTask corresponding to one segment discard task. + * It's main function is to send DeAllocateSegment request to MDS + * and clear cached segment info on success. + */ +class DiscardTask { + public: + DiscardTask(DiscardTaskManager* taskManager, SegmentIndex segmentIndex, + MetaCache* metaCache, MDSClient* mdsClient, + DiscardMetric* metric) + : taskManager_(taskManager), + segmentIndex_(segmentIndex), + metaCache_(metaCache), + mdsClient_(mdsClient), + timerId_(0), + metric_(metric) {} + + void Run(); + + bthread_timer_t Id() const { + return timerId_; + } + + void SetId(bthread_timer_t id) { + timerId_ = id; + } + + private: + DiscardTaskManager* taskManager_; + SegmentIndex segmentIndex_; + MetaCache* metaCache_; + MDSClient* mdsClient_; + bthread_timer_t timerId_; + DiscardMetric* metric_; + + static std::atomic taskId_; +}; + +/** + * DiscardTaskManager stores all pending DiscardTasks + */ +class DiscardTaskManager { + public: + explicit DiscardTaskManager(DiscardMetric* metric); + + void OnTaskFinish(bthread_timer_t timerId); + + bool ScheduleTask(SegmentIndex segmentIndex, MetaCache* metaCache, + MDSClient* mdsclient, timespec abstime); + + /** + * @brief Cancel all unfinished discard tasks + */ + void Stop(); + + private: + bthread::Mutex mtx_; + bthread::ConditionVariable cond_; + std::unordered_map> unfinishedTasks_; // NOLINT + + DiscardMetric* metric_; +}; + +} // namespace client +} // namespace curve + +#endif // SRC_CLIENT_DISCARD_TASK_H_ diff --git a/src/client/file_instance.cpp b/src/client/file_instance.cpp index 2c0348282a..3511e7744b 100644 --- a/src/client/file_instance.cpp +++ b/src/client/file_instance.cpp @@ -123,6 +123,24 @@ int FileInstance::AioWrite(CurveAioContext* aioctx, UserDataType dataType) { return iomanager4file_.AioWrite(aioctx, mdsclient_, dataType); } +int FileInstance::Discard(off_t offset, size_t length) { + if (!readonly_) { + return iomanager4file_.Discard(offset, length, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support Discard"; + return -1; +} + +int FileInstance::AioDiscard(CurveAioContext* aioctx) { + if (!readonly_) { + return iomanager4file_.AioDiscard(aioctx, mdsclient_); + } + + LOG(ERROR) << "Open with read only, not support AioDiscard"; + return -1; +} + // 两种场景会造成在Open的时候返回LIBCURVE_ERROR::FILE_OCCUPIED // 1. 强制重启qemu不会调用close逻辑，然后启动的时候原来的文件sessio还没过期. // 导致再次去发起open的时候，返回被占用，这种情况可以通过load sessionmap diff --git a/src/client/file_instance.h b/src/client/file_instance.h index b49000ebb8..a4c0def626 100644 --- a/src/client/file_instance.h +++ b/src/client/file_instance.h @@ -109,6 +109,21 @@ class CURVE_CACHELINE_ALIGNMENT FileInstance { */ int AioWrite(CurveAioContext* aioctx, UserDataType dataType); + /** + * @param offset discard offset + * @param length discard length + * @return On success, returns 0. + * On error, returns a negative value. + */ + int Discard(off_t offset, size_t length); + + /** + * @brief Asynchronous discard operation + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ + int AioDiscard(CurveAioContext* aioctx); + int Close(); void UnInitialize(); diff --git a/src/client/io_tracker.cpp b/src/client/io_tracker.cpp index 61ae695ef3..188aaae1c0 100644 --- a/src/client/io_tracker.cpp +++ b/src/client/io_tracker.cpp @@ -23,6 +23,8 @@ #include #include +#include +#include #include "src/client/splitor.h" #include "src/client/iomanager.h" @@ -30,8 +32,9 @@ #include "src/client/request_scheduler.h" #include "src/client/request_closure.h" #include "src/common/timeutility.h" -#include "src/client/libcurve_file.h" #include "src/client/source_reader.h" +#include "src/client/metacache_struct.h" +#include "src/client/discard_task.h" namespace curve { namespace client { @@ -39,6 +42,7 @@ namespace client { using curve::chunkserver::CHUNK_OP_STATUS; std::atomic IOTracker::tracekerID_(1); +DiscardOption IOTracker::discardOption_; IOTracker::IOTracker(IOManager* iomanager, MetaCache* mc, @@ -63,6 +67,12 @@ IOTracker::IOTracker(IOManager* iomanager, opStartTimePoint_ = curve::common::TimeUtility::GetTimeofDayUs(); } +void IOTracker::ReleaseAllSegmentLocks() { + for (auto& readlock : segmentLocks_) { + readlock->ReleaseLock(); + } +} + void IOTracker::StartRead(void* buf, off_t offset, size_t length, MDSClient* mdsclient, const FInfo_t* fileInfo, Throttle* throttle) { @@ -221,6 +231,55 @@ void IOTracker::DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo, } } +void IOTracker::StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo* fileInfo, + DiscardTaskManager* taskManager) { + offset_ = offset; + length_ = length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo, taskManager); +} + +void IOTracker::StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo, + DiscardTaskManager* taskManager) { + aioctx_ = ctx; + offset_ = ctx->offset; + length_ = ctx->length; + type_ = OpType::DISCARD; + + DoDiscard(mdsclient, fileInfo, taskManager); +} + +void IOTracker::DoDiscard(MDSClient* mdsClient, const FInfo* fileInfo, + DiscardTaskManager* taskManager) { + int ret = Splitor::IO2ChunkRequests(this, mc_, &reqlist_, nullptr, offset_, + length_, mdsClient, fileInfo); + + if (ret == 0 && !discardSegments_.empty()) { + for (auto index : discardSegments_) { + timespec abstime = + butil::milliseconds_from_now(discardOption_.taskDelayMs); + bool ret = + taskManager->ScheduleTask(index, mc_, mdsClient, abstime); + std::ostringstream taskinfo; + taskinfo << "filename = " << fileInfo->fullPathName + << ", segment index = " << index + << ", segment offset = " << index * GiB; + if (!ret) { + LOG(ERROR) << "Schedule discard task failed, " + << taskinfo.str(); + } else { + LOG(INFO) << "Schedule discard task, " << taskinfo.str(); + } + } + } + + errcode_ = LIBCURVE_ERROR::OK; + Done(); +} + void IOTracker::ReadSnapChunk(const ChunkIDInfo &cinfo, uint64_t seq, uint64_t offset, uint64_t len, char *buf, SnapCloneClosure* scc) { @@ -392,11 +451,19 @@ void IOTracker::HandleResponse(RequestContext* reqctx) { } } +void IOTracker::InitDiscardOption(const DiscardOption& opt) { + discardOption_ = opt; +} + int IOTracker::Wait() { return iocv_.Wait(); } void IOTracker::Done() { + if (type_ == OpType::READ || type_ == OpType::WRITE) { + ReleaseAllSegmentLocks(); + } + if (errcode_ == LIBCURVE_ERROR::OK) { uint64_t duration = TimeUtility::GetTimeofDayUs() - opStartTimePoint_; MetricHelper::UserLatencyRecord(fileMetric_, duration, type_); @@ -457,18 +524,16 @@ void IOTracker::Done() { // scc_和aioctx都为空的时候肯定是个同步调用 if (scc_ == nullptr && aioctx_ == nullptr) { - errcode_ == LIBCURVE_ERROR::OK ? iocv_.Complete(length_) - : iocv_.Complete(-errcode_); + iocv_.Complete(ToReturnCode()); return; } // 异步函数调用，在此处发起回调 if (aioctx_ != nullptr) { - aioctx_->ret = errcode_ == LIBCURVE_ERROR::OK ? length_ : -errcode_; + aioctx_->ret = ToReturnCode(); aioctx_->cb(aioctx_); } else { - int ret = errcode_ == LIBCURVE_ERROR::OK ? length_ : -errcode_; - scc_->SetRetCode(ret); + scc_->SetRetCode(ToReturnCode()); scc_->Run(); } diff --git a/src/client/io_tracker.h b/src/client/io_tracker.h index ce4dcfd61e..6795a519bd 100644 --- a/src/client/io_tracker.h +++ b/src/client/io_tracker.h @@ -46,11 +46,15 @@ namespace client { using curve::common::Throttle; class IOManager; +class FileSegment; +class DiscardTaskManager; // IOTracker用于跟踪一个用户IO，因为一个用户IO可能会跨chunkserver， // 因此在真正下发的时候会被拆分成多个小IO并发的向下发送，因此我们需要 // 跟踪发送的request的执行情况。 class CURVE_CACHELINE_ALIGNMENT IOTracker { + friend class Splitor; + public: IOTracker(IOManager* iomanager, MetaCache* mc, @@ -100,6 +104,14 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { */ void StartAioWrite(CurveAioContext* ctx, MDSClient* mdsclient, const FInfo_t* fileInfo, Throttle* throttle = nullptr); + + void StartDiscard(off_t offset, size_t length, MDSClient* mdsclient, + const FInfo_t* fileInfo, DiscardTaskManager* taskManager); + + void StartAioDiscard(CurveAioContext* ctx, MDSClient* mdsclient, + const FInfo_t* fileInfo, + DiscardTaskManager* taskManager); + /** * chunk相关接口是提供给snapshot使用的，上层的snapshot和file * 接口是分开的，在IOTracker这里会将其统一，这样对下层来说不用 @@ -165,7 +177,7 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { * @return: 返回读写信息，异步IO的时候返回0或-1.0代表成功，-1代表失败 * 同步IO返回length或-1，length代表真实读写长度，-1代表读写失败 */ - int Wait(); + int Wait(); /** * 每个request都要有自己的OP类型，这里提供接口可以在io拆分的时候获取类型 @@ -211,7 +223,11 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { return disableStripe_; } + static void InitDiscardOption(const DiscardOption& opt); + private: + void ReleaseAllSegmentLocks(); + /** * 当IO返回的时候调用done，由done负责向上返回 */ @@ -266,6 +282,15 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { void DoWrite(MDSClient* mdsclient, const FInfo_t* fileInfo, Throttle* throttle); + void DoDiscard(MDSClient* mdsclient, const FInfo_t* fileInfo, + DiscardTaskManager* taskManager); + + int ToReturnCode() const { + return errcode_ == LIBCURVE_ERROR::OK + ? (type_ != OpType::DISCARD ? length_ : 0) + : (-errcode_); + } + private: // io 类型 OpType type_; @@ -304,6 +329,8 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { // 大IO被拆分成多个request，这些request放在reqlist中国保存 std::vector reqlist_; + std::vector discardSegments_; + // scheduler用来将用户线程与client自己的线程切分 // 大IO被切分之后，将切分的reqlist传给scheduler向下发送 RequestScheduler* scheduler_; @@ -329,8 +356,14 @@ class CURVE_CACHELINE_ALIGNMENT IOTracker { bool disableStripe_; + // read/write operations will hold segment's read lock, + // so store corresponding segment lock and release after operations finished + std::vector segmentLocks_; + // id生成器 static std::atomic tracekerID_; + + static DiscardOption discardOption_; }; } // namespace client } // namespace curve diff --git a/src/client/iomanager4file.cpp b/src/client/iomanager4file.cpp index 2a2f100341..9ad1d8979c 100644 --- a/src/client/iomanager4file.cpp +++ b/src/client/iomanager4file.cpp @@ -33,8 +33,7 @@ namespace curve { namespace client { Atomic IOManager::idRecorder_(1); -IOManager4File::IOManager4File(): scheduler_(nullptr), exit_(false) { -} +IOManager4File::IOManager4File() : scheduler_(nullptr), exit_(false) {} bool IOManager4File::Initialize(const std::string& filename, const IOOption& ioOpt, @@ -43,6 +42,8 @@ bool IOManager4File::Initialize(const std::string& filename, disableStripe_ = false; mc_.Init(ioopt_.metaCacheOpt, mdsclient); + + IOTracker::InitDiscardOption(ioopt_.discardOption); Splitor::Init(ioopt_.ioSplitOpt); inflightRpcCntl_.SetMaxInflightNum( @@ -83,6 +84,9 @@ bool IOManager4File::Initialize(const std::string& filename, return false; } + discardTaskManager_.reset( + new DiscardTaskManager(&(fileMetric_->discardMetric))); + LOG(INFO) << "iomanager init success! conf info: " << "isolationTaskThreadPoolSize = " << ioopt_.taskThreadOpt.isolationTaskThreadPoolSize @@ -121,6 +125,8 @@ void IOManager4File::UnInitialize() { scheduler_->Fini(); } + discardTaskManager_->Stop(); + { // 这个锁保证设置exit_和delete scheduler_是原子的 // 这样保证在scheduler_被析构的时候lease线程不会使用scheduler_ @@ -223,6 +229,50 @@ int IOManager4File::AioWrite(CurveAioContext* ctx, MDSClient* mdsclient, return LIBCURVE_ERROR::OK; } +int IOManager4File::Discard(off_t offset, size_t length, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!IsNeedDiscard(length)) { + return 0; + } + + FlightIOGuard guard(this); + + IOTracker tracker(this, &mc_, scheduler_, fileMetric_); + tracker.StartDiscard(offset, length, mdsclient, GetFileInfo(), + discardTaskManager_.get()); + return tracker.Wait(); +} + +int IOManager4File::AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient) { + MetricHelper::IncremUserRPSCount(fileMetric_, OpType::DISCARD); + + if (!IsNeedDiscard(aioctx->length)) { + aioctx->ret = 0; + aioctx->cb(aioctx); + return LIBCURVE_ERROR::OK; + } + + IOTracker* ioTracker = + new (std::nothrow) IOTracker(this, &mc_, scheduler_, fileMetric_); + + if (ioTracker == nullptr) { + aioctx->ret = -LIBCURVE_ERROR::FAILED; + aioctx->cb(aioctx); + LOG(ERROR) << "allocate tracker failed!"; + return LIBCURVE_ERROR::OK; + } + + inflightCntl_.IncremInflightNum(); + auto task = [this, aioctx, mdsclient, ioTracker]() { + ioTracker->StartAioDiscard(aioctx, mdsclient, this->GetFileInfo(), + discardTaskManager_.get()); + }; + + taskPool_.Enqueue(task); + return LIBCURVE_ERROR::OK; +} + void IOManager4File::UpdateFileInfo(const FInfo_t& fi) { mc_.UpdateFileInfo(fi); } @@ -243,6 +293,15 @@ void IOManager4File::HandleAsyncIOResponse(IOTracker* iotracker) { delete iotracker; } +bool IOManager4File::IsNeedDiscard(size_t len) const { + if (ioopt_.discardOption.enable && + len >= ioopt_.metaCacheOpt.discardGranularity) { + return true; + } + + return false; +} + void IOManager4File::LeaseTimeoutBlockIO() { std::unique_lock lk(exitMtx_); if (exit_ == false) { diff --git a/src/client/iomanager4file.h b/src/client/iomanager4file.h index ae379dd25a..e154eaeaec 100644 --- a/src/client/iomanager4file.h +++ b/src/client/iomanager4file.h @@ -42,6 +42,7 @@ #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/task_thread_pool.h" #include "src/common/throttle.h" +#include "src/client/discard_task.h" namespace curve { namespace client { @@ -66,6 +67,11 @@ class IOManager4File : public IOManager { const IOOption& ioOpt, MDSClient* mdsclient); + /** + * @brief Recycle resources + */ + void UnInitialize(); + /** * 同步模式读 * @param: buf为当前待读取的缓冲区 @@ -105,9 +111,21 @@ class IOManager4File : public IOManager { UserDataType dataType); /** - * 析构，回收资源 + * @brief Synchronous discard operation + * @param offset discard offset + * @param length discard length + * @return On success, returns 0. + * On error, returns a negative value. */ - void UnInitialize(); + int Discard(off_t offset, size_t length, MDSClient* mdsclient); + + /** + * @brief Asynchronous discard operation + * @param aioctx async request context + * @param mdsclient for communicate with MDS + * @return 0 means success, otherwise it means failure + */ + int AioDiscard(CurveAioContext* aioctx, MDSClient* mdsclient); /** * @brief 获取rpc发送令牌 @@ -232,6 +250,8 @@ class IOManager4File : public IOManager { IOManager4File* iomanager; }; + bool IsNeedDiscard(size_t len) const; + private: // 每个IOManager都有其IO配置，保存在iooption里 IOOption ioopt_; @@ -276,6 +296,8 @@ class IOManager4File : public IOManager { // chunkserver use client to read clone source file, // because client's IO requests already transformed by stripe parameters bool disableStripe_; + + std::unique_ptr discardTaskManager_; }; } // namespace client diff --git a/src/client/libcbd.h b/src/client/libcbd.h index 887a510711..196534e2fd 100644 --- a/src/client/libcbd.h +++ b/src/client/libcbd.h @@ -65,8 +65,10 @@ int cbd_ext4_open(const char* filename); int cbd_ext4_close(int fd); int cbd_ext4_pread(int fd, void* buf, off_t offset, size_t length); int cbd_ext4_pwrite(int fd, const void* buf, off_t offset, size_t length); +int cbd_ext4_pdiscard(int fd, off_t offset, size_t length); int cbd_ext4_aio_pread(int fd, CurveAioContext* context); int cbd_ext4_aio_pwrite(int fd, CurveAioContext* context); +int cbd_ext4_aio_pdiscard(int fd, CurveAioContext* context); int cbd_ext4_sync(int fd); int64_t cbd_ext4_filesize(const char* filename); @@ -76,35 +78,41 @@ int cbd_libcurve_open(const char* filename); int cbd_libcurve_close(int fd); int cbd_libcurve_pread(int fd, void* buf, off_t offset, size_t length); int cbd_libcurve_pwrite(int fd, const void* buf, off_t offset, size_t length); +int cbd_libcurve_pdiscard(int fd, off_t offset, size_t length); int cbd_libcurve_aio_pread(int fd, CurveAioContext* context); int cbd_libcurve_aio_pwrite(int fd, CurveAioContext* context); +int cbd_libcurve_aio_pdiscard(int fd, CurveAioContext* context); int cbd_libcurve_sync(int fd); int64_t cbd_libcurve_filesize(const char* filename); int cbd_libcurve_resize(const char* filename, int64_t size); #ifndef CBD_BACKEND_FAKE -#define cbd_lib_init cbd_libcurve_init -#define cbd_lib_fini cbd_libcurve_fini -#define cbd_lib_open cbd_libcurve_open -#define cbd_lib_close cbd_libcurve_close -#define cbd_lib_pread cbd_libcurve_pread -#define cbd_lib_pwrite cbd_libcurve_pwrite -#define cbd_lib_aio_pread cbd_libcurve_aio_pread -#define cbd_lib_aio_pwrite cbd_libcurve_aio_pwrite -#define cbd_lib_sync cbd_libcurve_sync -#define cbd_lib_filesize cbd_libcurve_filesize -#define cbd_lib_resize cbd_libcurve_resize +#define cbd_lib_init cbd_libcurve_init +#define cbd_lib_fini cbd_libcurve_fini +#define cbd_lib_open cbd_libcurve_open +#define cbd_lib_close cbd_libcurve_close +#define cbd_lib_pread cbd_libcurve_pread +#define cbd_lib_pwrite cbd_libcurve_pwrite +#define cbd_lib_pdiscard cbd_libcurve_pdiscard +#define cbd_lib_aio_pread cbd_libcurve_aio_pread +#define cbd_lib_aio_pwrite cbd_libcurve_aio_pwrite +#define cbd_lib_aio_pdiscard cbd_libcurve_aio_pdiscard +#define cbd_lib_sync cbd_libcurve_sync +#define cbd_lib_filesize cbd_libcurve_filesize +#define cbd_lib_resize cbd_libcurve_resize #else -#define cbd_lib_init cbd_ext4_init -#define cbd_lib_fini cbd_ext4_fini -#define cbd_lib_open cbd_ext4_open -#define cbd_lib_close cbd_ext4_close -#define cbd_lib_pread cbd_ext4_pread -#define cbd_lib_pwrite cbd_ext4_pwrite -#define cbd_lib_aio_pread cbd_ext4_aio_pread -#define cbd_lib_aio_pwrite cbd_ext4_aio_pwrite -#define cbd_lib_sync cbd_ext4_sync -#define cbd_lib_filesize cbd_ext4_filesize +#define cbd_lib_init cbd_ext4_init +#define cbd_lib_fini cbd_ext4_fini +#define cbd_lib_open cbd_ext4_open +#define cbd_lib_close cbd_ext4_close +#define cbd_lib_pread cbd_ext4_pread +#define cbd_lib_pwrite cbd_ext4_pwrite +#define cbd_lib_pdiscard cbd_ext4_pdiscard +#define cbd_lib_aio_pread cbd_ext4_aio_pread +#define cbd_lib_aio_pwrite cbd_ext4_aio_pwrite +#define cbd_lib_aio_pdiscard cbd_ext4_aio_pdiscard +#define cbd_lib_sync cbd_ext4_sync +#define cbd_lib_filesize cbd_ext4_filesize #endif #ifdef __cplusplus diff --git a/src/client/libcbd_ext4.cpp b/src/client/libcbd_ext4.cpp index e4e97989fe..8eddda26c6 100644 --- a/src/client/libcbd_ext4.cpp +++ b/src/client/libcbd_ext4.cpp @@ -75,6 +75,10 @@ int cbd_ext4_pwrite(int fd, const void* buf, off_t offset, size_t length) { return pwrite(fd, buf, length, offset); } +int cbd_ext4_pdiscard(int fd, off_t offset, size_t length) { + return 0; +} + void cbd_ext4_aio_callback(union sigval sigev_value) { CurveAioContext* context = (CurveAioContext *)sigev_value.sival_ptr; //NOLINT context->cb(context); @@ -120,6 +124,12 @@ int cbd_ext4_aio_pwrite(int fd, CurveAioContext* context) { return aio_write(cb); } +int cbd_ext4_aio_pdiscard(int fd, CurveAioContext* aioctx) { + aioctx->ret = aioctx->length; + aioctx->cb(aioctx); + return 0; +} + int cbd_ext4_sync(int fd) { return fsync(fd); } diff --git a/src/client/libcbd_libcurve.cpp b/src/client/libcbd_libcurve.cpp index a18ee3190e..c91fe4d86f 100644 --- a/src/client/libcbd_libcurve.cpp +++ b/src/client/libcbd_libcurve.cpp @@ -73,6 +73,10 @@ int cbd_libcurve_pwrite(int fd, const void* buf, off_t offset, size_t length) { return Write(fd, reinterpret_cast(buf), offset, length); } +int cbd_libcurve_pdiscard(int fd, off_t offset, size_t length) { + return Discard(fd, offset, length); +} + int cbd_libcurve_aio_pread(int fd, CurveAioContext* context) { return AioRead(fd, context); } @@ -81,6 +85,10 @@ int cbd_libcurve_aio_pwrite(int fd, CurveAioContext* context) { return AioWrite(fd, context); } +int cbd_libcurve_aio_pdiscard(int fd, CurveAioContext* context) { + return AioDiscard(fd, context); +} + int cbd_libcurve_sync(int fd) { // Ignored as it always sync writes to chunkserver currently return 0; diff --git a/src/client/libcurve_client.cpp b/src/client/libcurve_client.cpp index 58b5154932..f488321637 100644 --- a/src/client/libcurve_client.cpp +++ b/src/client/libcurve_client.cpp @@ -119,6 +119,10 @@ int CurveClient::AioWrite(int fd, CurveAioContext* aioctx, return fileClient_->AioWrite(fd, aioctx, dataType); } +int CurveClient::AioDiscard(int fd, CurveAioContext* aioctx) { + return fileClient_->AioDiscard(fd, aioctx); +} + void CurveClient::SetFileClient(FileClient* client) { delete fileClient_; fileClient_ = client; diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index 19874f3515..a364343909 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -332,6 +332,17 @@ int FileClient::Write(int fd, const char* buf, off_t offset, size_t len) { return fileserviceMap_[fd]->Write(buf, offset, len); } +int FileClient::Discard(int fd, off_t offset, size_t length) { + ReadLockGuard lk(rwlock_); + auto iter = fileserviceMap_.find(fd); + if (CURVE_UNLIKELY(iter == fileserviceMap_.end())) { + LOG(ERROR) << "invalid fd, fd = " << fd; + return -LIBCURVE_ERROR::BAD_FD; + } + + return iter->second->Discard(offset, length); +} + int FileClient::AioRead(int fd, CurveAioContext* aioctx, UserDataType dataType) { // 长度为0，直接返回，不做任何操作 @@ -383,6 +394,18 @@ int FileClient::AioWrite(int fd, CurveAioContext* aioctx, return ret; } +int FileClient::AioDiscard(int fd, CurveAioContext* aioctx) { + int ret = -LIBCURVE_ERROR::FAILED; + ReadLockGuard lk(rwlock_); + auto iter = fileserviceMap_.find(fd); + if (CURVE_UNLIKELY(iter == fileserviceMap_.end())) { + LOG(ERROR) << "invalid fd"; + return -LIBCURVE_ERROR::BAD_FD; + } else { + return iter->second->AioDiscard(aioctx); + } +} + int FileClient::Rename(const UserInfo_t& userinfo, const std::string& oldpath, const std::string& newpath) { LIBCURVE_ERROR ret; @@ -742,6 +765,15 @@ int Write(int fd, const char* buf, off_t offset, size_t length) { return globalclient->Write(fd, buf, offset, length); } +int Discard(int fd, off_t offset, size_t length) { + if (globalclient == nullptr) { + LOG(ERROR) << "Not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + + return globalclient->Discard(fd, offset, length); +} + int AioRead(int fd, CurveAioContext* aioctx) { if (globalclient == nullptr) { LOG(ERROR) << "not inited!"; @@ -767,6 +799,15 @@ int AioWrite(int fd, CurveAioContext* aioctx) { return globalclient->AioWrite(fd, aioctx); } +int AioDiscard(int fd, CurveAioContext* aioctx) { + if (globalclient == nullptr) { + LOG(ERROR) << "Not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + + return globalclient->AioDiscard(fd, aioctx); +} + int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size) { if (globalclient == nullptr) { LOG(ERROR) << "not inited!"; diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h index 63947af2e1..e6321e2287 100644 --- a/src/client/libcurve_file.h +++ b/src/client/libcurve_file.h @@ -148,6 +148,16 @@ class FileClient { */ virtual int Write(int fd, const char* buf, off_t offset, size_t length); + /** + * @brief Synchronous discard operation + * @param fd file descriptor + * @param offset discard offset + * @param length discard length + * @return On success, returns 0. + * On error, returns a negative value. + */ + virtual int Discard(int fd, off_t offset, size_t length); + /** * 异步模式读 * @param: fd为当前open返回的文件描述符 @@ -168,6 +178,14 @@ class FileClient { virtual int AioWrite(int fd, CurveAioContext* aioctx, UserDataType dataType = UserDataType::RawBuffer); + /** + * @brief Asynchronous discard operation + * @param fd file descriptor + * @param aioctx async request context + * @return 0 means success, otherwise it means failure + */ + virtual int AioDiscard(int fd, CurveAioContext* aioctx); + /** * 重命名文件 * @param: userinfo是用户信息 @@ -324,7 +342,7 @@ class FileClient { private: bool StartDummyServer(); - bool CheckAligned(off_t offset, size_t length) { + bool CheckAligned(off_t offset, size_t length) const { return (offset % IO_ALIGNED_BLOCK_SIZE == 0) && (length % IO_ALIGNED_BLOCK_SIZE == 0); } diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp index 0eb7f588e0..bf57f55a24 100644 --- a/src/client/mds_client.cpp +++ b/src/client/mds_client.cpp @@ -48,6 +48,7 @@ using curve::mds::DeleteFileResponse; using curve::mds::RecoverFileResponse; using curve::mds::GetFileInfoResponse; using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::RenameFileResponse; using curve::mds::ExtendFileResponse; using curve::mds::ChangeOwnerResponse; @@ -963,6 +964,42 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, return rpcExcutor.DoRPCTask(task, metaServerOpt_.mdsMaxRetryMsInIOPath); } +LIBCURVE_ERROR MDSClient::DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset) { + auto task = RPCTaskDefine { + DeAllocateSegmentResponse response; + mdsClientMetric_.deAllocateSegment.qps.count << 1; + LatencyGuard lg(&mdsClientMetric_.deAllocateSegment.latency); + mdsClientBase_.DeAllocateSegment(fileInfo, offset, &response, cntl, + channel); + + if (cntl->Failed()) { + mdsClientMetric_.deAllocateSegment.eps.count << 1; + LOG(WARNING) << "DeAllocateSegment failed, error = " + << cntl->ErrorText() + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + return -cntl->ErrorCode(); + } + + auto statusCode = response.statuscode(); + if (statusCode == StatusCode::kOK || + statusCode == StatusCode::kSegmentNotAllocated) { + return LIBCURVE_ERROR::OK; + } else { + LOG(WARNING) << "DeAllocateSegment mds return failed, error = " + << mds::StatusCode_Name(statusCode) + << ", filename = " << fileInfo->fullPathName + << ", offset = " << offset; + LIBCURVE_ERROR errCode; + MDSStatusCode2LibcurveError(statusCode, &errCode); + return errCode; + } + }; + + return rpcExcutor.DoRPCTask(task, metaServerOpt_.mdsMaxRetryMS); +} + LIBCURVE_ERROR MDSClient::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client.h b/src/client/mds_client.h index 3ca122cec7..b60c4fdf1b 100644 --- a/src/client/mds_client.h +++ b/src/client/mds_client.h @@ -56,7 +56,8 @@ struct LeaseRefreshResult; class MDSClient { public: explicit MDSClient(const std::string& metricPrefix = ""); - ~MDSClient() = default; + + virtual ~MDSClient() = default; using RPCFunc = std::function; @@ -129,6 +130,16 @@ class MDSClient { uint64_t offset, const FInfo_t* fi, SegmentInfo* segInfo); + + /** + * @brief Send DeAllocateSegment request to current working MDS + * @param fileInfo current file info + * @param offset segment start offset + * @return LIBCURVE_ERROR::OK means success, other value means fail + */ + virtual LIBCURVE_ERROR DeAllocateSegment(const FInfo* fileInfo, + uint64_t offset); + /** * 获取文件信息，fi是出参 * @param: filename是文件名 diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp index 385c87ae38..fe47c46520 100644 --- a/src/client/mds_client_base.cpp +++ b/src/client/mds_client_base.cpp @@ -374,6 +374,25 @@ void MDSClientBase::GetOrAllocateSegment(bool allocate, stub.GetOrAllocateSegment(cntl, &request, response, NULL); } +void MDSClientBase::DeAllocateSegment(const FInfo* fileInfo, + uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + DeAllocateSegmentRequest request; + request.set_filename(fileInfo->fullPathName); + request.set_offset(segmentOffset); + + FillUserInfo(&request, fileInfo->userinfo); + + LOG(INFO) << "DeAllocateSegment: filename = " << fileInfo->fullPathName + << ", offset = " << segmentOffset + << ", logid = " << cntl->log_id(); + + curve::mds::CurveFSService_Stub stub(channel); + stub.DeAllocateSegment(cntl, &request, response, nullptr); +} + void MDSClientBase::RenameFile(const UserInfo_t& userinfo, const std::string& origin, const std::string& destination, diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h index 3a8b78aea1..88584ff82b 100644 --- a/src/client/mds_client_base.h +++ b/src/client/mds_client_base.h @@ -72,6 +72,10 @@ using curve::mds::CreateCloneFileRequest; using curve::mds::CreateCloneFileResponse; using curve::mds::SetCloneFileStatusRequest; using curve::mds::SetCloneFileStatusResponse; +using curve::mds::GetOrAllocateSegmentRequest; +using curve::mds::GetOrAllocateSegmentResponse; +using curve::mds::DeAllocateSegmentRequest; +using curve::mds::DeAllocateSegmentResponse; using curve::mds::CheckSnapShotStatusRequest; using curve::mds::CheckSnapShotStatusResponse; using curve::mds::ListSnapShotFileInfoRequest; @@ -332,6 +336,11 @@ class MDSClientBase { GetOrAllocateSegmentResponse* response, brpc::Controller* cntl, brpc::Channel* channel); + + void DeAllocateSegment(const FInfo* fileInfo, uint64_t segmentOffset, + DeAllocateSegmentResponse* response, + brpc::Controller* cntl, brpc::Channel* channel); + /** * @brief 重名文件 * @param:userinfo 用户信息 diff --git a/src/client/metacache.cpp b/src/client/metacache.cpp index be88ec2ed1..4c551eba9e 100644 --- a/src/client/metacache.cpp +++ b/src/client/metacache.cpp @@ -67,6 +67,12 @@ MetaCacheErrorType MetaCache::GetChunkInfoByIndex(ChunkIndex chunkidx, return MetaCacheErrorType::CHUNKINFO_NOT_FOUND; } +void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& cinfo) { + WriteLockGuard wrlk(rwlock4ChunkInfo_); + chunkindex2idMap_[cindex] = cinfo; +} + bool MetaCache::IsLeaderMayChange(LogicPoolID logicPoolId, CopysetID copysetId) { rwlock4ChunkInfo_.RDLock(); @@ -262,12 +268,6 @@ int MetaCache::UpdateLeader(LogicPoolID logicPoolId, return iter->second.UpdateLeaderInfo(csAddr); } -void MetaCache::UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& cinfo) { - WriteLockGuard wrlk(rwlock4ChunkInfo_); - chunkindex2idMap_[cindex] = cinfo; -} - void MetaCache::UpdateCopysetInfo(LogicPoolID logicPoolid, CopysetID copysetid, const CopysetInfo& csinfo) { const auto key = CalcLogicPoolCopysetID(logicPoolid, copysetid); @@ -415,5 +415,39 @@ CopysetInfo MetaCache::GetCopysetinfo(LogicPoolID lpid, CopysetID csid) { return CopysetInfo(); } +FileSegment* MetaCache::GetFileSegment(SegmentIndex segmentIndex) { + { + ReadLockGuard lk(rwlock4Segments_); + auto iter = segments_.find(segmentIndex); + if (iter != segments_.end()) { + return &iter->second; + } + } + + WriteLockGuard lk(rwlock4Segments_); + auto ret = segments_.emplace( + std::piecewise_construct, + std::forward_as_tuple(segmentIndex), + std::forward_as_tuple(segmentIndex, + fileInfo_.segmentsize, + metacacheopt_.discardGranularity)); + + return &(ret.first->second); +} + +void MetaCache::CleanChunksInSegment(SegmentIndex segmentIndex) { + WriteLockGuard lk(rwlock4chunkInfoMap_); + ChunkIndex beginChunkIndex = static_cast(segmentIndex) * + fileInfo_.segmentsize / fileInfo_.chunksize; + ChunkIndex endChunkIndex = static_cast(segmentIndex + 1) * + fileInfo_.segmentsize / fileInfo_.chunksize; + + auto currentIndex = beginChunkIndex; + while (currentIndex < endChunkIndex) { + chunkindex2idMap_.erase(currentIndex); + ++currentIndex; + } +} + } // namespace client } // namespace curve diff --git a/src/client/metacache.h b/src/client/metacache.h index df6739ee5b..21cf9ad54a 100644 --- a/src/client/metacache.h +++ b/src/client/metacache.h @@ -81,6 +81,12 @@ class MetaCache { virtual MetaCacheErrorType GetChunkInfoByIndex(ChunkIndex chunkidx, ChunkIDInfo_t* chunkinfo); + /** + * @brief Update cached chunk info by chunk index + */ + virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, + const ChunkIDInfo& chunkinfo); + /** * sender发送数据的时候需要知道对应的leader然后发送给对应的chunkserver * 如果get不到的时候，外围设置refresh为true，然后向chunkserver端拉取最新的 @@ -121,13 +127,8 @@ class MetaCache { virtual void UpdateCopysetInfo(LogicPoolID logicPoolId, CopysetID copysetId, const CopysetInfo& csinfo); - /** - * 通过chunk index更新chunkid信息 - * @param: index为待更新的chunk index - * @param: chunkinfo为需要更新的info信息 - */ - virtual void UpdateChunkInfoByIndex(ChunkIndex cindex, - const ChunkIDInfo& chunkinfo); + + /** * 通过chunk id更新chunkid信息 * @param: cid为chunkid @@ -247,6 +248,16 @@ class MetaCache { return fileInfo_.id; } + /** + * @brief Get file segment info about the segmentIndex + */ + FileSegment* GetFileSegment(SegmentIndex segmentIndex); + + /** + * @brief Clean chunks of this segment + */ + virtual void CleanChunksInSegment(SegmentIndex segmentIndex); + private: /** * @brief 从mds更新copyset复制组信息 @@ -286,6 +297,9 @@ class MetaCache { // chunkindex到chunkidinfo的映射表 CURVE_CACHELINE_ALIGNMENT ChunkIndexInfoMap chunkindex2idMap_; + CURVE_CACHELINE_ALIGNMENT RWLock rwlock4Segments_; + CURVE_CACHELINE_ALIGNMENT std::unordered_map segments_; // NOLINT + // logicalpoolid和copysetid到copysetinfo的映射表 CURVE_CACHELINE_ALIGNMENT CopysetInfoMap lpcsid2CopsetInfoMap_; diff --git a/src/client/metacache_struct.h b/src/client/metacache_struct.h index af841cf688..61235f2c2e 100644 --- a/src/client/metacache_struct.h +++ b/src/client/metacache_struct.h @@ -31,11 +31,17 @@ #include "include/curve_compiler_specific.h" #include "src/client/client_common.h" #include "src/common/concurrent/spinlock.h" +#include "src/common/concurrent/rw_lock.h" +#include "src/common/bitmap.h" namespace curve { namespace client { using curve::common::SpinLock; +using curve::common::ReadLockGuard; +using curve::common::WriteLockGuard; +using curve::common::Bitmap; +using curve::common::BthreadRWLock; // copyset内的chunkserver节点的基本信息 // 包含当前chunkserver的id信息，以及chunkserver的地址信息 @@ -264,6 +270,142 @@ inline bool operator==(const CopysetIDInfo& cpidinfo1, return cpidinfo1.cpid == cpidinfo2.cpid && cpidinfo1.lpid == cpidinfo2.lpid; } +class FileSegment { + public: + FileSegment(SegmentIndex segmentIndex, uint32_t segmentSize, + uint32_t discardGranularity) + : segmentIndex_(segmentIndex), + segmentSize_(segmentSize), + discardGranularity_(discardGranularity), + rwlock_(), + discardBitmap_(segmentSize_ / discardGranularity_), + chunks_() {} + + /** + * @brief Confirm if all bit was discarded + * @return Return true if if all bits are set, otherwise return false + */ + bool IsAllBitSet() const { + return discardBitmap_.NextClearBit(0) == curve::common::Bitmap::NO_POS; + } + + void AcquireReadLock() { + rwlock_.RDLock(); + } + + void AcquireWriteLock() { + rwlock_.WRLock(); + } + + void ReleaseLock() { + rwlock_.Unlock(); + } + + /** + * @brief Get internal bitmap for unit-test + * @return Internal bitmap + */ + Bitmap& GetBitmap() { + return discardBitmap_; + } + + void SetBitmap(const uint64_t offset, const uint64_t length); + void ClearBitmap(const uint64_t offset, const uint64_t length); + + void ClearBitmap() { + discardBitmap_.Clear(); + } + + private: + const SegmentIndex segmentIndex_; + const uint32_t segmentSize_; + const uint32_t discardGranularity_; + BthreadRWLock rwlock_; + Bitmap discardBitmap_; + std::unordered_map chunks_; +}; + +inline void FileSegment::SetBitmap(const uint64_t offset, + const uint64_t length) { + if (length < discardGranularity_) { + return; + } + + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Set(); + } + + auto res = std::div(static_cast(offset), + static_cast(discardGranularity_)); + uint32_t startIndex = res.quot; + if (res.rem != 0) { + ++startIndex; + } + + uint32_t endIndex = (offset + length) / discardGranularity_ - 1; + + return discardBitmap_.Set(startIndex, endIndex); +} + +inline void FileSegment::ClearBitmap(const uint64_t offset, + const uint64_t length) { + if (offset == 0 && length == segmentSize_) { + return discardBitmap_.Clear(); + } + + uint32_t startIndex = offset / discardGranularity_; + auto res = std::div(static_cast(offset + length), + static_cast(discardGranularity_)); + + uint32_t endIndex = res.quot; + if (res.rem == 0 && endIndex != 0) { + --endIndex; + } + + return discardBitmap_.Clear(startIndex, endIndex); +} + +enum class FileSegmentLockType { + Read, + Write +}; + +template +class FileSegmentLockGuard { + public: + explicit FileSegmentLockGuard(FileSegment* segment) : segment_(segment) { + Lock(); + } + + FileSegmentLockGuard(const FileSegmentLockGuard&) = delete; + FileSegmentLockGuard& operator=(const FileSegmentLockGuard&) = delete; + + ~FileSegmentLockGuard() { + UnLock(); + } + + void Lock() { + if (type == FileSegmentLockType::Read) { + segment_->AcquireReadLock(); + } else { + segment_->AcquireWriteLock(); + } + } + + void UnLock() { + segment_->ReleaseLock(); + } + + private: + FileSegment* segment_; +}; + +using FileSegmentReadLockGuard = + FileSegmentLockGuard; + +using FileSegmentWriteLockGuard = + FileSegmentLockGuard; + } // namespace client } // namespace curve diff --git a/src/client/splitor.cpp b/src/client/splitor.cpp index 275a245069..2dc68dc6b3 100644 --- a/src/client/splitor.cpp +++ b/src/client/splitor.cpp @@ -131,10 +131,29 @@ int Splitor::SingleChunkIO2ChunkRequests( bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector* targetlist, butil::IOBuf* data, off_t off, size_t len, - MDSClient* mdsclient, const FInfo_t* fileinfo, + MDSClient* mdsclient, const FInfo_t* fileInfo, ChunkIndex chunkidx) { const auto maxSplitSizeBytes = 1024 * iosplitopt_.fileIOSplitMaxSizeKB; + lldiv_t res = std::div( + static_cast(chunkidx) * fileInfo->chunksize, // NOLINT + static_cast(fileInfo->segmentsize)); // NOLINT + + SegmentIndex segmentIndex = res.quot; + uint64_t startOffset = res.rem + off; + + FileSegment* fileSegment = metaCache->GetFileSegment(segmentIndex); + + if (iotracker->Optype() == OpType::DISCARD) { + return MarkDiscardBitmap(iotracker, fileSegment, segmentIndex, + startOffset, len); + } + + FileSegmentReadLockGuard lk(fileSegment); + + // clear discard bitmap + fileSegment->ClearBitmap(startOffset, len); + ChunkIDInfo chunkIdInfo; MetaCacheErrorType errCode = metaCache->GetChunkInfoByIndex(chunkidx, &chunkIdInfo); @@ -147,8 +166,8 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, iotracker->Optype() == OpType::READ ? false : true; if (false == GetOrAllocateSegment( isAllocateSegment, - static_cast(chunkidx) * fileinfo->chunksize, - mdsclient, metaCache, fileinfo, chunkidx)) { + static_cast(chunkidx) * fileInfo->chunksize, + mdsclient, metaCache, fileInfo, chunkidx)) { return false; } @@ -168,7 +187,7 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, std::vector templist; ret = SingleChunkIO2ChunkRequests(iotracker, metaCache, &templist, chunkIdInfo, data, off, len, - fileinfo->seqnum); + fileInfo->seqnum); for (auto& ctx : templist) { ctx->appliedindex_ = appliedindex_; @@ -179,6 +198,12 @@ bool Splitor::AssignInternal(IOTracker* iotracker, MetaCache* metaCache, targetlist->insert(targetlist->end(), templist.begin(), templist.end()); + if (ret == 0) { + // acquire filesegment read lock + fileSegment->AcquireReadLock(); + iotracker->segmentLocks_.emplace_back(fileSegment); + } + return ret == 0; } @@ -355,6 +380,19 @@ int Splitor::SplitForStripe(IOTracker* iotracker, MetaCache* metaCache, return 0; } +bool Splitor::MarkDiscardBitmap(IOTracker* iotracker, FileSegment* fileSegment, + SegmentIndex segmentIndex, uint64_t offset, + uint64_t len) { + FileSegmentWriteLockGuard lk(fileSegment); + fileSegment->SetBitmap(offset, len); + + if (fileSegment->IsAllBitSet()) { + iotracker->discardSegments_.push_back(segmentIndex); + } + + return true; +} + RequestSourceInfo Splitor::CalcRequestSourceInfo(IOTracker* ioTracker, MetaCache* metaCache, ChunkIndex chunkIdx) { diff --git a/src/client/splitor.h b/src/client/splitor.h index f1e3b90cc4..4860d95912 100644 --- a/src/client/splitor.h +++ b/src/client/splitor.h @@ -132,6 +132,12 @@ class Splitor { butil::IOBuf* data, off_t offset, size_t length, MDSClient* mdsclient, const FInfo_t* fileInfo); + static bool MarkDiscardBitmap(IOTracker* iotracker, + FileSegment* fileSegment, + SegmentIndex segmentIndex, + uint64_t offset, + uint64_t len); + private: // IO拆分模块所使用的配置信息 static IOSplitOption iosplitopt_; diff --git a/src/common/encode.h b/src/common/encode.h index ab23b24ad2..ae2116afa6 100644 --- a/src/common/encode.h +++ b/src/common/encode.h @@ -27,6 +27,8 @@ namespace curve { namespace common { + +// NOTE: value passed to this function will convert to `uint64_t' static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[0] = (value >> 56) & 0xff; buf[1] = (value >> 48) & 0xff; @@ -37,6 +39,7 @@ static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[6] = (value >> 8) & 0xff; buf[7] = value & 0xff; } + } // namespace common } // namespace curve diff --git a/src/common/namespace_define.h b/src/common/namespace_define.h index 3f142d527f..442b7e36c4 100644 --- a/src/common/namespace_define.h +++ b/src/common/namespace_define.h @@ -59,10 +59,14 @@ const char SNAPINFOKEYEND[] = "12"; const char CLONEINFOKEYPREFIX[] = "12"; const char CLONEINFOKEYEND[] = "13"; +const char DISCARDSEGMENTKEYPREFIX[] = "13"; +const char DISCARDSEGMENTKEYEND[] = "14"; + // TODO(hzsunjianliang): if use single prefix for snapshot file? const int COMMON_PREFIX_LENGTH = 2; const int LEADER_PREFIX_LENGTH = 8; const int SEGMENTKEYLEN = 18; +const int DISCARDSEGMENTKEYLEN = 26; } // namespace common } // namespace curve diff --git a/src/kvstorageclient/etcd_client.cpp b/src/kvstorageclient/etcd_client.cpp index 96147fcde9..1d80cc96b5 100644 --- a/src/kvstorageclient/etcd_client.cpp +++ b/src/kvstorageclient/etcd_client.cpp @@ -96,8 +96,26 @@ int EtcdClientImp::Get(const std::string &key, std::string *out) { return errCode; } -int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, - std::vector *out) { +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector* out) { + assert(out != nullptr); + out->clear(); + + std::vector> kvs; + int errCode = List(startKey, endKey, &kvs); + if (errCode != EtcdErrCode::EtcdOK) { + return errCode; + } + + for (const auto& kv : kvs) { + out->push_back(kv.second); + } + + return errCode; +} + +int EtcdClientImp::List(const std::string& startKey, const std::string& endKey, + std::vector>* out) { assert(out != nullptr); out->clear(); @@ -113,8 +131,9 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, needRetry = NeedRetry(errCode); if (res.r0 != EtcdErrCode::EtcdOK) { LOG(WARNING) << "list file of [start:" << startKey - << ", end:" << endKey << "] err: " << res.r0 - << ", retry: " << retry << ", needRetry: " << needRetry; + << ", end:" << endKey << "] err: " << res.r0 + << ", retry: " << retry + << ", needRetry: " << needRetry; } else { for (int i = 0; i < res.r2; i++) { EtcdClientGetMultiObject_return objRes = @@ -127,6 +146,7 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey, } out->emplace_back( + std::string(objRes.r3, objRes.r3 + objRes.r4), std::string(objRes.r1, objRes.r1 + objRes.r2)); free(objRes.r1); free(objRes.r3); diff --git a/src/kvstorageclient/etcd_client.h b/src/kvstorageclient/etcd_client.h index 61219dc6e5..16aec44e6a 100644 --- a/src/kvstorageclient/etcd_client.h +++ b/src/kvstorageclient/etcd_client.h @@ -26,6 +26,7 @@ #include #include #include +#include namespace curve { namespace kvstorage { @@ -78,6 +79,18 @@ class KVStorageClient { virtual int List(const std::string &startKey, const std::string &endKey, std::vector *values) = 0; + /** + * @brief List all the key and values between [startKey, endKey) + * + * @param[in] startKey + * @param[in] endKey + * @param[out] out store key/value pairs that key is between [startKey, endKey) + * + * @return error code + */ + virtual int List(const std::string& startKey, const std::string& endKey, + std::vector>* out) = 0; + /** * @brief Delete Delete the value of the specified key * @@ -151,6 +164,9 @@ class EtcdClientImp : public KVStorageClient { int List(const std::string &startKey, const std::string &endKey, std::vector *values) override; + int List(const std::string& startKey, const std::string& endKey, + std::vector >* out) override; + int Delete(const std::string &key) override; int DeleteRewithRevision( diff --git a/src/mds/nameserver2/clean_core.cpp b/src/mds/nameserver2/clean_core.cpp index d90e891a2d..cf5907d5fd 100644 --- a/src/mds/nameserver2/clean_core.cpp +++ b/src/mds/nameserver2/clean_core.cpp @@ -120,25 +120,15 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, return StatusCode::kCommonFileDeleteError; } - // delete chunks in chunkserver - LogicalPoolID logicalPoolID = segment.logicalpoolid(); - uint32_t chunkNum = segment.chunks_size(); - for (uint32_t j = 0; j != chunkNum; j++) { - SeqNum seq = commonFile.seqnum(); - int ret = copysetClient_->DeleteChunk(logicalPoolID, - segment.chunks()[j].copysetid(), - segment.chunks()[j].chunkid(), - seq); - if (ret != 0) { - LOG(ERROR) << "Clean common File Error: " - << "DeleteChunk Error" - << ", ret = " << ret - << ", inodeid = " << commonFile.id() - << ", filename = " << commonFile.filename() - << ", sequenceNum = " << seq; - progress->SetStatus(TaskStatus::FAILED); - return StatusCode::kCommonFileDeleteError; - } + int ret = DeleteChunksInSegment(segment, commonFile.seqnum()); + if (ret != 0) { + LOG(ERROR) << "Clean common File Error: " + << ", ret = " << ret + << ", inodeid = " << commonFile.id() + << ", filename = " << commonFile.filename() + << ", sequenceNum = " << commonFile.seqnum(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::kCommonFileDeleteError; } // delete segment @@ -176,5 +166,81 @@ StatusCode CleanCore::CleanFile(const FileInfo & commonFile, progress->SetStatus(TaskStatus::SUCCESS); return StatusCode::kOK; } + +StatusCode CleanCore::CleanDiscardSegment( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, TaskProgress* progress) { + const FileInfo& fileInfo = discardSegmentInfo.fileinfo(); + const PageFileSegment& segment = discardSegmentInfo.pagefilesegment(); + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + const SeqNum seq = fileInfo.seqnum(); + + LOG(INFO) << "Start CleanDiscardSegment, filename = " << fileInfo.filename() + << ", inodeid = " << fileInfo.id() + << ", segment offset = " << segment.startoffset(); + + butil::Timer timer; + timer.start(); + + // delete chunks + int ret = DeleteChunksInSegment(segment, seq); + if (ret != 0) { + LOG(ERROR) << "CleanDiscardSegment failed, DeleteChunk Error, ret = " + << ret << ", filename = " << fileInfo.filename() + << ", inodeid = " << fileInfo.id() + << ", segment offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + // delete segment + int64_t revision; + auto storeRet = storage_->CleanDiscardSegment(segment.segmentsize(), + cleanSegmentKey, &revision); + if (storeRet != StoreStatus::OK) { + LOG(ERROR) << "CleanDiscardSegment failed, filename = " + << fileInfo.filename() + << ", offset = " << segment.startoffset(); + progress->SetStatus(TaskStatus::FAILED); + return StatusCode::KInternalError; + } + + allocStatistic_->DeAllocSpace(segment.logicalpoolid(), + segment.segmentsize(), revision); + progress->SetProgress(100); + progress->SetStatus(TaskStatus::SUCCESS); + + timer.stop(); + LOG(INFO) << "CleanDiscardSegment success, filename = " + << fileInfo.filename() << ", inodeid = " << fileInfo.id() + << ", segment offset = " << segment.startoffset() << ", cost " + << timer.m_elapsed(1.0) << " ms"; + + return StatusCode::kOK; +} + +int CleanCore::DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq) { + const LogicalPoolID logicalPoolId = segment.logicalpoolid(); + for (int i = 0; i < segment.chunks_size(); ++i) { + int ret = copysetClient_->DeleteChunk( + logicalPoolId, + segment.chunks()[i].copysetid(), + segment.chunks()[i].chunkid(), + seq); + + if (ret != 0) { + LOG(ERROR) << "DeleteChunk failed, ret = " << ret + << ", logicalpoolid = " << logicalPoolId + << ", copysetid = " << segment.chunks()[i].copysetid() + << ", chunkid = " << segment.chunks()[i].chunkid() + << ", seq = " << seq; + return ret; + } + } + + return 0; +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_core.h b/src/mds/nameserver2/clean_core.h index 540794c1ac..0cb4f3f8ab 100644 --- a/src/mds/nameserver2/clean_core.h +++ b/src/mds/nameserver2/clean_core.h @@ -24,6 +24,7 @@ #define SRC_MDS_NAMESERVER2_CLEAN_CORE_H_ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/common/mds_define.h" #include "src/mds/nameserver2/task_progress.h" @@ -65,7 +66,17 @@ class CleanCore { StatusCode CleanFile(const FileInfo & commonFile, TaskProgress* progress); + /** + * @brief clean discarded segment and chunks + */ + StatusCode CleanDiscardSegment(const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo, + TaskProgress* progress); + private: + int DeleteChunksInSegment(const PageFileSegment& segment, + const SeqNum& seq); + std::shared_ptr storage_; std::shared_ptr copysetClient_; std::shared_ptr allocStatistic_; diff --git a/src/mds/nameserver2/clean_manager.cpp b/src/mds/nameserver2/clean_manager.cpp index de7447eda4..62050e26a5 100644 --- a/src/mds/nameserver2/clean_manager.cpp +++ b/src/mds/nameserver2/clean_manager.cpp @@ -57,6 +57,15 @@ bool CleanManager::SubmitDeleteCommonFileJob(const FileInfo &fileInfo) { return taskMgr_->PushTask(commonFileCleanTask); } +bool CleanManager::SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) { + auto task = std::make_shared( + cleanCore_, cleanSegmentKey, discardSegmentInfo); + task->SetTaskID(reinterpret_cast(task.get())); + return taskMgr_->PushTask(task); +} + bool CleanManager::RecoverCleanTasks(void) { // load task from store std::vector snapShotFiles; @@ -94,5 +103,51 @@ std::shared_ptr CleanManager::GetTask(TaskIDType id) { return taskMgr_->GetTask(id); } +bool CleanDiscardSegmentTask::Start() { + if (!running_.exchange(true)) { + LOG(INFO) << "start CleanDiscardSegmentTask"; + taskThread_ = curve::common::Thread( + &CleanDiscardSegmentTask::ScanAndExecTask, this); + return true; + } + + LOG(WARNING) << "CleanDiscardSegmentTask has already started"; + return false; +} + +bool CleanDiscardSegmentTask::Stop() { + if (running_.exchange(false)) { + LOG(INFO) << "stop CleanDiscardSegmentTask..."; + sleeper_.interrupt(); + taskThread_.join(); + LOG(INFO) << "stop CleanDiscardSegmentTask success"; + return true; + } + + LOG(WARNING) << "CleanDiscardSegmentTask has already stopped"; + return false; +} + +void CleanDiscardSegmentTask::ScanAndExecTask() { + std::map discardSegments; + + while (sleeper_.wait_for(std::chrono::milliseconds(scanIntervalMs_))) { + discardSegments.clear(); + auto err = storage_->ListDiscardSegment(&discardSegments); + if (err != StoreStatus::OK) { + LOG(ERROR) << "ListDiscardSegment failed"; + continue; + } + + for (const auto& kv : discardSegments) { + if (!cleanManager_->SubmitCleanDiscardSegmentJob(kv.first, + kv.second)) { + LOG(ERROR) << "SubmitCleanDiscardSegmentJob failed"; + continue; + } + } + } +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/clean_manager.h b/src/mds/nameserver2/clean_manager.h index b3940fd787..8aa8539085 100644 --- a/src/mds/nameserver2/clean_manager.h +++ b/src/mds/nameserver2/clean_manager.h @@ -23,16 +23,21 @@ #ifndef SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ #define SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ +#include #include +#include #include "proto/nameserver2.pb.h" #include "src/mds/nameserver2/clean_task_manager.h" #include "src/mds/nameserver2/clean_core.h" #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" +#include "src/common/concurrent/concurrent.h" namespace curve { namespace mds { +class CleanDiscardSegmentTask; + class CleanManagerInterface { public: virtual ~CleanManagerInterface() {} @@ -40,6 +45,10 @@ class CleanManagerInterface { std::shared_ptr entity) = 0; virtual std::shared_ptr GetTask(TaskIDType id) = 0; virtual bool SubmitDeleteCommonFileJob(const FileInfo&) = 0; + + virtual bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) = 0; }; /** * CleanManager 用于异步清理 删除快照对应的数据 @@ -61,6 +70,10 @@ class CleanManager : public CleanManagerInterface { bool SubmitDeleteCommonFileJob(const FileInfo&fileInfo) override; + bool SubmitCleanDiscardSegmentJob( + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) override; + bool RecoverCleanTasks(void); std::shared_ptr GetTask(TaskIDType id) override; @@ -71,6 +84,35 @@ class CleanManager : public CleanManagerInterface { std::shared_ptr taskMgr_; }; +class CleanDiscardSegmentTask { + public: + CleanDiscardSegmentTask(std::shared_ptr cleanManager, + std::shared_ptr storage, + uint32_t scanIntervalMs) + : cleanManager_(cleanManager), + storage_(storage), + scanIntervalMs_(scanIntervalMs), + sleeper_(), + running_(false), + taskThread_() {} + + bool Start(); + + bool Stop(); + + private: + void ScanAndExecTask(); + + private: + std::shared_ptr cleanManager_; + std::shared_ptr storage_; + uint32_t scanIntervalMs_; + curve::common::InterruptibleSleeper sleeper_; + curve::common::Atomic running_; + curve::common::Thread taskThread_; +}; + } // namespace mds } // namespace curve + #endif // SRC_MDS_NAMESERVER2_CLEAN_MANAGER_H_ diff --git a/src/mds/nameserver2/clean_task.h b/src/mds/nameserver2/clean_task.h index e6f765bb78..bdf9ce74ca 100644 --- a/src/mds/nameserver2/clean_task.h +++ b/src/mds/nameserver2/clean_task.h @@ -25,6 +25,7 @@ #include #include //NOLINT +#include #include //NOLINT #include //NOLINT #include "proto/nameserver2.pb.h" @@ -160,6 +161,28 @@ class CommonFileCleanTask: public Task { FileInfo fileInfo_; }; +class SegmentCleanTask : public Task { + public: + SegmentCleanTask(std::shared_ptr cleanCore, + const std::string& cleanSegmentKey, + const DiscardSegmentInfo& discardSegmentInfo) + : Task(), + cleanCore_(cleanCore), + cleanSegmentKey_(cleanSegmentKey), + discardSegmentInfo_(discardSegmentInfo) {} + + void Run() override { + cleanCore_->CleanDiscardSegment(cleanSegmentKey_, discardSegmentInfo_, + GetMutableTaskProgress()); + return; + } + + private: + std::shared_ptr cleanCore_; + std::string cleanSegmentKey_; + DiscardSegmentInfo discardSegmentInfo_; +}; + } // namespace mds } // namespace curve #endif // SRC_MDS_NAMESERVER2_CLEAN_TASK_H_ diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp index 80d63eb6d0..86496161c3 100644 --- a/src/mds/nameserver2/curvefs.cpp +++ b/src/mds/nameserver2/curvefs.cpp @@ -42,6 +42,23 @@ using curve::mds::topology::CopySetIdType; namespace curve { namespace mds { + +inline bool CurveFS::CheckSegmentOffset(const FileInfo& fileInfo, + uint64_t offset) const { + if (offset % fileInfo.segmentsize() != 0) { + LOG(ERROR) << "offset not align with segment, offset = " << offset + << ", file segmentsize = " << fileInfo.segmentsize(); + return false; + } + + if (offset + fileInfo.segmentsize() > fileInfo.length()) { + LOG(ERROR) << "bigger than file length"; + return false; + } + + return true; +} + bool CurveFS::InitRecycleBinDir() { FileInfo recyclebinFileInfo; @@ -1160,6 +1177,74 @@ StatusCode CurveFS::GetOrAllocateSegment(const std::string & filename, } } +StatusCode CurveFS::DeAllocateSegment(const std::string& fileName, + uint64_t offset) { + FileInfo fileInfo; + auto ret = GetFileInfo(fileName, &fileInfo); + if (ret != StatusCode::kOK) { + LOG(ERROR) << "get source file error, errCode = " << ret; + return ret; + } + + if (fileInfo.filetype() != FileType::INODE_PAGEFILE) { + LOG(ERROR) << "Only PAGEFILE support discard, filename = " << fileName + << ", offset = " << offset; + return StatusCode::kNotSupported; + } + + if (CheckSegmentOffset(fileInfo, offset) == false) { + LOG(ERROR) << "DeAllocateSegment check offset failed, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kParaError; + } + + if (fileInfo.filestatus() == FileStatus::kFileBeingCloned) { + LOG(WARNING) + << "DeAllocateSegment failed, file is being cloned, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kNotSupported; + } + + PageFileSegment segment; + auto storeRet = storage_->GetSegment(fileInfo.id(), offset, &segment); + if (StoreStatus::KeyNotExist == storeRet) { + LOG(WARNING) << "DeAllocateSegment segment not exist, filename = " + << fileName << ", offset = " << offset; + return StatusCode::kSegmentNotAllocated; + } + + if (segment.startoffset() != offset) { + LOG(ERROR) + << "DeAllocateSegment check offset failed, segment startoffset = " + << segment.startoffset() << ", request offset = " << offset + << ", filename = " << fileName; + return StatusCode::kParaError; + } + + std::vector snapShotFiles; + if (storage_->ListSnapshotFile(fileInfo.id(), fileInfo.id() + 1, + &snapShotFiles) != StoreStatus::OK) { + LOG(WARNING) << fileName << " list snapshot failed"; + return StatusCode::kStorageError; + } + + if (!snapShotFiles.empty()) { + LOG(WARNING) << fileName + << " exist snapshot, num = " << snapShotFiles.size(); + return StatusCode::kFileUnderSnapShot; + } + + storeRet = storage_->DiscardSegment(fileInfo, segment); + if (storeRet != StoreStatus::OK) { + LOG(WARNING) << "Storage CleanSegment return error, filename = " + << fileName << ", offset = " << offset + << ", error = " << storeRet; + return StatusCode::kStorageError; + } + + return StatusCode::kOK; +} + StatusCode CurveFS::CreateSnapShotFile(const std::string &fileName, FileInfo *snapshotFileInfo) { FileInfo parentFileInfo; diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h index 61c38cc731..eb0a459687 100644 --- a/src/mds/nameserver2/curvefs.h +++ b/src/mds/nameserver2/curvefs.h @@ -273,6 +273,14 @@ class CurveFS { offset_t offset, bool allocateIfNoExist, PageFileSegment *segment); + /** + * @brief deallocate file segment start at offset + * @param filename + * @param offset + * @return On success, return StatusCode::kOK + */ + StatusCode DeAllocateSegment(const std::string& filename, uint64_t offset); + /** * @brief get the root file info * @param @@ -622,6 +630,8 @@ class CurveFS { const std::string& signature, uint64_t date); + bool CheckSegmentOffset(const FileInfo& fileInfo, uint64_t offset) const; + StatusCode CheckPathOwnerInternal(const std::string &filename, const std::string &owner, const std::string &signature, diff --git a/src/mds/nameserver2/helper/namespace_helper.cpp b/src/mds/nameserver2/helper/namespace_helper.cpp index 5eb2bfd4f8..4a27952864 100644 --- a/src/mds/nameserver2/helper/namespace_helper.cpp +++ b/src/mds/nameserver2/helper/namespace_helper.cpp @@ -23,6 +23,7 @@ #include #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/string_util.h" +#include "src/common/timeutility.h" #include "src/common/namespace_define.h" using ::curve::common::COMMON_PREFIX_LENGTH; @@ -31,6 +32,9 @@ using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SEGMENTKEYLEN; using ::curve::common::SEGMENTINFOKEYPREFIX; using ::curve::common::SEGMENTALLOCSIZEKEY; +using ::curve::common::DISCARDSEGMENTKEYLEN; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -68,6 +72,18 @@ std::string NameSpaceStorageCodec::EncodeSegmentStoreKey(uint64_t inodeID, return storeKey; } +std::string NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey( + const InodeID inodeId, const uint64_t offset) { + std::string storeKey; + storeKey.resize(DISCARDSEGMENTKEYLEN); + ::memcpy(&(storeKey[0]), DISCARDSEGMENTKEYPREFIX, COMMON_PREFIX_LENGTH); + ::curve::common::EncodeBigEndian(&(storeKey[2]), inodeId); + ::curve::common::EncodeBigEndian(&(storeKey[10]), offset); + ::curve::common::EncodeBigEndian( + &(storeKey[18]), curve::common::TimeUtility::GetTimeofDayUs()); + return storeKey; +} + bool NameSpaceStorageCodec::EncodeFileInfo(const FileInfo &fileInfo, std::string *out) { return fileInfo.SerializeToString(out); @@ -132,5 +148,16 @@ bool NameSpaceStorageCodec::DecodeSegmentAllocValue( return true; } + +bool NameSpaceStorageCodec::EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out) { + return info.SerializeToString(out); +} + +bool NameSpaceStorageCodec::DecodeDiscardSegment( + const std::string& info, DiscardSegmentInfo* discardSegmentInfo) { + return discardSegmentInfo->ParseFromString(info); +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/helper/namespace_helper.h b/src/mds/nameserver2/helper/namespace_helper.h index 54ed099ba7..8eacb04c05 100644 --- a/src/mds/nameserver2/helper/namespace_helper.h +++ b/src/mds/nameserver2/helper/namespace_helper.h @@ -38,6 +38,8 @@ class NameSpaceStorageCodec { static std::string EncodeSnapShotFileStoreKey(uint64_t parentID, const std::string &fileName); static std::string EncodeSegmentStoreKey(uint64_t inodeID, offset_t offset); + static std::string EncodeDiscardSegmentStoreKey(const InodeID inodeId, + const uint64_t offset); static bool EncodeFileInfo(const FileInfo &finlInfo, std::string *out); static bool DecodeFileInfo(const std::string info, FileInfo *fileInfo); @@ -46,6 +48,11 @@ class NameSpaceStorageCodec { static std::string EncodeID(uint64_t value); static bool DecodeID(const std::string &value, uint64_t *out); + static bool EncodeDiscardSegment(const DiscardSegmentInfo& info, + std::string* out); + static bool DecodeDiscardSegment(const std::string& info, + DiscardSegmentInfo* discardSegmentInfo); + static std::string EncodeSegmentAllocKey(uint16_t lid); static std::string EncodeSegmentAllocValue(uint16_t lid, uint64_t alloc); static bool DecodeSegmentAllocValue( diff --git a/src/mds/nameserver2/nameserverMetrics.cpp b/src/mds/nameserver2/metric.cpp similarity index 76% rename from src/mds/nameserver2/nameserverMetrics.cpp rename to src/mds/nameserver2/metric.cpp index 921713fb6a..3dd558b915 100644 --- a/src/mds/nameserver2/nameserverMetrics.cpp +++ b/src/mds/nameserver2/metric.cpp @@ -20,7 +20,7 @@ * Author: lixiaocui */ -#include "src/mds/nameserver2/nameserverMetrics.h" +#include "src/mds/nameserver2/metric.h" namespace curve { namespace mds { @@ -40,5 +40,18 @@ void NameserverCacheMetrics::UpdateRemoveFromCacheBytes(uint64_t size) { cacheBytes << (0 - size); } +void SegmentDiscardMetric::OnReceiveDiscardRequest(uint64_t size) { + pendingSegments_ << 1; + pendingSize_ << size; +} + +void SegmentDiscardMetric::OnDiscardFinish(uint64_t size) { + pendingSegments_ << -1; + pendingSize_ << -size; + + totalCleanedSegments_ << 1; + totalCleanedSize_ << size; +} + } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/nameserverMetrics.h b/src/mds/nameserver2/metric.h similarity index 66% rename from src/mds/nameserver2/nameserverMetrics.h rename to src/mds/nameserver2/metric.h index 7046b55f3c..f3fe49b755 100644 --- a/src/mds/nameserver2/nameserverMetrics.h +++ b/src/mds/nameserver2/metric.h @@ -20,8 +20,8 @@ * Author: lixiaocui */ -#ifndef SRC_MDS_NAMESERVER2_NAMESERVERMETRICS_H_ -#define SRC_MDS_NAMESERVER2_NAMESERVERMETRICS_H_ +#ifndef SRC_MDS_NAMESERVER2_METRIC_H_ +#define SRC_MDS_NAMESERVER2_METRIC_H_ #include #include @@ -62,8 +62,31 @@ class NameserverCacheMetrics { bvar::Adder cacheMiss; }; +class SegmentDiscardMetric { + public: + SegmentDiscardMetric() + : prefix_("mds_nameserver_discard"), + totalCleanedSegments_(prefix_ + "total_cleaned_segment_count"), + pendingSegments_(prefix_ + "pending_segment_count"), + totalCleanedSize_(prefix_ + "total_cleaned_size"), + pendingSize_(prefix_ + "pending_size") {} + + ~SegmentDiscardMetric() = default; + + void OnReceiveDiscardRequest(uint64_t size); + void OnDiscardFinish(uint64_t size); + + public: + const std::string prefix_; + + bvar::Adder totalCleanedSegments_; + bvar::Adder pendingSegments_; + + bvar::Adder totalCleanedSize_; + bvar::Adder pendingSize_; +}; + } // namespace mds } // namespace curve -#endif // SRC_MDS_NAMESERVER2_NAMESERVERMETRICS_H_ - +#endif // SRC_MDS_NAMESERVER2_METRIC_H_ diff --git a/src/mds/nameserver2/namespace_service.cpp b/src/mds/nameserver2/namespace_service.cpp index ba3f74e79b..3c3940ce5f 100644 --- a/src/mds/nameserver2/namespace_service.cpp +++ b/src/mds/nameserver2/namespace_service.cpp @@ -486,6 +486,85 @@ void NameSpaceService::GetOrAllocateSegment( return; } +void NameSpaceService::DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + butil::Timer timer; + timer.start(); + + if (!isPathValid(request->filename())) { + response->set_statuscode(StatusCode::kParaError); + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment request path is invalid, " + << request->ShortDebugString(); + return; + } + + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment request, " + << request->ShortDebugString(); + + FileWriteLockGuard guard(fileLockManager_, request->filename()); + + std::string signature; + if (request->has_signature()) { + signature = request->signature(); + } + + StatusCode retCode; + retCode = kCurveFS.CheckFileOwner(request->filename(), request->owner(), + signature, request->date()); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment CheckFileOwner fail, " + << request->ShortDebugString() + << ", retCode = " << retCode; + } + + return; + } + + retCode = + kCurveFS.DeAllocateSegment(request->filename(), request->offset()); + + timer.stop(); + if (retCode != StatusCode::kOK) { + response->set_statuscode(retCode); + if (google::ERROR != GetMdsLogLevel(retCode)) { + LOG(WARNING) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } else { + LOG(ERROR) << "logid = " << cntl->log_id() + << ", DeAllocateSegment fail, " + << request->ShortDebugString() + << ", statusCode = " << retCode + << ", StatusCode_Name = " << StatusCode_Name(retCode) + << ", cost " << timer.m_elapsed(0.0) << " ms"; + } + } else { + response->set_statuscode(StatusCode::kOK); + LOG(INFO) << "logid = " << cntl->log_id() << ", DeAllocateSegment ok, " + << request->ShortDebugString() << ", cost " + << timer.m_elapsed(0.0) << " ms"; + } + + return; +} + void NameSpaceService::RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_service.h b/src/mds/nameserver2/namespace_service.h index dc8f668038..020dcb7bc7 100644 --- a/src/mds/nameserver2/namespace_service.h +++ b/src/mds/nameserver2/namespace_service.h @@ -96,6 +96,12 @@ class NameSpaceService: public CurveFSService { ::curve::mds::GetOrAllocateSegmentResponse* response, ::google::protobuf::Closure* done) override; + void DeAllocateSegment( + ::google::protobuf::RpcController* controller, + const ::curve::mds::DeAllocateSegmentRequest* request, + ::curve::mds::DeAllocateSegmentResponse* response, + ::google::protobuf::Closure* done) override; + void RenameFile(::google::protobuf::RpcController* controller, const ::curve::mds::RenameFileRequest* request, ::curve::mds::RenameFileResponse* response, diff --git a/src/mds/nameserver2/namespace_storage.cpp b/src/mds/nameserver2/namespace_storage.cpp index c65dfb46a2..9191178884 100644 --- a/src/mds/nameserver2/namespace_storage.cpp +++ b/src/mds/nameserver2/namespace_storage.cpp @@ -21,12 +21,15 @@ */ #include +#include #include "src/mds/nameserver2/namespace_storage.h" #include "src/mds/nameserver2/helper/namespace_helper.h" #include "src/common/namespace_define.h" using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX; using ::curve::common::SNAPSHOTFILEINFOKEYEND; +using ::curve::common::DISCARDSEGMENTKEYPREFIX; +using ::curve::common::DISCARDSEGMENTKEYEND; namespace curve { namespace mds { @@ -37,10 +40,8 @@ std::ostream& operator << (std::ostream & os, StoreStatus &s) { } NameServerStorageImp::NameServerStorageImp( - std::shared_ptr client, std::shared_ptr cache) { - this->client_ = client; - this->cache_ = cache; -} + std::shared_ptr client, std::shared_ptr cache) + : client_(client), cache_(cache), discardMetric_() {} StoreStatus NameServerStorageImp::PutFile(const FileInfo &fileInfo) { std::string storeKey; @@ -525,6 +526,93 @@ StoreStatus NameServerStorageImp::DeleteSegment( return getErrorCode(errCode); } +StoreStatus NameServerStorageImp::ListDiscardSegment( + std::map* discardSegments) { + assert(discardSegments != nullptr); + + std::vector> out; + int err = + client_->List(DISCARDSEGMENTKEYPREFIX, DISCARDSEGMENTKEYEND, &out); + if (err != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "ListDiscardSegment return error, err = " << err; + return StoreStatus::InternalError; + } + + for (const auto& kv : out) { + DiscardSegmentInfo info; + if (!NameSpaceStorageCodec::DecodeDiscardSegment(kv.second, &info)) { + LOG(ERROR) << "Decode DiscardSegment failed"; + return StoreStatus::InternalError; + } + + discardSegments->emplace(kv.first, std::move(info)); + } + + return StoreStatus::OK; +} + +StoreStatus NameServerStorageImp::DiscardSegment( + const FileInfo& fileInfo, const PageFileSegment& segment) { + const uint64_t inodeId = fileInfo.id(); + const uint64_t offset = segment.startoffset(); + const std::string segmentKey = + NameSpaceStorageCodec::EncodeSegmentStoreKey(inodeId, offset); + const std::string cleanSegmentKey = + NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey(inodeId, offset); + + std::string encodeSegment; + if (!NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)) { + return StoreStatus::InternalError; + } + + std::string encodeDiscardSegment; + DiscardSegmentInfo discardInfo; + discardInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardInfo.set_allocated_pagefilesegment(new PageFileSegment(segment)); + if (!NameSpaceStorageCodec::EncodeDiscardSegment(discardInfo, + &encodeDiscardSegment)) { + return StoreStatus::InternalError; + } + + Operation op1{ + OpType::OpDelete, + const_cast(segmentKey.c_str()), + const_cast(encodeSegment.c_str()), + segmentKey.size(), encodeSegment.size()}; + Operation op2{ + OpType::OpPut, + const_cast(cleanSegmentKey.c_str()), + const_cast(encodeDiscardSegment.c_str()), + cleanSegmentKey.size(), encodeDiscardSegment.size()}; + + std::vector ops{op1, op2}; + auto errCode = client_->TxnN(ops); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "Discard segment failed, filename: " + << fileInfo.filename() << ", inodeid = " << inodeId + << ", offset: " << offset << ", errCode: " << errCode; + } else { + cache_->Remove(segmentKey); + discardMetric_.OnReceiveDiscardRequest(segment.segmentsize()); + } + + return getErrorCode(errCode); +} + +StoreStatus NameServerStorageImp::CleanDiscardSegment(uint64_t segmentSize, + const std::string& key, + int64_t* revision) { + int errCode = client_->DeleteRewithRevision(key, revision); + if (errCode != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "CleanDiscardSegment failed, key = " << key + << ", err = " << errCode; + } else { + discardMetric_.OnDiscardFinish(segmentSize); + } + + return getErrorCode(errCode); +} + StoreStatus NameServerStorageImp::SnapShotFile(const FileInfo *originFInfo, const FileInfo *snapshotFInfo) { std::string originFileKey; diff --git a/src/mds/nameserver2/namespace_storage.h b/src/mds/nameserver2/namespace_storage.h index f30497843f..f8b42f92c9 100644 --- a/src/mds/nameserver2/namespace_storage.h +++ b/src/mds/nameserver2/namespace_storage.h @@ -34,6 +34,7 @@ #include "src/mds/common/mds_define.h" #include "src/kvstorageclient/etcd_client.h" #include "src/mds/nameserver2/namespace_storage_cache.h" +#include "src/mds/nameserver2/metric.h" namespace curve { namespace mds { @@ -220,6 +221,37 @@ class NameServerStorage { virtual StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) = 0; + /** + * @brief Move segment metadata from SegmentTable to DiscardSegmentTable, + * another background task will delete all chunks and delete segment + * in DiscardSegmentTable + * @param[in] id: Inode ID of the target file + * @param[in] off: Offset of the target segment + * + * @return StoreStatus: error code + */ + virtual StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) = 0; + + /** + * @brief Remove discard segment from DiscardSegmentTable + * @param[in] segmentSize segment's size + * @param[in] key discard segment's key + * @param[out] revision: the version number of this operation + * @return On success, return StoreStatus::OK + */ + virtual StoreStatus CleanDiscardSegment(uint64_t segmentSize, + const std::string& key, + int64_t* revision) = 0; + + /** + * @brief list all segment from DiscardSegmentTable + * @param[out] store key and values + * @return On success, return StoreStatus::OK + */ + virtual StoreStatus ListDiscardSegment( + std::map* out) = 0; + /** * @brief SnapShotFile: Transaction for storing metadata of snapshotFile, * and update source file metadata @@ -245,9 +277,9 @@ class NameServerStorage { class NameServerStorageImp : public NameServerStorage { public: - explicit NameServerStorageImp( - std::shared_ptr client, std::shared_ptr cache); - ~NameServerStorageImp() {} + explicit NameServerStorageImp( + std::shared_ptr client, std::shared_ptr cache); + ~NameServerStorageImp() {} StoreStatus PutFile(const FileInfo & fileInfo) override; @@ -295,6 +327,16 @@ class NameServerStorageImp : public NameServerStorage { StoreStatus DeleteSegment( InodeID id, uint64_t off, int64_t *revision) override; + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override; + + StoreStatus CleanDiscardSegment(uint64_t segmentSize, + const std::string& key, + int64_t* revision) override; + + StoreStatus ListDiscardSegment( + std::map* out) override; + StoreStatus SnapShotFile(const FileInfo *originalFileInfo, const FileInfo * snapshotFileInfo) override; @@ -316,6 +358,9 @@ class NameServerStorageImp : public NameServerStorage { // underlying storage std::shared_ptr client_; + + // metric for discard + SegmentDiscardMetric discardMetric_; }; } // namespace mds } // namespace curve diff --git a/src/mds/nameserver2/namespace_storage_cache.h b/src/mds/nameserver2/namespace_storage_cache.h index 176cba0bb9..43e280d709 100644 --- a/src/mds/nameserver2/namespace_storage_cache.h +++ b/src/mds/nameserver2/namespace_storage_cache.h @@ -28,7 +28,7 @@ #include #include #include "src/common/concurrent/concurrent.h" -#include "src/mds/nameserver2/nameserverMetrics.h" +#include "src/mds/nameserver2/metric.h" namespace curve { namespace mds { diff --git a/src/mds/server/mds.cpp b/src/mds/server/mds.cpp index 461bf0b320..5c23905726 100644 --- a/src/mds/server/mds.cpp +++ b/src/mds/server/mds.cpp @@ -144,6 +144,9 @@ void MDS::Run() { LOG_IF(FATAL, !cleanManager_->Start()) << "start cleanManager fail."; // recover unfinished tasks cleanManager_->RecoverCleanTasks(); + + cleanDiscardSegmentTask_->Start(); + // start scheduler module coordinator_->Run(); // start brpc server @@ -166,6 +169,8 @@ void MDS::Stop() { kCurveFS.Uninit(); + cleanDiscardSegmentTask_->Stop(); + cleanManager_->Stop(); topologyMetricService_->Stop(); @@ -422,6 +427,17 @@ void MDS::InitCurveFS(const CurveFSOption& curveFSOptions) { // init clean manager InitCleanManager(); + // init clean discard segment task + uint32_t scanDiscardTaskIntervalMs = 0; + if (!conf_->GetUInt32Value("mds.segment.discard.scanIntevalMs", + &scanDiscardTaskIntervalMs)) { + LOG(FATAL) << "Load mds.segment.discard.scanIntevalMs from config " + "file failed"; + } + + cleanDiscardSegmentTask_ = std::make_shared( + cleanManager_, nameServerStorage_, scanDiscardTaskIntervalMs); + // init FileRecordManager auto fileRecordManager = std::make_shared(); diff --git a/src/mds/server/mds.h b/src/mds/server/mds.h index 8d64caf757..edf21f4c88 100644 --- a/src/mds/server/mds.h +++ b/src/mds/server/mds.h @@ -225,6 +225,7 @@ class MDS { std::shared_ptr topologyMetricService_; std::shared_ptr topologyServiceManager_; std::shared_ptr cleanManager_; + std::shared_ptr cleanDiscardSegmentTask_; std::shared_ptr coordinator_; std::shared_ptr heartbeatManager_; char* etcdEndpoints_; diff --git a/test/chunkserver/clone/BUILD b/test/chunkserver/clone/BUILD index 2c171281b0..086e3b25f7 100644 --- a/test/chunkserver/clone/BUILD +++ b/test/chunkserver/clone/BUILD @@ -36,7 +36,7 @@ cc_test( "//test/chunkserver/datastore:datastore_mock", "//test/chunkserver:chunkserver_mock", "//test/chunkserver/clone:clone_mock", - "//test/client/mock:curve_client_mock", + "//test/client/mock:client_mock_lib", "//test/common:common_mock", ], ) diff --git a/test/client/BUILD b/test/client/BUILD index 98229cfc58..d9a1cb4fc2 100644 --- a/test/client/BUILD +++ b/test/client/BUILD @@ -75,6 +75,7 @@ cc_test( "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "//test/integration/cluster_common:integration_cluster_common", + "//test/client/mock:client_mock_lib", ], ) @@ -147,7 +148,7 @@ cc_test( deps = [ "//include/client:include_client", "//src/client:curve_client", - "//test/client/mock:curve_client_mock", + "//test/client/mock:client_mock_lib", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -175,6 +176,7 @@ cc_test( "//proto:topology_cc_proto", "//src/client:curve_client", "//src/common:curve_common", + "//src/client:cbd", "//test/client/fake:fake_lib", "//src/common/concurrent:curve_concurrent", "@com_google_googletest//:gtest", @@ -260,6 +262,7 @@ cc_test( "//src/client:curve_client", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", + "//test/client/mock:client_mock_lib" ], ) @@ -336,6 +339,6 @@ cc_test( "//src/common/concurrent:curve_concurrent", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", - "//test/client/mock:curve_client_mock" + "//test/client/mock:client_mock_lib" ], ) diff --git a/test/client/client_common_unittest.cpp b/test/client/client_common_unittest.cpp index e09aa36a84..d0faf57d82 100644 --- a/test/client/client_common_unittest.cpp +++ b/test/client/client_common_unittest.cpp @@ -24,8 +24,8 @@ #include "src/client/client_common.h" -using curve::client::EndPoint; -using curve::client::ChunkServerAddr; +namespace curve { +namespace client { TEST(ClientCommon, ChunkServerAddrTest) { // 默认构造函数创建的成员变量内容为空 @@ -70,3 +70,6 @@ TEST(ClientCommon, ChunkServerAddrTest) { str2endpoint("127.0.0.1:9000", &ep1); ASSERT_EQ(caddr2.addr_, ep1); } + +} // namespace client +} // namespace curve diff --git a/test/client/client_mdsclient_metacache_unittest.cpp b/test/client/client_mdsclient_metacache_unittest.cpp index f88c28f0d0..deadbdc54b 100644 --- a/test/client/client_mdsclient_metacache_unittest.cpp +++ b/test/client/client_mdsclient_metacache_unittest.cpp @@ -51,7 +51,8 @@ #include "src/common/net_common.h" #include "test/integration/cluster_common/cluster.h" #include "test/util/config_generator.h" -#include "test/client/mock_curvefs_service.h" +// #include "test/client/mock_curvefs_service.h" +#include "test/client/mock/mock_namespace_service.h" extern std::string mdsMetaServerAddr; extern uint32_t chunk_size; @@ -83,20 +84,20 @@ class MDSClientTest : public ::testing::Test { userinfo.owner = "test"; if (server.AddService(&topologyservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } if (server.AddService(&curvefsservice, - brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { - LOG(FATAL) << "Fail to add service"; + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + ASSERT_TRUE(false) << "Fail to add service"; } - curve::mds::topology::GetChunkServerInfoResponse* response - = new curve::mds::topology::GetChunkServerInfoResponse(); + curve::mds::topology::GetChunkServerInfoResponse* response = + new curve::mds::topology::GetChunkServerInfoResponse(); response->set_statuscode(0); - curve::mds::topology::ChunkServerInfo* serverinfo - = new curve::mds::topology::ChunkServerInfo(); + curve::mds::topology::ChunkServerInfo* serverinfo = + new curve::mds::topology::ChunkServerInfo(); serverinfo->set_chunkserverid(888); serverinfo->set_disktype("nvme"); serverinfo->set_hostip("10.182.26.2"); @@ -117,7 +118,6 @@ class MDSClientTest : public ::testing::Test { LOG(INFO) << "meta server addr = " << mdsMetaServerAddr.c_str(); ASSERT_EQ(server.Start(mdsMetaServerAddr.c_str(), &options), 0); - LOG(INFO) << configpath.c_str(); ASSERT_EQ(0, Init(configpath.c_str())) << "Fail to init config, path = " << configpath; } @@ -2223,6 +2223,67 @@ TEST_F(MDSClientTest, StatFileStatusTest) { } } +TEST_F(MDSClientTest, DeAllocateSegmentTest) { + FInfo fileInfo; + fileInfo.fullPathName = "/DeAllocateSegmentTest"; + + // rpc failed + { + brpc::Controller cntl; + cntl.SetFailed(-1, "rpc failed"); + + FakeReturn* fakeRet = new FakeReturn(&cntl, nullptr); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + uint64_t startMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_EQ(LIBCURVE_ERROR::FAILED, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + uint64_t endMs = curve::common::TimeUtility::GetTimeofDayMs(); + ASSERT_GE(endMs - startMs, metaopt.mdsMaxRetryMS); + } + + // rpc return ok + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kOK); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // rpc return segment not allocated + { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(curve::mds::StatusCode::kSegmentNotAllocated); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + + // other error code + { + std::vector errorCodes{ + curve::mds::StatusCode::kOwnerAuthFail, + curve::mds::StatusCode::kParaError, + curve::mds::StatusCode::kNotSupported, + curve::mds::StatusCode::kFileUnderSnapShot}; + + for (auto err : errorCodes) { + curve::mds::DeAllocateSegmentResponse response; + response.set_statuscode(err); + FakeReturn* fakeRet = new FakeReturn(nullptr, &response); + curvefsservice.SetDeAllocateSegmentFakeReturn(fakeRet); + + ASSERT_NE(LIBCURVE_ERROR::OK, + mdsclient_.DeAllocateSegment(&fileInfo, 0ull)); + } + } +} + using ::testing::_; using ::testing::DoAll; using ::testing::ElementsAre; @@ -2263,7 +2324,7 @@ class MDSClientRefreshSessionTest : public ::testing::Test { const uint32_t kTestPort = 1234; brpc::Server server_; - curve::client::MockCurveFsService curveFsService_; + curve::mds::MockNameService curveFsService_; }; TEST_F(MDSClientRefreshSessionTest, StartDummyServerTest) { diff --git a/test/client/client_unittest_main.cpp b/test/client/client_unittest_main.cpp index cf91b17a6b..f146abebc9 100644 --- a/test/client/client_unittest_main.cpp +++ b/test/client/client_unittest_main.cpp @@ -26,6 +26,7 @@ #include #include +#include #include "test/integration/cluster_common/cluster.h" diff --git a/test/client/client_userifo_unittest.cpp b/test/client/client_userinfo_unittest.cpp similarity index 100% rename from test/client/client_userifo_unittest.cpp rename to test/client/client_userinfo_unittest.cpp diff --git a/test/client/copyset_client_test.cpp b/test/client/copyset_client_test.cpp index 9b64cfa1ab..bd003de82f 100644 --- a/test/client/copyset_client_test.cpp +++ b/test/client/copyset_client_test.cpp @@ -30,14 +30,14 @@ #include // NOLINT #include "src/client/copyset_client.h" -#include "test/client/mock_meta_cache.h" +#include "test/client/mock/mock_meta_cache.h" #include "src/common/concurrent/count_down_event.h" -#include "test/client/mock_chunkservice.h" -#include "test/client/mock_request_context.h" +#include "test/client/mock/mock_chunkservice.h" +#include "test/client/mock/mock_request_context.h" #include "src/client/chunk_closure.h" #include "src/common/timeutility.h" #include "test/client/fake/fakeChunkserver.h" -#include "test/client/mock_request_scheduler.h" +#include "test/client/mock/mock_request_scheduler.h" #include "src/client/request_closure.h" #include "src/client/metacache.h" @@ -3549,11 +3549,12 @@ TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { const std::string& configPath = "./conf/client.conf"; cc.Init(configPath.c_str()); FileInstance fileinstance; - UserInfo_t userinfo; + UserInfo userinfo; userinfo.owner = "userinfo"; MDSClient mdsclient; - mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt); + ASSERT_EQ(LIBCURVE_ERROR::OK, + mdsclient.Initialize(cc.GetFileServiceOption().metaServerOpt)); ASSERT_TRUE(fileinstance.Initialize("/test", &mdsclient, userinfo, cc.GetFileServiceOption())); @@ -3573,8 +3574,8 @@ TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { << "Fail to start server add 127.0.0.1:9102"; // fill metacache - curve::client::MetaCache* mc - = fileinstance.GetIOManager4File()->GetMetaCache(); + curve::client::MetaCache* mc = + fileinstance.GetIOManager4File()->GetMetaCache(); curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); mc->UpdateChunkInfoByIndex(0, chunkinfo); curve::client::CopysetInfo cpinfo; diff --git a/test/client/discard_task_test.cpp b/test/client/discard_task_test.cpp new file mode 100644 index 0000000000..9ac3a08d28 --- /dev/null +++ b/test/client/discard_task_test.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sat Dec 19 22:49:56 CST 2020 + */ + +#include "src/client/discard_task.h" + +#include +#include + +#include + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::Return; + +class DiscardTaskTest : public ::testing::Test { + public: + void SetUp() override { + metric.reset(new DiscardMetric("DiscardTaskTest")); + discardTaskManager_.reset(new DiscardTaskManager(metric.get())); + + mockMetaCache_.reset(new MockMetaCache()); + mockMDSClient_.reset(new MockMDSClient()); + + fileInfo_.fullPathName = "/TestDiscardTask"; + fileInfo_.chunksize = 16ull * 1024 * 1024; + fileInfo_.segmentsize = 1ull * 1024 * 1024 * 1024; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + void TearDown() override { + discardTaskManager_->Stop(); + } + + protected: + FInfo fileInfo_; + std::unique_ptr discardTaskManager_; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; + std::unique_ptr metric; +}; + +TEST_F(DiscardTaskTest, TestDiscardBitmapCleared) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(discardTaskManager_.get(), segmentIndex, + mockMetaCache_.get(), mockMDSClient_.get(), metric.get()); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)).Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)).Times(0); + + ASSERT_NO_FATAL_FAILURE(task.Run()); +} + +TEST_F(DiscardTaskTest, TestAllDiscard) { + SegmentIndex segmentIndex = 100; + uint64_t offset = segmentIndex * 1024ull * 1024 * 1024; + DiscardTask task(discardTaskManager_.get(), segmentIndex, + mockMetaCache_.get(), mockMDSClient_.get(), metric.get()); + + // mdsclient return OK + { + // set all bit + FileSegment* segment = mockMetaCache_->GetFileSegment(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(1); + + ASSERT_NO_FATAL_FAILURE(task.Run()); + ASSERT_FALSE(segment->IsAllBitSet()); + } + + // mdsclient return under snapshot + { + // set all bit + FileSegment* segment = mockMetaCache_->GetFileSegment(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::UNDER_SNAPSHOT)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_NO_FATAL_FAILURE(task.Run()); + ASSERT_TRUE(segment->IsAllBitSet()); + } + + // mdsclient return failed + { + // set all bit + FileSegment* segment = mockMetaCache_->GetFileSegment(segmentIndex); + segment->GetBitmap().Set(); + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offset)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndex)) + .Times(0); + + ASSERT_NO_FATAL_FAILURE(task.Run()); + ASSERT_TRUE(segment->IsAllBitSet()); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/fake/fakeMDS.cpp b/test/client/fake/fakeMDS.cpp index 6dcb166a72..8ec2ada4d9 100644 --- a/test/client/fake/fakeMDS.cpp +++ b/test/client/fake/fakeMDS.cpp @@ -150,6 +150,15 @@ bool FakeMDS::StartService() { FakeReturn* fakeGetFileInforet = new FakeReturn(nullptr, static_cast(getfileinforesponse)); // NOLINT fakecurvefsservice_.SetGetFileInfoFakeReturn(fakeGetFileInforet); + // set DeAllocateSegment fake return + auto* deAllocateSegmentResponse = + new curve::mds::DeAllocateSegmentResponse(); + deAllocateSegmentResponse->set_statuscode(curve::mds::StatusCode::kOK); + auto* deAllocateSegmentFakeRet = + new FakeReturn(nullptr, deAllocateSegmentResponse); + fakecurvefsservice_.SetDeAllocateSegmentFakeReturn( + deAllocateSegmentFakeRet); + /** * set GetOrAllocateSegment fake return */ diff --git a/test/client/fake/fakeMDS.h b/test/client/fake/fakeMDS.h index 2f7aa8d8a6..33b80e79e5 100644 --- a/test/client/fake/fakeMDS.h +++ b/test/client/fake/fakeMDS.h @@ -201,6 +201,22 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { response->CopyFrom(*resp); } + void DeAllocateSegment(google::protobuf::RpcController* cntl_base, + const curve::mds::DeAllocateSegmentRequest* request, + curve::mds::DeAllocateSegmentResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + if (fakeDeAllocateSegment_->controller_ != nullptr && + fakeDeAllocateSegment_->controller_->Failed()) { + cntl_base->SetFailed("failed"); + return; + } + + auto fakeResponse = + static_cast(fakeDeAllocateSegment_->response_); + response->CopyFrom(*fakeResponse); + } + void OpenFile(::google::protobuf::RpcController* controller, const ::curve::mds::OpenFileRequest* request, ::curve::mds::OpenFileResponse* response, @@ -610,6 +626,10 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { fakeGetOrAllocateSegmentretForClone_ = fakeret; } + void SetDeAllocateSegmentFakeReturn(FakeReturn* fakeret) { + fakeDeAllocateSegment_ = fakeret; + } + void SetOpenFile(FakeReturn* fakeret) { fakeopenfile_ = fakeret; } @@ -717,6 +737,7 @@ class FakeMDSCurveFSService : public curve::mds::CurveFSService { FakeReturn* fakeGetAllocatedSizeRet_; FakeReturn* fakeGetOrAllocateSegmentret_; FakeReturn* fakeGetOrAllocateSegmentretForClone_; + FakeReturn* fakeDeAllocateSegment_; FakeReturn* fakeopenfile_; FakeReturn* fakeclosefile_; FakeReturn* fakerenamefile_; diff --git a/test/client/file_instance_test.cpp b/test/client/file_instance_test.cpp index 5fa91c0686..eeaf2948e2 100644 --- a/test/client/file_instance_test.cpp +++ b/test/client/file_instance_test.cpp @@ -59,5 +59,24 @@ TEST(FileInstanceTest, CommonTest) { fi4.UnInitialize(); } +TEST(FileInstanceTest, OpenReadonlyAndDiscardTest) { + FileInstance instance; + FileServiceOption opt; + MDSClient mdsClient; + UserInfo userInfo{"hello", "world"}; + + ASSERT_TRUE( + instance.Initialize("/FileInstanceTest-OpenReadonlyAndDiscardTest", + &mdsClient, userInfo, opt, true)); + + ASSERT_EQ(-1, instance.Discard(0, 0)); + + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP::LIBCURVE_OP_DISCARD; + aioctx.offset = 0; + aioctx.length = 0; + ASSERT_EQ(-1, instance.AioDiscard(&aioctx)); +} + } // namespace client } // namespace curve diff --git a/test/client/file_segment_test.cpp b/test/client/file_segment_test.cpp new file mode 100644 index 0000000000..1e5554f786 --- /dev/null +++ b/test/client/file_segment_test.cpp @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Tue Dec 15 20:38:06 CST 2020 + * Author: wuhanqing + */ + +#include +#include + +#include "src/client/metacache_struct.h" + +namespace curve { +namespace client { + +TEST(FileSegmentTest, TestDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // discard entire segment + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetBitmap(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextClearBit(0)); + } + + // discard length smaller than discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetBitmap(0, discardGranularity - 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + + segment.SetBitmap(segmentSize - discardGranularity, 1); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // discard offset and length are both align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetBitmap(0, discardGranularity); + segment.SetBitmap(25 * discardGranularity, 4 * discardGranularity); + segment.SetBitmap(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // discard offset or length aren't align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + + segment.SetBitmap(discardGranularity, + discardGranularity + 1); // {1, 1} + segment.SetBitmap(5 * discardGranularity - 1, + discardGranularity + 1); // {5, 5} + segment.SetBitmap(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // not valid + segment.SetBitmap(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {21, 21} + segment.SetBitmap(30 * discardGranularity - 1, + discardGranularity + 2); // {30, 30} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : setRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 1}, {5, 5}, {21, 21}, {30, 30}}; + + ASSERT_EQ(setRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < setRanges.size(); ++i) { + ASSERT_EQ(setRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(setRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.SetBitmap(1, segmentSize - 2); + clearRanges.clear(); + setRanges.clear(); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, setRanges.size()); + ASSERT_EQ(1, setRanges.front().beginIndex); + ASSERT_EQ(endIndex - 1, setRanges.front().endIndex); + } + } +} + +TEST(FileSegmentTest, TestClearDiscard) { + const SegmentIndex segmentIndex = 0; + const uint32_t segmentSize = 1 * GiB; + + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (const auto& discardGranularity : discardGranularities) { + LOG(INFO) << "discardGranularity: " << discardGranularity; + + // clear entire segment + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearBitmap(0, segmentSize); + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } + + // clear length smaller than discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearBitmap(0, discardGranularity - 1); + ASSERT_FALSE(segment.GetBitmap().Test(0)); + + segment.ClearBitmap(segmentSize - discardGranularity, 1); + ASSERT_FALSE( + segment.GetBitmap().Test(segmentSize / discardGranularity - 1)); + } + + // clear offset and length are both align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearBitmap(0, discardGranularity); + segment.ClearBitmap(25 * discardGranularity, + 4 * discardGranularity); + segment.ClearBitmap(segmentSize - 2 * discardGranularity, + 2 * discardGranularity); + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {0, 0}, {25, 28}, {endIndex - 1, endIndex}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + } + + // clear offset or length aren't align to discard granularity + { + FileSegment segment(segmentIndex, segmentSize, discardGranularity); + segment.GetBitmap().Set(); + + segment.ClearBitmap(discardGranularity, + discardGranularity + 1); // {1, 2} + segment.ClearBitmap(5 * discardGranularity - 1, + discardGranularity + 1); // {4, 5} + segment.ClearBitmap(10 * discardGranularity + 1, + 2 * discardGranularity - 2); // {10, 11} + segment.ClearBitmap(20 * discardGranularity + 1, + 3 * discardGranularity - 2); // {20, 22} + segment.ClearBitmap(30 * discardGranularity - 1, + discardGranularity + 2); // {29, 31} + + std::vector clearRanges; + std::vector setRanges; + + uint32_t endIndex = segmentSize / discardGranularity - 1; + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + + for (auto& r : clearRanges) { + LOG(INFO) << "begin: " << r.beginIndex + << " end: " << r.endIndex; + } + + std::vector expectedRanges{ + {1, 2}, {4, 5}, {10, 11}, {20, 22}, {29, 31}}; + + ASSERT_EQ(clearRanges.size(), expectedRanges.size()); + for (size_t i = 0; i < clearRanges.size(); ++i) { + ASSERT_EQ(clearRanges[i].beginIndex, + expectedRanges[i].beginIndex); + ASSERT_EQ(clearRanges[i].endIndex, expectedRanges[i].endIndex); + } + + segment.GetBitmap().Set(); + clearRanges.clear(); + setRanges.clear(); + segment.ClearBitmap(1, segmentSize - 2); + + segment.GetBitmap().Divide(0, endIndex, &clearRanges, &setRanges); + ASSERT_EQ(1, clearRanges.size()); + ASSERT_EQ(0, clearRanges.front().beginIndex); + ASSERT_EQ(endIndex, clearRanges.front().endIndex); + } + } +} + +TEST(FileSegmentTest, TestMultiSetDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegment segment(50, 1 * GiB, discardGranularity); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.SetBitmap(offset, length); + offset += length; + } + + ASSERT_TRUE(segment.IsAllBitSet()); + } +} + +TEST(FileSegmentTest, TestMultiClearDiscard) { + const std::vector discardGranularities{ + 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, 128 * KiB, 256 * KiB, + 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, 8 * MiB, 16 * MiB}; + + for (auto discardGranularity : discardGranularities) { + FileSegment segment(50, 1 * GiB, discardGranularity); + segment.GetBitmap().Set(); + + uint64_t offset = 0ull; + uint64_t length = 32 * MiB; + while (offset < 1 * GiB) { + segment.ClearBitmap(offset, length); + offset += length; + } + + ASSERT_EQ(Bitmap::NO_POS, segment.GetBitmap().NextSetBit(0)); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/iotracker_splitor_unittest.cpp b/test/client/iotracker_splitor_unittest.cpp index 48124f2a2e..9acaeacdb4 100644 --- a/test/client/iotracker_splitor_unittest.cpp +++ b/test/client/iotracker_splitor_unittest.cpp @@ -945,11 +945,13 @@ TEST_F(IOTrackerSplitorTest, InvalidParam) { iotracker, mc, &reqlist, nullptr, offset, length, &mdsclient_, &fi)); + // ASSERT_EQ(-1, Splitor::CalcDiscardSegments(nullptr)); + delete iotracker; delete[] buf; } -TEST(SplitorTest, RequestSourceInfoTest) { +TEST_F(IOTrackerSplitorTest, RequestSourceInfoTest) { IOTracker ioTracker(nullptr, nullptr, nullptr); ioTracker.SetOpType(OpType::READ); diff --git a/test/client/iotracker_test.cpp b/test/client/iotracker_test.cpp new file mode 100644 index 0000000000..61d8318fbe --- /dev/null +++ b/test/client/iotracker_test.cpp @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Sun Dec 20 10:25:08 CST 2020 + */ + +#include +#include +#include + +#include // NOLINT +#include +#include // NOLINT + +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock/mock_meta_cache.h" + +namespace curve { +namespace client { + +using ::testing::AllOf; +using ::testing::Ge; +using ::testing::Le; +using ::testing::Matcher; +using ::testing::Return; + +class IOTrackerTest : public ::testing::Test { + public: + void SetUp() override { + opt_.taskDelayMs = 10; + IOTracker::InitDiscardOption(opt_); + + metric.reset(new DiscardMetric("IOTrackerTest")); + discardTaskManager_.reset(new DiscardTaskManager(metric.get())); + + mockMDSClient_.reset(new MockMDSClient()); + mockMetaCache_.reset(new MockMetaCache()); + + MetaCacheOption metaCacheOpt; + metaCacheOpt.discardGranularity = discardGranularity_; + mockMetaCache_->Init(metaCacheOpt, mockMDSClient_.get()); + + fileInfo_.fullPathName = "/IOTrackerTest"; + fileInfo_.length = 100 * GiB; + fileInfo_.segmentsize = segmentSize_; + fileInfo_.chunksize = 16 * MiB; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + } + + void TearDown() override { + discardTaskManager_->Stop(); + } + + protected: + DiscardOption opt_; + FInfo fileInfo_; + std::unique_ptr metric; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; + std::unique_ptr discardTaskManager_; + uint64_t segmentSize_ = 1 * GiB; + uint32_t discardGranularity_ = 4 * KiB; +}; + +TEST_F(IOTrackerTest, TestDiscardNotSatisfyOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 512 * MiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, _)).Times(0); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(_)).Times(0); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_, + discardTaskManager_.get()); + ASSERT_EQ(0, iotracker.Wait()); + + // check bitmap + Bitmap& bitmap = mockMetaCache_->GetFileSegment(50)->GetBitmap(); + std::vector clearRanges; + std::vector setRanges; + bitmap.Divide(0, bitmap.Size(), &clearRanges, &setRanges); + ASSERT_EQ(1, clearRanges.size()); + ASSERT_EQ(1, setRanges.size()); + + ASSERT_EQ(0, setRanges[0].beginIndex); + ASSERT_EQ(segmentSize_ / discardGranularity_ / 2 - 1, + setRanges[0].endIndex); // NOLINT + ASSERT_EQ(segmentSize_ / discardGranularity_ / 2, + clearRanges[0].beginIndex); // NOLINT + ASSERT_EQ(segmentSize_ / discardGranularity_ - 1, clearRanges[0].endIndex); +} + +TEST_F(IOTrackerTest, TestDiscardOneSegment) { + IOTracker iotracker(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset = 50 * GiB; + uint32_t length = 1 * GiB; + + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, 50 * GiB)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(50)).Times(1); + + iotracker.StartDiscard(offset, length, mockMDSClient_.get(), &fileInfo_, + discardTaskManager_.get()); + ASSERT_EQ(0, iotracker.Wait()); + + // check bitmap + Bitmap& bitmap = mockMetaCache_->GetFileSegment(50)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap.NextClearBit(0)); + + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.taskDelayMs)); + + // check bitmap after discard + bitmap = mockMetaCache_->GetFileSegment(50)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap.NextSetBit(0)); +} + +TEST_F(IOTrackerTest, TestDiscardMultiSegment) { + // discard three times + IOTracker iotracker1(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset1 = 50 * GiB; + uint32_t length1 = 512 * MiB; + + IOTracker iotracker2(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset2 = 52 * GiB + 512 * MiB; + uint32_t length2 = 512 * MiB; + + IOTracker iotracker3(nullptr, mockMetaCache_.get(), nullptr); + uint64_t offset3 = 50 * GiB + 512 * MiB; + uint32_t length3 = 2 * GiB; + + Matcher offsetRange = AllOf(Ge(50 * GiB), Le(52 * GiB)); + EXPECT_CALL(*mockMDSClient_, DeAllocateSegment(_, offsetRange)) + .Times(3) + .WillRepeatedly(Return(LIBCURVE_ERROR::OK)); + + Matcher segmentIndexRange = AllOf(Ge(50), Le(52)); + EXPECT_CALL(*mockMetaCache_, CleanChunksInSegment(segmentIndexRange)) + .Times(3); + + iotracker1.StartDiscard(offset1, length1, mockMDSClient_.get(), &fileInfo_, + discardTaskManager_.get()); + ASSERT_EQ(0, iotracker1.Wait()); + + iotracker2.StartDiscard(offset2, length2, mockMDSClient_.get(), &fileInfo_, + discardTaskManager_.get()); + ASSERT_EQ(0, iotracker2.Wait()); + + iotracker3.StartDiscard(offset3, length3, mockMDSClient_.get(), &fileInfo_, + discardTaskManager_.get()); + ASSERT_EQ(0, iotracker3.Wait()); + + // check bitmap + Bitmap& bitmap1 = mockMetaCache_->GetFileSegment(50)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap1.NextClearBit(0)); + Bitmap& bitmap2 = mockMetaCache_->GetFileSegment(51)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap2.NextClearBit(0)); + Bitmap& bitmap3 = mockMetaCache_->GetFileSegment(52)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap3.NextClearBit(0)); + + std::this_thread::sleep_for( + std::chrono::milliseconds(5 * opt_.taskDelayMs)); + + // check bitmap after discard + bitmap1 = mockMetaCache_->GetFileSegment(50)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap1.NextSetBit(0)); + bitmap2 = mockMetaCache_->GetFileSegment(51)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap2.NextSetBit(0)); + bitmap3 = mockMetaCache_->GetFileSegment(52)->GetBitmap(); + ASSERT_EQ(Bitmap::NO_POS, bitmap3.NextSetBit(0)); +} + +} // namespace client +} // namespace curve diff --git a/test/client/libcbd_libcurve_test.cpp b/test/client/libcbd_libcurve_test.cpp index 68c4446067..16e5f85054 100644 --- a/test/client/libcbd_libcurve_test.cpp +++ b/test/client/libcbd_libcurve_test.cpp @@ -49,6 +49,8 @@ using curve::client::EndPoint; #define filename "1_userinfo_test.img" +const uint64_t GiB = 1024ull * 1024 * 1024; + DECLARE_string(chunkserver_list); extern std::string configpath; @@ -56,6 +58,12 @@ void LibcbdLibcurveTestCallback(CurveAioContext* context) { context->op = LIBCURVE_OP_MAX; } +std::atomic discardComplete(false); +void AioDiscardCallback(CurveAioContext* context) { + ASSERT_EQ(context->ret, 0); + discardComplete.store(true, std::memory_order_release); +} + class TestLibcbdLibcurve : public ::testing::Test { public: void SetUp() { @@ -201,6 +209,26 @@ TEST_F(TestLibcbdLibcurve, ReadWriteTest) { ASSERT_EQ(ret, LIBCURVE_ERROR::OK); } +TEST_F(TestLibcbdLibcurve, DiscardTest) { + int fd; + CurveOptions opts; + memset(&opts, 0, sizeof(opts)); + + opts.conf = const_cast(configpath.c_str()); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_init(&opts)); + + fd = cbd_lib_open(filename); + ASSERT_GE(fd, 0); + + ASSERT_EQ(0, cbd_lib_pdiscard(fd, 0, 4096)); + ASSERT_EQ(0, cbd_lib_pdiscard(fd, 1 * GiB, 1 * GiB)); + + sleep(1); + + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_close(fd)); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_fini()); +} + TEST_F(TestLibcbdLibcurve, AioReadWriteTest) { int ret; int fd; @@ -260,6 +288,52 @@ TEST_F(TestLibcbdLibcurve, AioReadWriteTest) { ASSERT_EQ(ret, LIBCURVE_ERROR::OK); } +TEST_F(TestLibcbdLibcurve, TestAioDiscard) { + int fd; + CurveOptions opts; + memset(&opts, 0, sizeof(opts)); + opts.conf = const_cast(configpath.c_str()); + + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_init(&opts)); + + fd = cbd_lib_open(filename); + ASSERT_GE(fd, 0); + + { + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP_DISCARD; + aioctx.offset = 0; + aioctx.length = 4096; + aioctx.cb = AioDiscardCallback; + + discardComplete.store(false); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_aio_pdiscard(fd, &aioctx)); + + while (discardComplete.load(std::memory_order_consume) != true) { + usleep(100); + } + } + + { + CurveAioContext aioctx; + aioctx.op = LIBCURVE_OP_DISCARD; + aioctx.offset = 1 * GiB; + aioctx.length = 1 * GiB; + aioctx.cb = AioDiscardCallback; + + discardComplete.store(false); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_aio_pdiscard(fd, &aioctx)); + + while (discardComplete.load(std::memory_order_consume) != true) { + usleep(100); + } + } + + sleep(1); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_close(fd)); + ASSERT_EQ(LIBCURVE_ERROR::OK, cbd_lib_fini()); +} + TEST_F(TestLibcbdLibcurve, StatFileTest) { int64_t ret; CurveOptions opt; @@ -344,6 +418,7 @@ const std::vector clientConf { std::string("metacache.rpcRetryIntervalUS=500"), std::string("mds.rpcRetryIntervalUS=500"), std::string("schedule.threadpoolSize=2"), + std::string("discard.discardTaskDelayMs=10") }; int main(int argc, char ** argv) { diff --git a/test/client/metacache_test.cpp b/test/client/metacache_test.cpp new file mode 100644 index 0000000000..725ac25f51 --- /dev/null +++ b/test/client/metacache_test.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * Date: Wed Dec 23 16:21:20 CST 2020 + * Author: wuhanqing + */ + +#include "src/client/metacache.h" + +#include +#include + +#include +#include + +namespace curve { +namespace client { + +class MetaCacheTest : public ::testing::Test { + public: + void SetUp() override {} + + void TearDown() override {} + + protected: + void InsertMetaCache(uint64_t fileLength, uint64_t segmentSize, + uint64_t chunkSize) { + fileInfo_.fullPathName = "/MetaCacheTest"; + fileInfo_.length = fileLength; + fileInfo_.segmentsize = segmentSize; + fileInfo_.chunksize = chunkSize; + + metaCache_.UpdateFileInfo(fileInfo_); + + uint64_t chunks = fileLength / chunkSize; + for (uint64_t i = 0; i < chunks; ++i) { + ChunkIDInfo info(i, i, i); + metaCache_.UpdateChunkInfoByIndex(i, info); + } + } + + uint64_t CountAvaiableChunks(uint64_t fileLength, uint64_t segmentSize, + uint64_t chunksize) { + uint64_t count = 0; + uint64_t chunks = fileLength / chunksize; + ChunkIDInfo info; + + for (uint64_t i = 0; i < chunks; ++i) { + if (MetaCacheErrorType::OK == + metaCache_.GetChunkInfoByIndex(i, &info)) { + ++count; + } + } + + return count; + } + + protected: + FInfo fileInfo_; + MetaCache metaCache_; +}; + +TEST_F(MetaCacheTest, TestCleanChunksInSegment) { + std::vector> testValues{ + {20 * GiB, 1 * GiB, 16 * MiB}, + {20 * GiB, 512 * MiB, 16 * MiB}, + {20 * GiB, 16 * MiB, 16 * MiB}, + {20 * GiB, 16 * MiB, 4 * MiB}, + }; + + for (auto& values : testValues) { + uint64_t fileLength = std::get<0>(values); + uint64_t segmentSize = std::get<1>(values); + uint64_t chunkSize = std::get<2>(values); + + InsertMetaCache(fileLength, segmentSize, chunkSize); + + uint64_t totalChunks = fileLength / chunkSize; + uint64_t totalSegments = fileLength / segmentSize; + uint64_t chunksInSegment = segmentSize / chunkSize; + + ASSERT_EQ(totalChunks, + CountAvaiableChunks(fileLength, segmentSize, chunkSize)); + + SegmentIndex segmentIndex = 0; + uint64_t count = 0; + while (segmentIndex < fileLength / segmentSize) { + metaCache_.CleanChunksInSegment(segmentIndex++); + ++count; + ASSERT_EQ(totalChunks - count * chunksInSegment, + CountAvaiableChunks(fileLength, segmentSize, chunkSize)); + } + + ASSERT_EQ(0, CountAvaiableChunks(fileLength, segmentIndex, chunkSize)); + } +} + +} // namespace client +} // namespace curve diff --git a/test/client/mock/BUILD b/test/client/mock/BUILD index 4b28caaf91..f26e4ca0b9 100644 --- a/test/client/mock/BUILD +++ b/test/client/mock/BUILD @@ -34,15 +34,13 @@ COPTS = [ ] cc_library( - name = "curve_client_mock", - srcs = glob( - ["*.h"] - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - "//external:gflags", + name="client_mock_lib", + srcs=glob([ + "*.h" + ]), + deps=[ + "//src/client:curve_client", ], - copts = COPTS + copts=COPTS, + visibility = ["//visibility:public"], ) diff --git a/test/client/mock_chunkservice.h b/test/client/mock/mock_chunkservice.h similarity index 98% rename from test/client/mock_chunkservice.h rename to test/client/mock/mock_chunkservice.h index 14c14534ae..beafdd3380 100644 --- a/test/client/mock_chunkservice.h +++ b/test/client/mock/mock_chunkservice.h @@ -20,8 +20,8 @@ * Author: wudemiao */ -#ifndef TEST_CLIENT_MOCK_CHUNKSERVICE_H_ -#define TEST_CLIENT_MOCK_CHUNKSERVICE_H_ +#ifndef TEST_CLIENT_MOCK_MOCK_CHUNKSERVICE_H_ +#define TEST_CLIENT_MOCK_MOCK_CHUNKSERVICE_H_ #include #include @@ -192,4 +192,4 @@ class MockChunkServiceImpl : public ChunkService { } // namespace client } // namespace curve -#endif // TEST_CLIENT_MOCK_CHUNKSERVICE_H_ +#endif // TEST_CLIENT_MOCK_MOCK_CHUNKSERVICE_H_ diff --git a/test/client/mock_curvefs_service.h b/test/client/mock/mock_mdsclient.h similarity index 53% rename from test/client/mock_curvefs_service.h rename to test/client/mock/mock_mdsclient.h index a8c622e8e7..3c54cdcc5a 100644 --- a/test/client/mock_curvefs_service.h +++ b/test/client/mock/mock_mdsclient.h @@ -16,32 +16,25 @@ /** * Project: curve - * Date: Mon Apr 27 11:34:41 CST 2020 - * Author: wuhanqing + * Date: Sat Dec 19 23:01:09 CST 2020 */ -#ifndef TEST_CLIENT_MOCK_CURVEFS_SERVICE_H_ -#define TEST_CLIENT_MOCK_CURVEFS_SERVICE_H_ +#ifndef TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ +#define TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ #include -#include "proto/nameserver2.pb.h" + +#include "src/client/mds_client.h" namespace curve { namespace client { -class MockCurveFsService : public ::curve::mds::CurveFSService { +class MockMDSClient : public MDSClient { public: - MockCurveFsService() = default; - ~MockCurveFsService() = default; - - MOCK_METHOD4(RefreshSession, - void(::google::protobuf::RpcController* controller, - const curve::mds::ReFreshSessionRequest* request, - curve::mds::ReFreshSessionResponse* response, - ::google::protobuf::Closure* done)); + MOCK_METHOD2(DeAllocateSegment, LIBCURVE_ERROR(const FInfo*, uint64_t)); }; } // namespace client } // namespace curve -#endif // TEST_CLIENT_MOCK_CURVEFS_SERVICE_H_ +#endif // TEST_CLIENT_MOCK_MOCK_MDSCLIENT_H_ diff --git a/test/client/mock_meta_cache.h b/test/client/mock/mock_meta_cache.h similarity index 92% rename from test/client/mock_meta_cache.h rename to test/client/mock/mock_meta_cache.h index 430bdca640..ca5aaf562d 100644 --- a/test/client/mock_meta_cache.h +++ b/test/client/mock/mock_meta_cache.h @@ -20,8 +20,8 @@ * Author: wudemiao */ -#ifndef TEST_CLIENT_MOCK_META_CACHE_H_ -#define TEST_CLIENT_MOCK_META_CACHE_H_ +#ifndef TEST_CLIENT_MOCK_MOCK_META_CACHE_H_ +#define TEST_CLIENT_MOCK_MOCK_META_CACHE_H_ #include #include @@ -75,6 +75,8 @@ class MockMetaCache : public MetaCache { &FakeMetaCache::UpdateLeader)); } + MOCK_METHOD1(CleanChunksInSegment, void(SegmentIndex)); + private: FakeMetaCache fakeMetaCache_; }; @@ -82,4 +84,4 @@ class MockMetaCache : public MetaCache { } // namespace client } // namespace curve -#endif // TEST_CLIENT_MOCK_META_CACHE_H_ +#endif // TEST_CLIENT_MOCK_MOCK_META_CACHE_H_ diff --git a/test/client/mock/mock_namespace_service.h b/test/client/mock/mock_namespace_service.h index edceaeb202..f6d3d6a3ed 100644 --- a/test/client/mock/mock_namespace_service.h +++ b/test/client/mock/mock_namespace_service.h @@ -52,6 +52,12 @@ class MockNameService : public CurveFSService { const ChangeOwnerRequest* request, ChangeOwnerResponse* response, google::protobuf::Closure* done)); + + MOCK_METHOD4(RefreshSession, + void(::google::protobuf::RpcController* controller, + const curve::mds::ReFreshSessionRequest* request, + curve::mds::ReFreshSessionResponse* response, + ::google::protobuf::Closure* done)); }; } // namespace mds diff --git a/test/client/mock_request_context.h b/test/client/mock/mock_request_context.h similarity index 93% rename from test/client/mock_request_context.h rename to test/client/mock/mock_request_context.h index e9a82498cf..9248ccf32c 100644 --- a/test/client/mock_request_context.h +++ b/test/client/mock/mock_request_context.h @@ -20,8 +20,8 @@ * Author: wudemiao */ -#ifndef TEST_CLIENT_MOCK_REQUEST_CONTEXT_H_ -#define TEST_CLIENT_MOCK_REQUEST_CONTEXT_H_ +#ifndef TEST_CLIENT_MOCK_MOCK_REQUEST_CONTEXT_H_ +#define TEST_CLIENT_MOCK_MOCK_REQUEST_CONTEXT_H_ #include "src/client/client_common.h" #include "src/client/request_context.h" @@ -86,4 +86,4 @@ class FakeRequestClosure : public RequestClosure { } // namespace client } // namespace curve -#endif // TEST_CLIENT_MOCK_REQUEST_CONTEXT_H_ +#endif // TEST_CLIENT_MOCK_MOCK_REQUEST_CONTEXT_H_ diff --git a/test/client/mock_request_scheduler.h b/test/client/mock/mock_request_scheduler.h similarity index 71% rename from test/client/mock_request_scheduler.h rename to test/client/mock/mock_request_scheduler.h index 624c61c0d5..f496687a47 100644 --- a/test/client/mock_request_scheduler.h +++ b/test/client/mock/mock_request_scheduler.h @@ -5,8 +5,8 @@ * Copyright (c) 2020 NetEase */ -#ifndef TEST_CLIENT_MOCK_REQUEST_SCHEDULER_H_ -#define TEST_CLIENT_MOCK_REQUEST_SCHEDULER_H_ +#ifndef TEST_CLIENT_MOCK_MOCK_REQUEST_SCHEDULER_H_ +#define TEST_CLIENT_MOCK_MOCK_REQUEST_SCHEDULER_H_ #include "src/client/request_scheduler.h" @@ -23,4 +23,4 @@ class MockRequestScheduler : public RequestScheduler { } // namespace client } // namespace curve -#endif // TEST_CLIENT_MOCK_REQUEST_SCHEDULER_H_ +#endif // TEST_CLIENT_MOCK_MOCK_REQUEST_SCHEDULER_H_ diff --git a/test/client/request_scheduler_test.cpp b/test/client/request_scheduler_test.cpp index 65b8f6c0e2..6220daf4ac 100644 --- a/test/client/request_scheduler_test.cpp +++ b/test/client/request_scheduler_test.cpp @@ -28,9 +28,9 @@ #include "src/client/request_scheduler.h" #include "src/client/client_common.h" -#include "test/client/mock_meta_cache.h" -#include "test/client/mock_chunkservice.h" -#include "test/client/mock_request_context.h" +#include "test/client/mock/mock_meta_cache.h" +#include "test/client/mock/mock_chunkservice.h" +#include "test/client/mock/mock_request_context.h" #include "src/common/concurrent/count_down_event.h" namespace curve { diff --git a/test/client/request_sender_test.cpp b/test/client/request_sender_test.cpp index 814ced8d34..e16c31e426 100644 --- a/test/client/request_sender_test.cpp +++ b/test/client/request_sender_test.cpp @@ -26,7 +26,7 @@ #include "src/client/client_common.h" #include "src/client/request_sender.h" #include "src/common/concurrent/count_down_event.h" -#include "test/client/mock_chunkservice.h" +#include "test/client/mock/mock_chunkservice.h" namespace curve { namespace client { diff --git a/test/kvstorageclient/etcdclient_test.cpp b/test/kvstorageclient/etcdclient_test.cpp index 6876db6c3f..b61fd64bcf 100644 --- a/test/kvstorageclient/etcdclient_test.cpp +++ b/test/kvstorageclient/etcdclient_test.cpp @@ -184,12 +184,14 @@ TEST_F(TestEtcdClinetImp, test_EtcdClientInterface) { // 3. list file, 可以list到file0~file9 std::vector listRes; - int errCode = client_->List("01", "02", &listRes); + std::vector> listRes2; + int errCode = client_->List("01", "02", &listRes2); ASSERT_EQ(EtcdErrCode::EtcdOK, errCode); - ASSERT_EQ(keyMap.size(), listRes.size()); - for (int i = 0; i < listRes.size(); i++) { + ASSERT_EQ(keyMap.size(), listRes2.size()); + for (int i = 0; i < listRes2.size(); i++) { FileInfo finfo; - ASSERT_TRUE(NameSpaceStorageCodec::DecodeFileInfo(listRes[i], &finfo)); + ASSERT_TRUE( + NameSpaceStorageCodec::DecodeFileInfo(listRes2[i].second, &finfo)); ASSERT_EQ(fileName[i], finfo.filename()); } diff --git a/test/mds/chunkserverclient/test_copyset_client.cpp b/test/mds/chunkserverclient/test_copyset_client.cpp index 44e3a10b57..6cf57a80cc 100644 --- a/test/mds/chunkserverclient/test_copyset_client.cpp +++ b/test/mds/chunkserverclient/test_copyset_client.cpp @@ -34,7 +34,7 @@ #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_topology.h" #include "test/mds/mock/mock_chunkserver.h" -#include "test/mds/chunkserverclient/mock_chunkserverclient.h" +#include "test/mds/mock/mock_chunkserverclient.h" #include "src/mds/chunkserverclient/chunkserverclient_config.h" diff --git a/test/mds/chunkserverclient/mock_chunkserverclient.h b/test/mds/mock/mock_chunkserverclient.h similarity index 90% rename from test/mds/chunkserverclient/mock_chunkserverclient.h rename to test/mds/mock/mock_chunkserverclient.h index 85d0da8596..3d4a8582e9 100644 --- a/test/mds/chunkserverclient/mock_chunkserverclient.h +++ b/test/mds/mock/mock_chunkserverclient.h @@ -20,8 +20,8 @@ * Author: xuchaojie */ -#ifndef TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ -#define TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#ifndef TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ +#define TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ #include #include "src/mds/chunkserverclient/chunkserver_client.h" @@ -65,4 +65,4 @@ class MockChunkServerClient : public ChunkServerClient { } // namespace mds } // namespace curve -#endif // TEST_MDS_CHUNKSERVERCLIENT_MOCK_CHUNKSERVERCLIENT_H_ +#endif // TEST_MDS_MOCK_MOCK_CHUNKSERVERCLIENT_H_ diff --git a/test/mds/mock/mock_etcdclient.h b/test/mds/mock/mock_etcdclient.h index 2d2d1a28ea..a8da477b2c 100644 --- a/test/mds/mock/mock_etcdclient.h +++ b/test/mds/mock/mock_etcdclient.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/kvstorageclient/etcd_client.h" #include "src/mds/nameserver2/namespace_storage_cache.h" @@ -42,6 +43,8 @@ class MockEtcdClient : public EtcdClientImp { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp index a2bcc1bbad..11c70f8572 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -44,8 +45,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 1. list失败 - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); std::map out; ASSERT_EQ(-1, AllocStatisticHelper::GetExistSegmentAllocValues( @@ -55,8 +57,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { { // 2. list成功，解析失败 std::vector values{"hello"}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; @@ -67,8 +70,9 @@ TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { // 3. 获取已有的segment alloc value成功 std::vector values{ NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); std::map out; diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp index 935a79778a..c51e91587c 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp @@ -31,6 +31,7 @@ using ::testing::_; using ::testing::Return; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; @@ -70,8 +71,9 @@ TEST_F(AllocStatisticTest, test_Init) { LOG(INFO) << "test2......"; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(-1, allocStatistic_->Init()); int64_t alloc; @@ -84,8 +86,9 @@ TEST_F(AllocStatisticTest, test_Init) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce( DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); @@ -102,8 +105,9 @@ TEST_F(AllocStatisticTest, test_PeriodicPersist_CalculateSegmentAlloc) { NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, List( - SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, _)) + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(0, allocStatistic_->Init()); diff --git a/test/mds/nameserver2/clean_core_test.cpp b/test/mds/nameserver2/clean_core_test.cpp index a3831307ea..5288fd83d6 100644 --- a/test/mds/nameserver2/clean_core_test.cpp +++ b/test/mds/nameserver2/clean_core_test.cpp @@ -28,11 +28,15 @@ #include "test/mds/mock/mock_topology.h" #include "src/mds/chunkserverclient/copyset_client.h" #include "test/mds/mock/mock_alloc_statistic.h" +#include "test/mds/mock/mock_chunkserverclient.h" using ::testing::_; using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::DoAll; using curve::mds::topology::MockTopology; using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::chunkserverclient::MockChunkServerClient; namespace curve { namespace mds { @@ -40,24 +44,43 @@ namespace mds { const uint64_t DefaultSegmentSize = kGB * 1; const uint64_t kMiniFileLength = 10 * kGB; -TEST(CleanCore, testcleansnapshotfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); +class CleanCoreTest : public testing::Test { + public: + void SetUp() override { + storage_ = std::make_shared(); + topology_ = std::make_shared(); + channelPool_ = std::make_shared(); + client_ = + std::make_shared(topology_, option_, channelPool_); + allocStatistic_ = std::make_shared(); + cleanCore_ = + std::make_shared(storage_, client_, allocStatistic_); + + csClient_ = std::make_shared( + topology_, option_, channelPool_); + } + + void TearDown() override {} + protected: + std::shared_ptr storage_; + std::shared_ptr topology_; + ChunkServerClientOption option_; + std::shared_ptr channelPool_; + std::shared_ptr client_; + std::shared_ptr allocStatistic_; + std::shared_ptr cleanCore_; + std::shared_ptr csClient_; +}; + +TEST_F(CleanCoreTest, testcleansnapshotfile) { { // segment size = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -65,11 +88,11 @@ TEST(CleanCore, testcleansnapshotfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -77,7 +100,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -87,24 +110,24 @@ TEST(CleanCore, testcleansnapshotfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -112,7 +135,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kSnapshotFileDeleteError); } { @@ -121,12 +144,12 @@ TEST(CleanCore, testcleansnapshotfile) { uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; uint64_t expectParentID = 101; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, + EXPECT_CALL(*storage_, GetSegment(expectParentID, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -135,7 +158,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_segmentsize(DefaultSegmentSize); cleanFile.set_parentid(expectParentID); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -149,11 +172,11 @@ TEST(CleanCore, testcleansnapshotfile) { // get segment ok, DeleteSnapShotChunk OK uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::OK)); } - EXPECT_CALL(*storage, DeleteSnapshotFile(_, _)) + EXPECT_CALL(*storage_, DeleteSnapshotFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -161,7 +184,7 @@ TEST(CleanCore, testcleansnapshotfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanSnapShotFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanSnapShotFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -169,24 +192,14 @@ TEST(CleanCore, testcleansnapshotfile) { } } -TEST(CleanCore, testcleanfile) { - auto storage = std::make_shared(); - auto topology = std::make_shared(); - ChunkServerClientOption option; - auto channelPool = std::make_shared(); - auto client = std::make_shared(topology, - option, channelPool); - auto allocStatistic = std::make_shared(); - auto cleanCore = std::make_shared(storage, - client, allocStatistic); - +TEST_F(CleanCoreTest, testcleanfile) { { // segmentsize = 0 FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(0); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::KInternalError); } @@ -194,11 +207,11 @@ TEST(CleanCore, testcleanfile) { // delete ok (no, segment) uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .Times(1) .WillOnce(Return(StoreStatus::OK)); @@ -206,7 +219,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kOK); ASSERT_EQ(progress.GetStatus(), TaskStatus::SUCCESS); @@ -217,25 +230,25 @@ TEST(CleanCore, testcleanfile) { // all ok , but do DeleteFile namespace meta error uint32_t segmentNum = kMiniFileLength / DefaultSegmentSize; for (uint32_t i = 0; i < segmentNum; i++) { - EXPECT_CALL(*storage, GetSegment(_, i * DefaultSegmentSize, _)) + EXPECT_CALL(*storage_, GetSegment(_, i * DefaultSegmentSize, _)) .WillOnce(Return(StoreStatus::KeyNotExist)); } - EXPECT_CALL(*storage, DeleteFile(_, _)) + EXPECT_CALL(*storage_, DeleteFile(_, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } { // get segment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .Times(1) .WillOnce(Return(StoreStatus::InternalError)); @@ -243,7 +256,7 @@ TEST(CleanCore, testcleanfile) { cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } @@ -252,20 +265,120 @@ TEST(CleanCore, testcleanfile) { } { // get segment ok, DeleteSnapShotChunk ok, DeleteSegment error - EXPECT_CALL(*storage, GetSegment(_, 0, _)) + EXPECT_CALL(*storage_, GetSegment(_, 0, _)) .WillOnce(Return(StoreStatus::OK)); - EXPECT_CALL(*storage, DeleteSegment(_, _, _)) + EXPECT_CALL(*storage_, DeleteSegment(_, _, _)) .WillOnce(Return(StoreStatus::InternalError)); FileInfo cleanFile; cleanFile.set_length(kMiniFileLength); cleanFile.set_segmentsize(DefaultSegmentSize); TaskProgress progress; - ASSERT_EQ(cleanCore->CleanFile(cleanFile, &progress), + ASSERT_EQ(cleanCore_->CleanFile(cleanFile, &progress), StatusCode::kCommonFileDeleteError); ASSERT_EQ(progress.GetStatus(), TaskStatus::FAILED); } } + +TEST_F(CleanCoreTest, TestCleanDiscardSegment) { + const std::string fakeKey = "fakekey"; + const int kDefaultChunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("/test_file"); + fileInfo.set_id(1234); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_length(kMiniFileLength); + + PageFileSegment segment; + segment.set_logicalpoolid(1); + segment.set_segmentsize(DefaultSegmentSize); + segment.set_chunksize(kDefaultChunkSize); + segment.set_startoffset(0); + + for (int i = 0; i < DefaultSegmentSize / kDefaultChunkSize; ++i) { + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(i); + chunk->set_chunkid(i); + } + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + // CopysetClient DeleteChunk failed + { + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .WillOnce(Return(false)); + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _, _)) + .Times(0); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // NameServerStorage CleanDiscardSegment failed + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(0); + + TaskProgress progress; + ASSERT_EQ(StatusCode::KInternalError, + cleanCore_->CleanDiscardSegment(fakeKey, discardSegmentInfo, + &progress)); + ASSERT_EQ(TaskStatus::FAILED, progress.GetStatus()); + } + + // ok + { + client_->SetChunkServerClient(csClient_); + + CopySetInfo copyset; + + copyset.SetLeader(1); + + EXPECT_CALL(*topology_, GetCopySet(_, _)) + .Times(segment.chunks_size()) + .WillRepeatedly( + DoAll(SetArgPointee<1>(copyset), Return(true))); + EXPECT_CALL(*csClient_, DeleteChunk(_, _, _, _, _)) + .Times(segment.chunks_size()) + .WillRepeatedly(Return(kMdsSuccess)); + + EXPECT_CALL(*storage_, CleanDiscardSegment(_, _, _)) + .WillOnce(Return(StoreStatus::OK)); + EXPECT_CALL(*allocStatistic_, DeAllocSpace(_, _, _)) + .Times(1); + + TaskProgress progress; + ASSERT_EQ(StatusCode::kOK, cleanCore_->CleanDiscardSegment( + fakeKey, discardSegmentInfo, &progress)); + ASSERT_EQ(100, progress.GetProgress()); + ASSERT_EQ(TaskStatus::SUCCESS, progress.GetStatus()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/nameserver2/clean_discard_segment_task_test.cpp b/test/mds/nameserver2/clean_discard_segment_task_test.cpp new file mode 100644 index 0000000000..dd4eb0eafb --- /dev/null +++ b/test/mds/nameserver2/clean_discard_segment_task_test.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Dec 22 14:25:52 CST 2020 + */ + +#include +#include +#include + +#include "test/mds/mock/mock_alloc_statistic.h" +#include "test/mds/mock/mock_chunkserverclient.h" +#include "test/mds/mock/mock_topology.h" +#include "test/mds/nameserver2/mock/mock_clean_manager.h" +#include "test/mds/nameserver2/mock/mock_namespace_storage.h" + +namespace curve { +namespace mds { + +using ::curve::mds::chunkserverclient::ChunkServerClientOption; +using ::curve::mds::chunkserverclient::MockChunkServerClient; +using curve::mds::topology::MockTopology; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; + +class CleanDiscardSegmentTaskTest : public ::testing::Test { + public: + void SetUp() override { + storage_ = std::make_shared(); + topology_ = std::make_shared(); + channelPool_ = std::make_shared(); + client_ = + std::make_shared(topology_, option_, channelPool_); + allocStatistic_ = std::make_shared(); + cleanCore_ = + std::make_shared(storage_, client_, allocStatistic_); + + csClient_ = std::make_shared(topology_, option_, + channelPool_); + + cleanManager_ = std::make_shared(); + } + + void TearDown() override {} + + protected: + std::shared_ptr storage_; + std::shared_ptr topology_; + ChunkServerClientOption option_; + std::shared_ptr channelPool_; + std::shared_ptr client_; + std::shared_ptr allocStatistic_; + std::shared_ptr cleanCore_; + std::shared_ptr csClient_; + std::shared_ptr cleanManager_; +}; + +TEST_F(CleanDiscardSegmentTaskTest, CommonTest) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + DoAll(SetArgPointee<0>(discardSegments), + Return(StoreStatus::OK))); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .WillRepeatedly(Return(true)); + + ASSERT_TRUE(task.Start()); + ASSERT_FALSE(task.Start()); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); + ASSERT_FALSE(task.Stop()); +} + +TEST_F(CleanDiscardSegmentTaskTest, TestListDiscardSegmentFailed) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + Return(StoreStatus::InternalError)); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .Times(0); + + task.Start(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); +} + +TEST_F(CleanDiscardSegmentTaskTest, TestSubmitJobFailed) { + CleanDiscardSegmentTask task(cleanManager_, storage_, 50); + std::map discardSegments; + discardSegments.emplace("hello", DiscardSegmentInfo()); + + EXPECT_CALL(*storage_, ListDiscardSegment(_)) + .WillRepeatedly( + DoAll(SetArgPointee<0>(discardSegments), + Return(StoreStatus::OK))); + + EXPECT_CALL(*cleanManager_, SubmitCleanDiscardSegmentJob(_, _)) + .WillRepeatedly(Return(false)); + + task.Start(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_NO_THROW(task.Stop()); +} + +} // namespace mds +} // namespace curve diff --git a/test/mds/nameserver2/curvefs_test.cpp b/test/mds/nameserver2/curvefs_test.cpp index affd6904b2..5b4ce08cb3 100644 --- a/test/mds/nameserver2/curvefs_test.cpp +++ b/test/mds/nameserver2/curvefs_test.cpp @@ -2222,6 +2222,218 @@ TEST_F(CurveFSTest, testGetOrAllocateSegment) { } } +TEST_F(CurveFSTest, TestDeAllocateSegment) { + const std::string filename = "/TestDeAllocateSegment"; + const uint64_t offset = 1ull * 1024 * 1024 * 1024; + + // GetFileInfo failed + { + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file type not support + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_DIRECTORY); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kNotSupported, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment offset wrong + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(100 * kGB); + fileInfo.set_segmentsize(1 * kGB); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, 100 * kGB)); + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, 1 * kGB + 1)); + } + + // file is being cloned + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileBeingCloned); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kNotSupported, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment not exists + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce(Return(StoreStatus::KeyNotExist)); + + ASSERT_EQ(StatusCode::kSegmentNotAllocated, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // segment offset not equal to argument + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(0ull); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kParaError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // list snapshot failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // file under snapshot + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + snapshotFiles.push_back(FileInfo()); + snapshotFiles.push_back(FileInfo()); + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + + ASSERT_EQ(StatusCode::kFileUnderSnapShot, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment failed + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::InternalError)); + + ASSERT_EQ(StatusCode::kStorageError, + curvefs_->DeAllocateSegment(filename, offset)); + } + + // storage discard segment success + { + FileInfo fileInfo; + fileInfo.set_filetype(FileType::INODE_PAGEFILE); + fileInfo.set_length(kMiniFileLength); + fileInfo.set_segmentsize(DefaultSegmentSize); + fileInfo.set_filestatus(FileStatus::kFileCreated); + + PageFileSegment segment; + segment.set_startoffset(offset); + + std::vector snapshotFiles; + + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(fileInfo), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, GetSegment(_, _, _)) + .WillOnce( + DoAll(SetArgPointee<2>(segment), Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, ListSnapshotFile(_, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(snapshotFiles), + Return(StoreStatus::OK))); + EXPECT_CALL(*storage_, DiscardSegment(_, _)) + .WillOnce(Return(StoreStatus::OK)); + + ASSERT_EQ(StatusCode::kOK, + curvefs_->DeAllocateSegment(filename, offset)); + } +} + TEST_F(CurveFSTest, testCreateSnapshotFile) { { // test client time not expired diff --git a/test/mds/nameserver2/fakes.h b/test/mds/nameserver2/fakes.h index a1345e17e5..d41907fff2 100644 --- a/test/mds/nameserver2/fakes.h +++ b/test/mds/nameserver2/fakes.h @@ -411,6 +411,55 @@ class FakeNameServerStorage : public NameServerStorage { return StoreStatus::OK; } + StoreStatus DiscardSegment(const FileInfo& fileInfo, + const PageFileSegment& segment) override { + std::lock_guard guard(lock_); + std::string segmentKey = NameSpaceStorageCodec::EncodeSegmentStoreKey( + fileInfo.id(), segment.startoffset()); + std::string cleanSegmentKey = + NameSpaceStorageCodec::EncodeDiscardSegmentStoreKey( + fileInfo.id(), segment.startoffset()); + + std::string encodeSegment; + if (!NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)) { + return StoreStatus::InternalError; + } + + std::string encodeDiscardSegment; + DiscardSegmentInfo discardInfo; + discardInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardInfo.set_allocated_pagefilesegment(new PageFileSegment(segment)); + if (!NameSpaceStorageCodec::EncodeDiscardSegment( + discardInfo, &encodeDiscardSegment)) { + return StoreStatus::InternalError; + } + + if (memKvMap_.count(segmentKey) == 0) { + return StoreStatus::KeyNotExist; + } + + memKvMap_.erase(segmentKey); + memKvMap_.emplace(cleanSegmentKey, encodeDiscardSegment); + + return StoreStatus::OK; + } + + StoreStatus CleanDiscardSegment(uint64_t size, const std::string& key, + int64_t* revision) override { + std::lock_guard guard(lock_); + if (memKvMap_.count(key) == 0) { + return StoreStatus::KeyNotExist; + } + + memKvMap_.erase(key); + return StoreStatus::OK; + } + + StoreStatus ListDiscardSegment( + std::map* out) override { + return StoreStatus::OK; + } + private: std::mutex lock_; std::map memKvMap_; diff --git a/test/mds/nameserver2/mock/mock_clean_manager.h b/test/mds/nameserver2/mock/mock_clean_manager.h index 1680bd3d56..0439d6d9a8 100644 --- a/test/mds/nameserver2/mock/mock_clean_manager.h +++ b/test/mds/nameserver2/mock/mock_clean_manager.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "src/mds/nameserver2/clean_manager.h" #include "src/mds/nameserver2/async_delete_snapshot_entity.h" @@ -39,6 +40,8 @@ class MockCleanManager: public CleanManagerInterface { std::shared_ptr)); MOCK_METHOD1(GetTask, std::shared_ptr(TaskIDType id)); MOCK_METHOD1(SubmitDeleteCommonFileJob, bool(const FileInfo&)); + MOCK_METHOD2(SubmitCleanDiscardSegmentJob, + bool(const std::string&, const DiscardSegmentInfo&)); }; } // namespace mds diff --git a/test/mds/nameserver2/mock/mock_namespace_storage.h b/test/mds/nameserver2/mock/mock_namespace_storage.h index 2f3da1759a..2224232e79 100644 --- a/test/mds/nameserver2/mock/mock_namespace_storage.h +++ b/test/mds/nameserver2/mock/mock_namespace_storage.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "src/mds/nameserver2/namespace_storage.h" namespace curve { @@ -84,6 +85,13 @@ class MockNameServerStorage : public NameServerStorage { StoreStatus(std::vector *snapShotFiles)); MOCK_METHOD2(ListSegment, StoreStatus(InodeID, std::vector*)); + + MOCK_METHOD2(DiscardSegment, + StoreStatus(const FileInfo&, const PageFileSegment&)); + MOCK_METHOD3(CleanDiscardSegment, + StoreStatus(uint64_t, const std::string&, int64_t*)); + MOCK_METHOD1(ListDiscardSegment, + StoreStatus(std::map*)); }; } // namespace mds diff --git a/test/mds/nameserver2/nameserverMetrics_test.cpp b/test/mds/nameserver2/nameserver_metric_test.cpp similarity index 56% rename from test/mds/nameserver2/nameserverMetrics_test.cpp rename to test/mds/nameserver2/nameserver_metric_test.cpp index 434ab1d366..3955fcab7a 100644 --- a/test/mds/nameserver2/nameserverMetrics_test.cpp +++ b/test/mds/nameserver2/nameserver_metric_test.cpp @@ -21,7 +21,7 @@ */ #include -#include "src/mds/nameserver2/nameserverMetrics.h" +#include "src/mds/nameserver2/metric.h" namespace curve { namespace mds { @@ -52,5 +52,37 @@ TEST(TestNameserverCacheMetrics, testall) { cacheMetrics.OnCacheMiss(); ASSERT_EQ(1, cacheMetrics.cacheMiss.get_value()); } + +TEST(SegmentDiscardMetricTest, TestCommon) { + const uint64_t segmentSize = 128ull * 1024 * 1024; + + SegmentDiscardMetric metric; + + ASSERT_EQ(0, metric.pendingSegments_.get_value()); + ASSERT_EQ(0, metric.pendingSize_.get_value()); + ASSERT_EQ(0, metric.totalCleanedSegments_.get_value()); + ASSERT_EQ(0, metric.totalCleanedSize_.get_value()); + + metric.OnReceiveDiscardRequest(segmentSize); + metric.OnReceiveDiscardRequest(segmentSize); + + ASSERT_EQ(2, metric.pendingSegments_.get_value()); + ASSERT_EQ(2 * segmentSize, metric.pendingSize_.get_value()); + ASSERT_EQ(0, metric.totalCleanedSegments_.get_value()); + ASSERT_EQ(0, metric.totalCleanedSize_.get_value()); + + metric.OnDiscardFinish(segmentSize); + ASSERT_EQ(1, metric.pendingSegments_.get_value()); + ASSERT_EQ(1 * segmentSize, metric.pendingSize_.get_value()); + ASSERT_EQ(1, metric.totalCleanedSegments_.get_value()); + ASSERT_EQ(1 * segmentSize, metric.totalCleanedSize_.get_value()); + + metric.OnDiscardFinish(segmentSize); + ASSERT_EQ(0, metric.pendingSegments_.get_value()); + ASSERT_EQ(0, metric.pendingSize_.get_value()); + ASSERT_EQ(2, metric.totalCleanedSegments_.get_value()); + ASSERT_EQ(2 * segmentSize, metric.totalCleanedSize_.get_value()); +} + } // namespace mds } // namespace curve diff --git a/test/mds/nameserver2/namespace_service_test.cpp b/test/mds/nameserver2/namespace_service_test.cpp index 012b4fe031..6034777c4c 100644 --- a/test/mds/nameserver2/namespace_service_test.cpp +++ b/test/mds/nameserver2/namespace_service_test.cpp @@ -2666,7 +2666,124 @@ TEST_F(NameSpaceServiceTest, testRecoverFile) { server.Join(); } -} // namespace mds -} // namespace curve +TEST_F(NameSpaceServiceTest, TestDeAllocateSegment) { + brpc::Server server; + + NameSpaceService nameSpaceSerivce(new FileLockManager(13)); + + ASSERT_EQ(0, server.AddService(&nameSpaceSerivce, + brpc::SERVER_DOESNT_OWN_SERVICE)); + + brpc::ServerOptions opt; + opt.idle_timeout_sec = -1; + ASSERT_EQ(0, server.Start("127.0.0.1", {8900, 8999}, &opt)); + + // init channel + brpc::Channel channel; + ASSERT_EQ(channel.Init(server.listen_address(), nullptr), 0); + + CurveFSService_Stub stub(&channel); + brpc::Controller cntl; + uint64_t fileLength = 100 * kGB; + std::string filename = "/TestDeAllocateSegment"; + std::string owner = "curve"; + + // create file and allocate segment + { + CreateFileRequest createRequest; + CreateFileResponse createResponse; + createRequest.set_filename(filename); + createRequest.set_owner(owner); + createRequest.set_date(TimeUtility::GetTimeofDayUs()); + createRequest.set_filetype(INODE_PAGEFILE); + createRequest.set_filelength(fileLength); + + cntl.set_log_id(1); + stub.CreateFile(&cntl, &createRequest, &createResponse, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, createResponse.statuscode()); + + // allocate segment + cntl.Reset(); + GetOrAllocateSegmentRequest allocateRequest; + GetOrAllocateSegmentResponse allocateResponse; + + allocateRequest.set_filename(filename); + allocateRequest.set_offset(50ull * kGB); + allocateRequest.set_allocateifnotexist(true); + allocateRequest.set_owner(owner); + allocateRequest.set_date(TimeUtility::GetTimeofDayUs()); + + stub.GetOrAllocateSegment(&cntl, &allocateRequest, &allocateResponse, + nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, allocateResponse.statuscode()); + } + + // 1. filename not valid + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename + "/"); + request.set_offset(0); + request.set_owner("curve"); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kParaError, response.statuscode()); + } + + // 2. segment not allocated + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(0); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kSegmentNotAllocated, response.statuscode()); + } + + // 3. deallocate success + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(50ull * kGB); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kOK, response.statuscode()); + } + + // 4. retry rpc + { + cntl.Reset(); + DeAllocateSegmentRequest request; + DeAllocateSegmentResponse response; + + request.set_filename(filename); + request.set_offset(50ull * kGB); + request.set_owner(owner); + request.set_date(TimeUtility::GetTimeofDayUs()); + stub.DeAllocateSegment(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(StatusCode::kSegmentNotAllocated, response.statuscode()); + } +} + +} // namespace mds +} // namespace curve diff --git a/test/mds/nameserver2/namespace_storage_test.cpp b/test/mds/nameserver2/namespace_storage_test.cpp index 968a57d71c..12139e660b 100644 --- a/test/mds/nameserver2/namespace_storage_test.cpp +++ b/test/mds/nameserver2/namespace_storage_test.cpp @@ -34,6 +34,7 @@ using ::testing::Return; using ::testing::AtLeast; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -298,7 +299,7 @@ TEST_F(TestNameServerStorageImp, test_MoveFileToRecycle) { TEST_F(TestNameServerStorageImp, test_ListFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListFile(0, 0, &listRes)); @@ -309,7 +310,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { GetFileInfoForTest(&fileinfo); ASSERT_TRUE(NameSpaceStorageCodec::EncodeFileInfo(fileinfo, &encodeFileinfo)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeFileinfo}), Return(EtcdErrCode::EtcdOK))); @@ -322,7 +323,7 @@ TEST_F(TestNameServerStorageImp, test_ListFile) { TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { // 1. list err std::vector listRes; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ( StoreStatus::InternalError, storage_->ListSnapshotFile(1, 2, &listRes)); @@ -339,10 +340,11 @@ TEST_F(TestNameServerStorageImp, test_ListSnapshotFile) { std::string endStoreKey = NameSpaceStorageCodec::EncodeSnapShotFileStoreKey(2, ""); - EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, _)) - .WillOnce(DoAll( - SetArgPointee<2>(std::vector{encodeFileinfo}), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*client_, List(startStoreKey, endStoreKey, + Matcher*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(std::vector{encodeFileinfo}), + Return(EtcdErrCode::EtcdOK))); ASSERT_EQ(StoreStatus::OK, storage_->ListSnapshotFile(1, 2, &listRes)); ASSERT_EQ(1, listRes.size()); ASSERT_EQ(fileinfo.filename(), listRes[0].filename()); @@ -420,7 +422,7 @@ TEST_F(TestNameServerStorageImp, test_Snapshotfile) { TEST_F(TestNameServerStorageImp, test_ListSegment) { // 1. list err std::vector segments; - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdCanceled)); ASSERT_EQ(StoreStatus::InternalError, storage_->ListSegment(0, &segments)); @@ -430,7 +432,7 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { PageFileSegment segment; GetPageFileSegmentForTest(&key, &segment); ASSERT_TRUE(NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - EXPECT_CALL(*client_, List(_, _, _)) + EXPECT_CALL(*client_, List(_, _, Matcher*>(_))) .WillOnce(DoAll( SetArgPointee<2>(std::vector{encodeSegment}), Return(EtcdErrCode::EtcdOK))); @@ -439,5 +441,137 @@ TEST_F(TestNameServerStorageImp, test_ListSegment) { ASSERT_EQ(segment.DebugString(), segments[0].DebugString()); } +TEST_F(TestNameServerStorageImp, test_DiscardSegment) { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + // transaction failed + { + EXPECT_CALL(*client_, TxnN(_)) + .WillOnce(Return(EtcdErrCode::EtcdTxnUnkownOp)); + + ASSERT_EQ(StoreStatus::InternalError, + storage_->DiscardSegment(fileInfo, segment)); + } + + // ok + { + EXPECT_CALL(*client_, TxnN(_)).WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*cache_, Remove(_)).Times(1); + + ASSERT_EQ(StoreStatus::OK, storage_->DiscardSegment(fileInfo, segment)); + } +} + +TEST_F(TestNameServerStorageImp, test_CleanDisardSegment) { + // delete failed + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::InternalError, + storage_->CleanDiscardSegment(1, key, &revision)); + } + + // delete ok + { + EXPECT_CALL(*client_, DeleteRewithRevision(_, _)) + .WillOnce( + DoAll(SetArgPointee<1>(100), Return(EtcdErrCode::EtcdOK))); + + std::string key = "fakekey"; + int64_t revision; + ASSERT_EQ(StoreStatus::OK, + storage_->CleanDiscardSegment(1, key, &revision)); + ASSERT_EQ(100, revision); + } +} + +TEST_F(TestNameServerStorageImp, test_ListDiscardSegment) { + // list failed + { + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // decode failed + { + std::vector> kvs{ + {"hello", "world"}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::InternalError, + storage_->ListDiscardSegment(&out)); + } + + // ok + { + const uint32_t chunkSize = 16 * 1024 * 1024; + + FileInfo fileInfo; + fileInfo.set_filename("test_DiscardSegment"); + + PageFileSegment segment; + segment.set_logicalpoolid(0); + segment.set_segmentsize(chunkSize); + segment.set_chunksize(chunkSize); + segment.set_startoffset(0); + auto* chunk = segment.add_chunks(); + chunk->set_copysetid(1); + chunk->set_chunkid(2); + + DiscardSegmentInfo discardSegmentInfo; + discardSegmentInfo.set_allocated_fileinfo(new FileInfo(fileInfo)); + discardSegmentInfo.set_allocated_pagefilesegment( + new PageFileSegment(segment)); + + std::string encodeDiscardSegmentInfo; + ASSERT_TRUE(NameSpaceStorageCodec::EncodeDiscardSegment( + discardSegmentInfo, &encodeDiscardSegmentInfo)); + + std::vector> kvs{ + {"hello", encodeDiscardSegmentInfo}}; + + EXPECT_CALL( + *client_, + List(_, _, + Matcher>*>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(kvs), Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(StoreStatus::OK, storage_->ListDiscardSegment(&out)); + + ASSERT_EQ(discardSegmentInfo.DebugString(), out["hello"].DebugString()); + } +} + } // namespace mds } // namespace curve diff --git a/test/mds/topology/mock_topology.h b/test/mds/topology/mock_topology.h index 9aea13644c..8f629edbaa 100644 --- a/test/mds/topology/mock_topology.h +++ b/test/mds/topology/mock_topology.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "proto/topology.pb.h" #include "src/mds/topology/topology_service_manager.h" @@ -326,6 +327,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/mds/topology/test_topology_storage_etcd.cpp b/test/mds/topology/test_topology_storage_etcd.cpp index 8155a961e8..32bf8f6054 100644 --- a/test/mds/topology/test_topology_storage_etcd.cpp +++ b/test/mds/topology/test_topology_storage_etcd.cpp @@ -36,6 +36,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace mds { @@ -77,9 +78,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { ASSERT_TRUE(codec_->EncodeLogicalPoolData(data, &value)); std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -92,7 +93,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map logicalPoolMap; @@ -105,7 +107,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_success_ListEtcdEmpty) { } TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); std::unordered_map logicalPoolMap; @@ -120,9 +123,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_ListEtcdFail) { TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -147,9 +150,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadLogicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map logicalPoolMap; PoolIdType maxLogicalPoolId; @@ -166,9 +169,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -182,7 +185,8 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success) { } TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); std::unordered_map physicalPoolMap; @@ -198,9 +202,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_success_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -219,9 +223,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPhysicalPool_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map physicalPoolMap; PoolIdType maxPhysicalPoolId; @@ -239,9 +243,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -256,9 +260,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -273,9 +278,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadZone_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -294,9 +299,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadZone_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map zoneMap; ZoneIdType maxZoneId; @@ -315,9 +320,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -332,9 +337,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -349,9 +355,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -371,9 +377,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadServer_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map serverMap; ServerIdType maxServerId; @@ -399,9 +405,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -417,9 +423,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -434,9 +441,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -463,9 +470,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadChunkServer_IdDuplcated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::unordered_map chunkServerMap; ChunkServerIdType maxChunkServerId; @@ -486,9 +493,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { std::vector list; list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -505,9 +512,10 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { std::vector list; - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdKeyNotExist))); + Return(EtcdErrCode::EtcdKeyNotExist))); std::map copySetMap; std::map copySetIdMaxMap; @@ -522,9 +530,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_success_listEtcdEmpty) { TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_decodeError) { std::vector list; list.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; @@ -546,9 +554,9 @@ TEST_F(TestTopologyStorageEtcd, test_LoadCopyset_IdDuplicated) { std::vector list; list.push_back(value); list.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(list), - Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*kvStorageClient_, + List(_, _, Matcher*>(_))) + .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK))); std::map copySetMap; std::map copySetIdMaxMap; diff --git a/test/snapshotcloneserver/mock_snapshot_server.h b/test/snapshotcloneserver/mock_snapshot_server.h index fa2fbd2493..57854189fe 100644 --- a/test/snapshotcloneserver/mock_snapshot_server.h +++ b/test/snapshotcloneserver/mock_snapshot_server.h @@ -29,6 +29,7 @@ #include #include #include +#include #include "src/snapshotcloneserver/snapshot/snapshot_core.h" #include "src/snapshotcloneserver/clone/clone_core.h" @@ -421,6 +422,8 @@ class MockKVStorageClient : public KVStorageClient { MOCK_METHOD2(Get, int(const std::string&, std::string*)); MOCK_METHOD3(List, int(const std::string&, const std::string&, std::vector*)); + MOCK_METHOD3(List, int(const std::string&, const std::string&, + std::vector>*)); MOCK_METHOD1(Delete, int(const std::string&)); MOCK_METHOD1(TxnN, int(const std::vector&)); MOCK_METHOD3(CompareAndSwap, int(const std::string&, const std::string&, diff --git a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp index a399015555..746501a873 100644 --- a/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp +++ b/test/snapshotcloneserver/test_snapshotclone_meta_store_etcd.cpp @@ -47,6 +47,7 @@ using ::testing::AllOf; using ::testing::SetArgPointee; using ::testing::Invoke; using ::testing::DoAll; +using ::testing::Matcher; namespace curve { namespace snapshotcloneserver { @@ -569,7 +570,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector cloneOut; cloneOut.push_back(cloneValue); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(cloneOut), @@ -581,7 +582,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, TEST_F(TestSnapshotCloneMetaStoreEtcd, TestInitListSnapshotFail) { - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(Return(EtcdErrCode::EtcdUnknown)); int ret = metaStore_->Init(); @@ -599,7 +600,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back(value); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(Return(EtcdErrCode::EtcdUnknown)); @@ -613,7 +614,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out; out.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))); @@ -634,7 +635,7 @@ TEST_F(TestSnapshotCloneMetaStoreEtcd, std::vector out2; out2.push_back("xxx"); - EXPECT_CALL(*kvStorageClient_, List(_, _, _)) + EXPECT_CALL(*kvStorageClient_, List(_, _, Matcher*>(_))) // NOLINT .WillOnce(DoAll(SetArgPointee<2>(out), Return(EtcdErrCode::EtcdOK))) .WillOnce(DoAll(SetArgPointee<2>(out2), diff --git a/test/tools/BUILD b/test/tools/BUILD index 729c13ae93..180a8002e5 100644 --- a/test/tools/BUILD +++ b/test/tools/BUILD @@ -47,7 +47,7 @@ cc_binary( "//test/client/fake:fake_lib", "//test/fs:fs_mock", "//test/common:common_mock", - "//test/client/mock:curve_client_mock", + "//test/client/mock:client_mock_lib", "//test/tools/mock:curve_tool_mock", ], copts = COPTS