diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index 3e44aab314..18cd6f9095 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -58,10 +58,6 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20 # batch limit of get inode attr and xattr executorOpt.batchInodeAttrLimit=10000 -#### spaceserver -spaceServer.spaceAddr=127.0.0.1:19999 # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__ -spaceServer.rpcTimeoutMs=1000 - #### bdev # curve client's config file bdev.confPath=/etc/curve/client.conf @@ -177,6 +173,15 @@ volume.bitmapAllocator.smallAllocProportion=0.2 # number of block groups that allocated once volume.blockGroup.allocateOnce=4 +## spaceserver +# the space used by the blockgroup exceeds this percentage and can +# be returned to mds [0.8-1] +volume.space.useThreshold=0.95 + +# the background thread calculates the time interval for returning +# the blockgroup to mds +volume.space.releaseInterSec=300 + #### s3 # this is for test. if s3.fakeS3=true, all data will be discarded s3.fakeS3=false diff --git a/curvefs/conf/curvebs_client.conf b/curvefs/conf/curvebs_client.conf new file mode 100644 index 0000000000..e0eb4d70f2 --- /dev/null +++ b/curvefs/conf/curvebs_client.conf @@ -0,0 +1,191 @@ +# +################### mds一侧配置信息 ################## +# + +# mds的地址信息，对于mds集群，地址以逗号隔开 +mds.listen.addr=127.0.0.1:6666 + +# 初始化阶段向mds注册开关，默认为开 +mds.registerToMDS=true + +# 与mds通信的rpc超时时间 +mds.rpcTimeoutMS=500 + +# 与mds通信rpc最大的超时时间, 指数退避的超时间不能超过这个值 +mds.maxRPCTimeoutMS=2000 + +# 与mds通信重试总时间 +mds.maxRetryMS=8000 + +# 在当前mds上连续重试次数超过该限制就切换, 这个失败次数包含超时重试次数 +mds.maxFailedTimesBeforeChangeMDS=2 + +# 与MDS一侧保持一个lease时间内多少次续约 +mds.refreshTimesPerLease=4 + +# mds RPC接口每次重试之前需要先睡眠一段时间 +mds.rpcRetryIntervalUS=100000 + +# The normal retry times for trigger wait strategy +mds.normalRetryTimesBeforeTriggerWait=3 + +# Max retry time for IO-Path request +mds.maxRetryMsInIOPath=86400000 + +# Sleep interval for wait +mds.waitSleepMs=10000 + +# +################# metacache配置信息 ################ +# + +# 获取leader的rpc超时时间 +metacache.getLeaderTimeOutMS=500 + +# 获取leader的重试次数 +metacache.getLeaderRetry=5 + +# 获取leader接口每次重试之前需要先睡眠一段时间 +metacache.rpcRetryIntervalUS=100000 + +# +############### 调度层的配置信息 ############# +# + +# 调度层队列大小，每个文件对应一个队列 +# 调度队列的深度会影响client端整体吞吐，这个队列存放的是异步IO任务。。 +schedule.queueCapacity=1000000 + +# 队列的执行线程数量 +# 执行线程所要做的事情就是将IO取出，然后发到网络就返回取下一个网络任务。一个任务从 +# 队列取出到发送完rpc请求大概在(20us-100us)，20us是正常情况下不需要获取leader的时候 +# 如果在发送的时候需要获取leader，时间会在100us左右，一个线程的吞吐在10w-50w +# 性能已经满足需求 +schedule.threadpoolSize=2 + +# 为隔离qemu侧线程引入的任务队列，因为qemu一侧只有一个IO线程 +# 当qemu一侧调用aio接口的时候直接将调用push到任务队列就返回， +# 这样libcurve不占用qemu的线程，不阻塞其异步调用 +isolation.taskQueueCapacity=1000000 + +# 隔离qemu线程的任务队列线程池大小, 默认值为1个线程 +isolation.taskThreadPoolSize=1 + + +# +################ 与chunkserver通信相关配置 ############# +# +# 读写接口失败的OP之间重试睡眠 +chunkserver.opRetryIntervalUS=100000 + +# 失败的OP重试次数 +chunkserver.opMaxRetry=2500000 + +# 与chunkserver通信的rpc超时时间 +chunkserver.rpcTimeoutMS=1000 + +# 开启基于appliedindex的读，用于性能优化 +chunkserver.enableAppliedIndexRead=1 + +# 重试请求之间睡眠最长时间 +# 因为当网络拥塞的时候或者chunkserver出现过载的时候，需要增加睡眠时间 +# 这个时间最大为maxRetrySleepIntervalUs +chunkserver.maxRetrySleepIntervalUS=8000000 + +# 重试请求的超时rpc时间最大值，超时时间会遵循指数退避策略 +# 因为当网络拥塞的时候出现超时，需要增加RPC超时时间 +# 这个时间最大为maxTimeoutMS +chunkserver.maxRPCTimeoutMS=8000 + +# 同一个chunkserver连续超时上限次数 +# 如果超过这个值，就会进行健康检查，健康检查失败后，会标记为unstable +chunkserver.maxStableTimeoutTimes=10 +# chunkserver上rpc连续超时后，健康检查请求的超时间 +chunkserver.checkHealthTimeoutMs=100 +# 同一个server上unstable的chunkserver数量超过这个值之后 +# 所有的chunkserver都会标记为unstable +chunkserver.serverStableThreshold=3 + +# 当底层chunkserver压力大时，可能也会触发unstable +# 由于copyset leader may change，会导致请求超时时间设置为默认值，从而导致IO hang +# 真正宕机的情况下，请求重试一定次数后会处理完成 +# 如果一直重试，则不是宕机情况，这时候超时时间还是要进入指数退避逻辑 +# 当一个请求重试次数超过这个值时，其超时时间一定进入指数退避 +chunkserver.minRetryTimesForceTimeoutBackoff=5 + +# 当一个rpc重试超过次数maxRetryTimesBeforeConsiderSuspend的时候 +# 记为悬挂IO，metric会报警 +chunkserver.maxRetryTimesBeforeConsiderSuspend=20 + +# +################# 文件级别配置项 ############# +# +# libcurve底层rpc调度允许最大的未返回rpc数量，每个文件的inflight RPC独立 +global.fileMaxInFlightRPCNum=128 + +# 文件IO下发到底层chunkserver最大的分片KB +global.fileIOSplitMaxSizeKB=64 + +# +################# log相关配置 ############### +# +# enable logging or not +global.logging.enable=True +# +# log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3 +global.logLevel=0 +# 设置log的路径 +global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__ +# 单元测试情况下 +# logpath=./runlog/ + +# +################# 读源卷相关配置 ############### +# +# 读取源卷时打开的fd超时关闭时间300s +closefd.timeout=300 +# 读取源卷时打开的fd后台线程每600s扫描一遍fdMap，关闭超时fd +closefd.timeInterval=600 + +# +############### metric 配置信息 ############# +# +global.metricDummyServerStartPort=9000 + +# 是否关闭健康检查: true/关闭 false/不关闭 +global.turnOffHealthCheck=true + +# +### throttle config +# +throttle.enable=false + +##### discard configurations ##### +# enable/disable discard +discard.enable=true +# discard granularity +discard.granularity=4096 +# discard cleanup task delay times in millisecond +discard.taskDelayMs=60000 + +##### alignment ##### +# default alignment +global.alignment.commonVolume=512 +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume=4096 + +##### chunkserver client option ##### +# chunkserver client rpc timeout time +csClientOpt.rpcTimeoutMs=500 +# chunkserver client rpc max try +csClientOpt.rpcMaxTry=86400000 +# chunkserver client rpc retry interval +csClientOpt.rpcIntervalUs=100000 +# chunkserver client rpc max timeout time +csClientOpt.rpcMaxTimeoutMs=8000 + +##### chunkserver broadcaster option ##### +# broad cast max machine num +csBroadCasterOpt.broadCastMaxNum=200 + diff --git a/curvefs/conf/mds.conf b/curvefs/conf/mds.conf index 8fbf97829e..294e35cfe7 100644 --- a/curvefs/conf/mds.conf +++ b/curvefs/conf/mds.conf @@ -162,3 +162,6 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2 bs.mds.normalRetryTimesBeforeTriggerWait=3 # sleep interval in ms for wait bs.mds.waitSleepMs=1000 + +#### Options for volume space deallcatable setting +mds.space.calIntervalSec=60 diff --git a/curvefs/conf/metaserver.conf b/curvefs/conf/metaserver.conf index 9d56069f56..71fda40760 100644 --- a/curvefs/conf/metaserver.conf +++ b/curvefs/conf/metaserver.conf @@ -325,3 +325,9 @@ metaCacheOpt.metacacheGetLeaderRetry=3 metaCacheOpt.metacacheRPCRetryIntervalUS=100000 # RPC timeout of get leader metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000 + +#### volume deallocate +volume.deallocate.enable=true +volume.deallocate.workerNum=5 +volume.deallocate.batchClean=10 +volume.sdk.confPath=conf/curvebs_client.conf # __CURVEADM_TEMPLATE__ /curvefs/metaserver/conf/curvebs_client.conf __CURVEADM_TEMPLATE__ diff --git a/curvefs/proto/BUILD b/curvefs/proto/BUILD index 2a4727a611..db2a8e4dab 100644 --- a/curvefs/proto/BUILD +++ b/curvefs/proto/BUILD @@ -113,6 +113,7 @@ proto_library( name = "curvefs_heartbeat_proto", srcs = ["heartbeat.proto"], deps = [":curvefs_common_proto", + ":metaserver_proto", "//proto:heartbeat_proto"], ) diff --git a/curvefs/proto/common.proto b/curvefs/proto/common.proto index 4c72525de4..d2772dbf46 100644 --- a/curvefs/proto/common.proto +++ b/curvefs/proto/common.proto @@ -31,6 +31,11 @@ enum BitmapLocation { AtEnd = 2; } +message BlockGroupID { + required uint64 fsId = 1; + required uint64 offset = 2; +} + // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster message Volume { optional uint64 volumeSize = 1; diff --git a/curvefs/proto/heartbeat.proto b/curvefs/proto/heartbeat.proto index 4a6954f538..6327297bfb 100644 --- a/curvefs/proto/heartbeat.proto +++ b/curvefs/proto/heartbeat.proto @@ -16,6 +16,7 @@ syntax = "proto2"; import "curvefs/proto/common.proto"; +import "curvefs/proto/metaserver.proto"; import "proto/heartbeat.proto"; package curvefs.mds.heartbeat; option cc_generic_services = true; @@ -27,6 +28,17 @@ option go_package = "curvefs/proto/heartbeat"; // required uint64 diskUsedByte = 2; // the disk this copyset used // }; +enum BlockGroupDeallcateStatusCode { + BGDP_PROCESSING = 0; + BGDP_DONE = 1; +} + +message BlockGroupStatInfo { + required uint32 fsId = 1; + repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2; + map blockGroupDeallocateStatus = 3; +} + message CopySetInfo { required uint32 poolId = 1; required uint32 copysetId = 2; @@ -81,6 +93,7 @@ message MetaServerHeartbeatRequest { required uint32 leaderCount = 7; required uint32 copysetCount = 8; required MetaServerSpaceStatus spaceStatus = 9; + repeated BlockGroupStatInfo blockGroupStatInfos = 10; }; message CopySetConf { @@ -107,11 +120,14 @@ enum HeartbeatStatusCode { hbMetaServerIpPortNotMatch = 2; hbMetaServerTokenNotMatch = 3; hbAnalyseCopysetError = 4; + hbMetaServerFSUnkown = 5; } message MetaServerHeartbeatResponse { required HeartbeatStatusCode statusCode = 1; repeated CopySetConf needUpdateCopysets = 2; + // key is fsid, value is blockgroup offset + map issuedBlockGroups = 3; }; service HeartbeatService { diff --git a/curvefs/proto/mds.proto b/curvefs/proto/mds.proto index 36f243bbae..eebec8ee34 100644 --- a/curvefs/proto/mds.proto +++ b/curvefs/proto/mds.proto @@ -60,6 +60,7 @@ enum FSStatusCode { INSERT_MANAGE_INODE_FAIL = 35; DELETE_DENTRY_FAIL = 36; UPDATE_FS_FAIL = 37; + SPACE_RELEASE_FAIL = 38; } // fs interface diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index 6048e59482..4340d638cc 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -167,6 +167,46 @@ enum FsFileType { TYPE_S3 = 4; }; +message DeallocatableBlockGroup { + required uint64 blockGroupOffset = 1; + optional uint64 deallocatableSize = 2; + repeated uint64 inodeIdlist = 3; + repeated uint64 inodeIdUnderDeallocate = 4; + + oneof type { + IncreaseDeallocatableBlockGroup increase = 5; + DecreaseDeallocatableBlockGroup decrease = 6; + MarkDeallocatableBlockGroup mark = 7; + } +} + +message IncreaseDeallocatableBlockGroup { + required uint64 increaseDeallocatableSize = 1; + repeated uint64 inodeIdlistAdd = 2; +} + +message DecreaseDeallocatableBlockGroup { + required uint64 decreaseDeallocatableSize = 1; + repeated uint64 inodedDeallocated = 2; +} + +message MarkDeallocatableBlockGroup { + repeated uint64 inodeIdUnderDeallocate = 2; +} + +message UpdateDeallocatableBlockGroupRequest { + required uint32 poolId = 1; + required uint32 copysetId = 2; + required uint32 partitionId = 3; + required uint64 fsId = 4; + repeated DeallocatableBlockGroup update = 5; +} + +message UpdateDeallocatableBlockGroupResponse { + required MetaStatusCode statusCode = 1; + optional uint64 appliedIndex = 2; +} + message VolumeExtent { required uint64 fsOffset = 1; required uint64 volumeOffset = 2; @@ -179,7 +219,7 @@ message VolumeExtentSlice { repeated VolumeExtent extents = 2; } -message VolumeExtentList { +message VolumeExtentSliceList { repeated VolumeExtentSlice slices = 1; } @@ -317,7 +357,7 @@ message UpdateInodeRequest { map xattr = 20; repeated uint64 parent = 21; map s3ChunkInfoAdd = 22; - optional VolumeExtentList volumeExtents = 23; + optional VolumeExtentSliceList volumeExtents = 23; } message UpdateInodeResponse { @@ -464,7 +504,7 @@ message GetVolumeExtentRequest { message GetVolumeExtentResponse { required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; - optional VolumeExtentList slices = 3; + optional VolumeExtentSliceList slices = 3; } message UpdateVolumeExtentRequest { @@ -473,7 +513,7 @@ message UpdateVolumeExtentRequest { required uint32 partitionId = 3; required uint32 fsId = 4; required uint64 inodeId = 5; - required VolumeExtentList extents = 6; + required VolumeExtentSliceList extents = 6; } message UpdateVolumeExtentResponse { @@ -508,4 +548,7 @@ service MetaServerService { // volume extent interface rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse); rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse); + + // block group with deallocatable inode list interface + rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse); } diff --git a/curvefs/proto/space.proto b/curvefs/proto/space.proto index 345d50ab05..e60574a021 100644 --- a/curvefs/proto/space.proto +++ b/curvefs/proto/space.proto @@ -34,6 +34,7 @@ enum SpaceErrCode { SpaceErrParam = 10; SpaceErrNotSupport = 11; SpaceErrExtendVolumeError = 12; + SpaceErrRelease = 13; } message BlockGroup { @@ -48,6 +49,10 @@ message BlockGroup { required common.BitmapLocation bitmaplocation = 4; // owner, who owns this block group optional string owner = 5; + // deallocating, metaserver who is deallocate this block group + repeated uint32 deallocating = 6; + // deallocated, metaserver who finish deallocate this block group + repeated uint32 deallocated = 7; } message AllocateBlockGroupRequest { diff --git a/curvefs/src/client/async_request_closure.cpp b/curvefs/src/client/async_request_closure.cpp index aac58c932b..6f63899886 100644 --- a/curvefs/src/client/async_request_closure.cpp +++ b/curvefs/src/client/async_request_closure.cpp @@ -69,13 +69,20 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() { void UpdateVolumeExtentClosure::Run() { auto st = GetStatusCode(); if (!IsOK(st)) { - LOG(ERROR) << "UpdateVolumeExtent failed, error: " - << MetaStatusCode_Name(st) - << ", inodeid: " << inode_->GetInodeId(); - inode_->MarkInodeError(); + if (inode_ != nullptr) { + inode_->MarkInodeError(); + LOG(ERROR) << "UpdateVolumeExtent failed, error: " + << MetaStatusCode_Name(st) + << ", inodeid: " << inode_->GetInodeId(); + } else { + LOG(ERROR) << "UpdateVolumeExtent failed, error: " + << MetaStatusCode_Name(st); + } } - inode_->syncingVolumeExtentsMtx_.unlock(); + if (inode_ != nullptr) { + inode_->syncingVolumeExtentsMtx_.unlock(); + } if (sync_) { std::lock_guard lk(mtx_); diff --git a/curvefs/src/client/async_request_closure.h b/curvefs/src/client/async_request_closure.h index 2d83bfcd0e..36350097d5 100644 --- a/curvefs/src/client/async_request_closure.h +++ b/curvefs/src/client/async_request_closure.h @@ -63,7 +63,7 @@ class AsyncRequestClosureBase : public rpcclient::MetaServerClientDone { // // xxx // auto statusCode = closure.Wait(); // -// for synchronous update +// for asynchronous update // // auto* closure = new UpdateVolumeExtentClosure( // shared_ptr of a inode wrapper, /* sync */ false); diff --git a/curvefs/src/client/common/common.h b/curvefs/src/client/common/common.h index a1a1d704c2..b8b6cf9d7d 100644 --- a/curvefs/src/client/common/common.h +++ b/curvefs/src/client/common/common.h @@ -60,6 +60,7 @@ enum class MetaServerOpType { GetVolumeExtent, UpdateVolumeExtent, CreateManageInode, + UpdateDeallocatableBlockGroup, }; std::ostream &operator<<(std::ostream &os, MetaServerOpType optype); diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index 62dabcb726..bef20b5ab1 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -245,6 +245,10 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) { conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize); conf->GetValueFatalIfFail("volume.allocator.type", &volumeOpt->allocatorOption.type); + conf->GetValueFatalIfFail("volume.space.useThreshold", + &volumeOpt->threshold); + conf->GetValueFatalIfFail("volume.space.releaseInterSec", + &volumeOpt->releaseInterSec); conf->GetValueFatalIfFail( "volume.blockGroup.allocateOnce", diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index ff0b399e93..e53137ae45 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -73,11 +73,6 @@ struct LeaseOpt { uint32_t leaseTimeUs = 20000000; }; -struct SpaceAllocServerOption { - std::string spaceaddr; - uint64_t rpcTimeoutMs; -}; - struct KVClientManagerOpt { int setThreadPooln = 4; int getThreadPooln = 4; @@ -164,6 +159,9 @@ struct VolumeOption { uint64_t volBlockSize; uint64_t fsBlockSize; VolumeAllocatorOption allocatorOption; + + double threshold{1.0}; + uint64_t releaseInterSec{300}; }; struct ExtentManagerOption { @@ -232,7 +230,6 @@ struct FuseClientOption { MetaCacheOpt metaCacheOpt; ExcutorOpt excutorOpt; ExcutorOpt excutorInternalOpt; - SpaceAllocServerOption spaceOpt; BlockDeviceClientOptions bdevOpt; S3Option s3Opt; ExtentManagerOption extentManagerOpt; diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index 2baa7834da..b67d0faa66 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -117,10 +117,12 @@ using common::FLAGS_fuseClientAvgReadBytes; using common::FLAGS_fuseClientBurstReadBytes; using common::FLAGS_fuseClientBurstReadBytesSecs; -static void on_throttle_timer(void *arg) { - FuseClient *fuseClient = reinterpret_cast(arg); - fuseClient->InitQosParam(); -} + +// FIXME: multi-threaded concurrency issues with FuseClient::UnInit() +// static void on_throttle_timer(void *arg) { +// FuseClient *fuseClient = reinterpret_cast(arg); +// fuseClient->InitQosParam(); +// } CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) { option_ = option; @@ -197,9 +199,9 @@ void FuseClient::UnInit() { delete mdsBase_; mdsBase_ = nullptr; - while (bthread_timer_del(throttleTimer_) == 1) { - bthread_usleep(1000); - } + // while (bthread_timer_del(throttleTimer_) == 1) { + // bthread_usleep(1000); + // } } CURVEFS_ERROR FuseClient::Run() { @@ -1431,11 +1433,11 @@ void FuseClient::InitQosParam() { throttle_.UpdateThrottleParams(params); - int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1), - on_throttle_timer, this); - if (ret != 0) { - LOG(ERROR) << "Create fuse client throttle timer failed!"; - } + // int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1), + // on_throttle_timer, this); + // if (ret != 0) { + // LOG(ERROR) << "Create fuse client throttle timer failed!"; + // } } } // namespace client diff --git a/curvefs/src/client/fuse_volume_client.cpp b/curvefs/src/client/fuse_volume_client.cpp index fcdbb17f15..fc270a0282 100644 --- a/curvefs/src/client/fuse_volume_client.cpp +++ b/curvefs/src/client/fuse_volume_client.cpp @@ -41,10 +41,15 @@ namespace curvefs { namespace client { +namespace common { +DECLARE_bool(enableCto); +} // namespace common + using ::curvefs::volume::SpaceManagerImpl; using ::curvefs::volume::SpaceManagerOption; using ::curvefs::volume::BlockDeviceClientOptions; using ::curvefs::volume::BlockDeviceClientImpl; +using ::curvefs::client::common::FLAGS_enableCto; CURVEFS_ERROR FuseVolumeClient::Init(const FuseClientOption &option) { volOpts_ = option.volumeOpt; @@ -111,9 +116,12 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata, volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit; option.allocatorOption.bitmapAllocatorOption.smallAllocProportion = volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion; + option.threshold = volOpts_.threshold; + option.releaseInterSec = volOpts_.releaseInterSec; spaceManager_ = absl::make_unique(option, mdsClient_, blockDeviceClient_); + spaceManager_->Run(); storage_ = absl::make_unique( spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get()); diff --git a/curvefs/src/client/inode_cache_manager.cpp b/curvefs/src/client/inode_cache_manager.cpp index 42035b6604..b71ac2ffa7 100644 --- a/curvefs/src/client/inode_cache_manager.cpp +++ b/curvefs/src/client/inode_cache_manager.cpp @@ -85,6 +85,7 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId, option_.refreshDataIntervalSec); // refresh data + VLOG(9) << "get inode: " << inodeId << " from icache fail, get from remote"; REFRESH_DATA_REMOTE(out, streaming); return CURVEFS_ERROR::OK; diff --git a/curvefs/src/client/inode_wrapper.cpp b/curvefs/src/client/inode_wrapper.cpp index 3a49e8ee70..a85f66c8a2 100644 --- a/curvefs/src/client/inode_wrapper.cpp +++ b/curvefs/src/client/inode_wrapper.cpp @@ -251,6 +251,8 @@ void InodeWrapper::FlushS3ChunkInfoAsync() { CURVEFS_ERROR InodeWrapper::FlushVolumeExtent() { std::lock_guard<::curve::common::Mutex> guard(syncingVolumeExtentsMtx_); if (!extentCache_.HasDirtyExtents()) { + VLOG(9) << "FlushVolumeExtent, ino: " << inode_.inodeid() + << ", no dirty extents"; return CURVEFS_ERROR::OK; } @@ -476,12 +478,18 @@ void InodeWrapper::Async(MetaServerClientDone *done, bool internal) { void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done, bool /*internal*/) { + VLOG(9) << "async inode: " << inode_.ShortDebugString() + << ", is dirty: " << dirty_ + << ", has dirty extents: " << extentCache_.HasDirtyExtents(); if (dirty_ || extentCache_.HasDirtyExtents()) { LockSyncingInode(); syncingVolumeExtentsMtx_.lock(); DataIndices indices; if (extentCache_.HasDirtyExtents()) { indices.volumeExtents = extentCache_.GetDirtyExtents(); + VLOG(9) << "aync inode: " << inode_.ShortDebugString() + << ", volume extents: " + << indices.volumeExtents->ShortDebugString(); } metaClient_->UpdateInodeWithOutNlinkAsync( @@ -583,7 +591,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) { } CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() { - VolumeExtentList extents; + VolumeExtentSliceList extents; auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(), true, &extents); VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid() diff --git a/curvefs/src/client/inode_wrapper.h b/curvefs/src/client/inode_wrapper.h index 543745751c..c9d50eda73 100644 --- a/curvefs/src/client/inode_wrapper.h +++ b/curvefs/src/client/inode_wrapper.h @@ -52,8 +52,9 @@ constexpr int kAccessTime = 1 << 0; constexpr int kChangeTime = 1 << 1; constexpr int kModifyTime = 1 << 2; -using ::curvefs::metaserver::VolumeExtentList; +using ::curvefs::metaserver::VolumeExtentSliceList; using ::curvefs::client::filesystem::CURVEFS_ERROR; +using ::curvefs::metaserver::VolumeExtentSliceList; enum class InodeStatus { kNormal = 0, diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index af38ceb200..2787aa4d7f 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -92,13 +92,12 @@ struct MetaServerClientMetric { // volume extent InterfaceMetric updateVolumeExtent; InterfaceMetric getVolumeExtent; + InterfaceMetric updateDeallocatableBlockGroup; MetaServerClientMetric() - : getDentry(prefix, "getDentry"), - listDentry(prefix, "listDentry"), + : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"), createDentry(prefix, "createDentry"), - deleteDentry(prefix, "deleteDentry"), - getInode(prefix, "getInode"), + deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"), batchGetInodeAttr(prefix, "batchGetInodeAttr"), batchGetXattr(prefix, "batchGetXattr"), createInode(prefix, "createInode"), @@ -107,7 +106,9 @@ struct MetaServerClientMetric { appendS3ChunkInfo(prefix, "appendS3ChunkInfo"), prepareRenameTx(prefix, "prepareRenameTx"), updateVolumeExtent(prefix, "updateVolumeExtent"), - getVolumeExtent(prefix, "getVolumeExtent") {} + getVolumeExtent(prefix, "getVolumeExtent"), + updateDeallocatableBlockGroup(prefix, + "updateDeallocatableBlockGroup") {} }; struct InflightGuard { diff --git a/curvefs/src/client/rpcclient/mds_client.cpp b/curvefs/src/client/rpcclient/mds_client.cpp index 561351d286..91a6d5fb86 100644 --- a/curvefs/src/client/rpcclient/mds_client.cpp +++ b/curvefs/src/client/rpcclient/mds_client.cpp @@ -714,10 +714,11 @@ SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup( CHECK_RPC_AND_RETRY_IF_ERROR("AllocateVolumeBlockGroup"); auto status = response.status(); - if (status != SpaceErrCode::SpaceOk) { - LOG(WARNING) << "Allocate volume block group failed, err: " - << SpaceErrCode_Name(status); - } else if (response.blockgroups_size() == 0) { + LOG_IF(WARNING, status != SpaceErrCode::SpaceOk) + << "Allocate volume block group failed, err: " + << SpaceErrCode_Name(status); + + if (response.blockgroups_size() == 0) { LOG(WARNING) << "Allocate volume block group failed, no block " "group allcoated"; return SpaceErrCode::SpaceErrNoSpace; diff --git a/curvefs/src/client/rpcclient/metaserver_client.cpp b/curvefs/src/client/rpcclient/metaserver_client.cpp index 181601309f..0422436981 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.cpp +++ b/curvefs/src/client/rpcclient/metaserver_client.cpp @@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor; using GetOrModifyS3ChunkInfoExcutor = TaskExecutor; using UpdateVolumeExtentExecutor = TaskExecutor; using GetVolumeExtentExecutor = TaskExecutor; +using UpdateDeallocatableBlockGroupExcutor = TaskExecutor; using ::curvefs::common::LatencyUpdater; using ::curvefs::common::StreamConnection; @@ -886,6 +887,7 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request, req.set_poolid(poolID); req.set_copysetid(copysetID); req.set_partitionid(partitionID); + VLOG(9) << "update inode async req: " << req.ShortDebugString(); auto *rpcDone = new UpdateInodeRpcDone(taskExecutorDone, &metric_); curvefs::metaserver::MetaServerService_Stub stub(channel); @@ -1245,8 +1247,8 @@ MetaStatusCode MetaServerClientImpl::CreateManageInode(const InodeParam ¶m, auto taskCtx = std::make_shared( MetaServerOpType::CreateManageInode, task, param.fsId, 0); - CreateInodeExcutor excutor(opt_, metaCache_, channelManager_, - std::move(taskCtx)); + CreateManagerInodeExcutor excutor(opt_, metaCache_, channelManager_, + std::move(taskCtx)); return ConvertToMetaStatusCode(excutor.DoRPCTask()); } @@ -1342,14 +1344,14 @@ void UpdateVolumeExtentRpcDone::Run() { } while (0) void MetaServerClientImpl::AsyncUpdateVolumeExtent( - uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents, + uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents, MetaServerClientDone *done) { auto task = AsyncRPCTask { (void)txId; metric_.updateVolumeExtent.qps.count << 1; metaserver::UpdateVolumeExtentRequest request; SET_COMMON_FIELDS; - request.set_allocated_extents(new VolumeExtentList{extents}); + request.set_allocated_extents(new VolumeExtentSliceList{extents}); auto *rpcDone = new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_); @@ -1369,7 +1371,8 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent( namespace { struct ParseVolumeExtentCallBack { - explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {} + explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext) + : extents(ext) {} bool operator()(butil::IOBuf *data) const { metaserver::VolumeExtentSlice slice; @@ -1382,7 +1385,7 @@ struct ParseVolumeExtentCallBack { return true; } - VolumeExtentList *extents; + VolumeExtentSliceList *extents; }; } // namespace @@ -1390,7 +1393,7 @@ struct ParseVolumeExtentCallBack { MetaStatusCode MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId, bool streaming, - VolumeExtentList *extents) { + VolumeExtentSliceList *extents) { auto task = RPCTask { (void)txId; (void)taskExecutorDone; @@ -1467,9 +1470,59 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId, return ConvertToMetaStatusCode(executor.DoRPCTask()); } -MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId, - uint64_t inodeid, - InodeAttr *attr) { +MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup( + uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) { + auto task = RPCTask { + metric_.updateDeallocatableBlockGroup.qps.count << 1; + LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency); + + metaserver::UpdateDeallocatableBlockGroupRequest request; + metaserver::UpdateDeallocatableBlockGroupResponse response; + request.set_poolid(poolID); + request.set_copysetid(copysetID); + request.set_partitionid(partitionID); + request.set_fsid(fsId); + auto *update = request.mutable_update(); + for (auto &it : *statistic) { + *(update->Add()) = std::move(it.second); + } + + curvefs::metaserver::MetaServerService_Stub stub(channel); + stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr); + + if (cntl->Failed()) { + metric_.updateDeallocatableBlockGroup.eps.count << 1; + LOG(WARNING) << "UpdateDeallocatableBlockGroup failed" + << ", errorCode = " << cntl->ErrorCode() + << ", errorText = " << cntl->ErrorText() + << ", logId = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + + auto rc = response.statuscode(); + if (rc != MetaStatusCode::OK) { + metric_.updateDeallocatableBlockGroup.eps.count << 1; + LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc + << ", message = " << MetaStatusCode_Name(rc); + return -1; + } + + VLOG(6) << "UpdateDeallocatableBlockGroup done, request: " + << request.DebugString() + << "response: " << response.DebugString(); + return rc; + }; + + auto taskCtx = std::make_shared( + MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId); + + UpdateDeallocatableBlockGroupExcutor excutor( + opt_, metaCache_, channelManager_, std::move(taskCtx)); + return ConvertToMetaStatusCode(excutor.DoRPCTask()); +} + +MetaStatusCode MetaServerClientImpl::GetInodeAttr( + uint32_t fsId, uint64_t inodeid, InodeAttr *attr) { std::set inodeIds; inodeIds.insert(inodeid); std::list attrs; diff --git a/curvefs/src/client/rpcclient/metaserver_client.h b/curvefs/src/client/rpcclient/metaserver_client.h index 419ed34943..cd2a233da3 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.h +++ b/curvefs/src/client/rpcclient/metaserver_client.h @@ -30,6 +30,7 @@ #include #include #include +#include #include "curvefs/proto/common.pb.h" #include "curvefs/proto/metaserver.pb.h" @@ -49,21 +50,24 @@ using ::curvefs::metaserver::InodeAttr; using ::curvefs::metaserver::XAttr; using ::curvefs::metaserver::MetaStatusCode; using ::curvefs::metaserver::S3ChunkInfoList; +using ::curvefs::metaserver::DeallocatableBlockGroup; using ::curvefs::common::StreamStatus; using ::curvefs::common::StreamClient; -using S3ChunkInfoMap = google::protobuf::Map; using ::curvefs::metaserver::Time; +using DeallocatableBlockGroupMap = std::map; +using S3ChunkInfoMap = google::protobuf::Map; + namespace curvefs { namespace client { namespace rpcclient { using S3ChunkInfoMap = google::protobuf::Map; -using ::curvefs::metaserver::VolumeExtentList; +using ::curvefs::metaserver::VolumeExtentSliceList; struct DataIndices { absl::optional s3ChunkInfoMap; - absl::optional volumeExtents; + absl::optional volumeExtents; }; class MetaServerClient { @@ -161,13 +165,17 @@ class MetaServerClient { virtual void AsyncUpdateVolumeExtent(uint32_t fsId, uint64_t inodeId, - const VolumeExtentList &extents, + const VolumeExtentSliceList &extents, MetaServerClientDone *done) = 0; virtual MetaStatusCode GetVolumeExtent(uint32_t fsId, uint64_t inodeId, bool streaming, - VolumeExtentList *extents) = 0; + VolumeExtentSliceList *extents) = 0; + + virtual MetaStatusCode + UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId, + DeallocatableBlockGroupMap *statistic) = 0; }; class MetaServerClientImpl : public MetaServerClient { @@ -265,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient { void AsyncUpdateVolumeExtent(uint32_t fsId, uint64_t inodeId, - const VolumeExtentList &extents, + const VolumeExtentSliceList &extents, MetaServerClientDone *done) override; MetaStatusCode GetVolumeExtent(uint32_t fsId, uint64_t inodeId, bool streaming, - VolumeExtentList *extents) override; + VolumeExtentSliceList *extents) override; + + MetaStatusCode UpdateDeallocatableBlockGroup( + uint32_t fsId, uint64_t inodeId, + DeallocatableBlockGroupMap *statistic) override; private: MetaStatusCode UpdateInode(const UpdateInodeRequest &request, diff --git a/curvefs/src/client/rpcclient/task_excutor.cpp b/curvefs/src/client/rpcclient/task_excutor.cpp index d575109cab..191cff6e08 100644 --- a/curvefs/src/client/rpcclient/task_excutor.cpp +++ b/curvefs/src/client/rpcclient/task_excutor.cpp @@ -27,8 +27,10 @@ #include "curvefs/src/client/rpcclient/task_excutor.h" #include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/common/define.h" using ::curvefs::metaserver::MetaStatusCode; +using ::curvefs::RECYCLEINODEID; namespace curvefs { namespace client { @@ -337,6 +339,16 @@ bool CreateInodeExcutor::GetTarget() { return true; } +bool CreateManagerInodeExcutor::GetTarget() { + if (!metaCache_->GetTarget(task_->fsID, RECYCLEINODEID, &task_->target)) { + LOG(ERROR) << "CreateManagerInodeExcutor select target for task fail, " + << task_->TaskContextStr() + << ", recycleInodeId = " << RECYCLEINODEID; + return false; + } + return true; +} + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/rpcclient/task_excutor.h b/curvefs/src/client/rpcclient/task_excutor.h index f4f5478ff0..18a69346e5 100644 --- a/curvefs/src/client/rpcclient/task_excutor.h +++ b/curvefs/src/client/rpcclient/task_excutor.h @@ -272,6 +272,18 @@ class CreateInodeExcutor : public TaskExecutor { bool GetTarget() override; }; +class CreateManagerInodeExcutor : public TaskExecutor { + public: + explicit CreateManagerInodeExcutor( + const ExcutorOpt &opt, const std::shared_ptr &metaCache, + const std::shared_ptr> &channelManager, + const std::shared_ptr &task) + : TaskExecutor(opt, metaCache, channelManager, task) {} + + protected: + bool GetTarget() override; +}; + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/volume/default_volume_storage.cpp b/curvefs/src/client/volume/default_volume_storage.cpp index f6aa3e614d..4b2618b170 100644 --- a/curvefs/src/client/volume/default_volume_storage.cpp +++ b/curvefs/src/client/volume/default_volume_storage.cpp @@ -188,6 +188,8 @@ CURVEFS_ERROR DefaultVolumeStorage::Flush(uint64_t ino) { return CURVEFS_ERROR::OK; } + VLOG(9) << "volume storage flush: " << ino; + LatencyUpdater updater(&metric_.flushLatency); std::shared_ptr inodeWrapper; auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper); diff --git a/curvefs/src/client/volume/extent_cache.cpp b/curvefs/src/client/volume/extent_cache.cpp index 1e8d3ab7d6..7da6117148 100644 --- a/curvefs/src/client/volume/extent_cache.cpp +++ b/curvefs/src/client/volume/extent_cache.cpp @@ -90,6 +90,10 @@ void ExtentCache::Merge(uint64_t loffset, const PExtent& pExt) { slice->second.Merge(loffset, pExt); dirties_.insert(&slice->second); + VLOG(9) << "merge extent, loffset: " << loffset + << ", physical offset: " << pExt.pOffset << ", len: " << pExt.len + << ", written: " << !pExt.UnWritten + << ", slice: " << slice->second.ToVolumeExtentSlice().DebugString(); } void ExtentCache::DivideForWrite(uint64_t offset, @@ -162,12 +166,24 @@ void ExtentCache::MarkWritten(uint64_t offset, uint64_t len) { auto cur = align_down(offset, option_.blockSize); const auto end = align_up(offset + len, option_.blockSize); + VLOG(9) << "mark written for offset: " << offset << ", len: " << len + << ", cur: " << cur << ", end: " << end; + while (cur < end) { const auto length = std::min(end - cur, option_.sliceSize - (cur & ~option_.sliceSize)); auto slice = slices_.find(align_down(cur, option_.sliceSize)); assert(slice != slices_.end()); + VLOG(9) << "mark written for offset: " << offset << ", len: " << len + << ", cur: " << cur << ", end: " << end + << ", before mark written slice: " + << slice->second.ToVolumeExtentSlice().DebugString(); auto changed = slice->second.MarkWritten(cur, length); + VLOG(9) << "mark written for offset: " << offset << ", len: " << len + << ", cur: " << cur << ", end: " << end + << ", after mark written slice changed: " << changed + << ", slice: " + << slice->second.ToVolumeExtentSlice().DebugString(); cur += length; if (changed) { dirties_.insert((&slice->second)); @@ -194,6 +210,8 @@ void ExtentCache::DivideForRead(uint64_t offset, std::vector* holes) { LatencyUpdater updater(&g_read_divide_latency); ReadLockGuard lk(lock_); + VLOG(9) << "extent cache divide for read offset: " << offset + << ", length: " << len; const auto end = offset + len; char* datap = data; @@ -205,8 +223,15 @@ void ExtentCache::DivideForRead(uint64_t offset, auto slice = slices_.find(align_down(offset, option_.sliceSize)); if (slice != slices_.end()) { slice->second.DivideForRead(offset, length, datap, reads, holes); + VLOG(9) << "extent cache find slice for read offset: " << offset + << ", length: " << len << ", slice: " + << slice->second.ToVolumeExtentSlice().DebugString() + << ", slices size: " << slices_.size(); } else { holes->emplace_back(offset, length, datap); + VLOG(9) << "extent cache not find slice for read offset: " << offset + << ", length: " << len + << ", slices size: " << slices_.size(); } datap += length; @@ -228,7 +253,7 @@ void ExtentCache::SetOption(const ExtentCacheOption& option) { LOG(INFO) << "ExtentCacheOption: [" << option_ << "]"; } -void ExtentCache::Build(const VolumeExtentList &extents) { +void ExtentCache::Build(const VolumeExtentSliceList &extents) { WriteLockGuard lk(lock_); slices_.clear(); dirties_.clear(); @@ -238,14 +263,15 @@ void ExtentCache::Build(const VolumeExtentList &extents) { } } -VolumeExtentList ExtentCache::GetDirtyExtents() { - VolumeExtentList result; +VolumeExtentSliceList ExtentCache::GetDirtyExtents() { + VolumeExtentSliceList result; WriteLockGuard lk(lock_); for (const auto* slice : dirties_) { *result.add_slices() = slice->ToVolumeExtentSlice(); } dirties_.clear(); + VLOG(9) << "extent cache get and clear dirty extents"; return result; } diff --git a/curvefs/src/client/volume/extent_cache.h b/curvefs/src/client/volume/extent_cache.h index 87f747cfd7..49bac560e5 100644 --- a/curvefs/src/client/volume/extent_cache.h +++ b/curvefs/src/client/volume/extent_cache.h @@ -43,7 +43,7 @@ namespace client { using ::curvefs::volume::ReadPart; using ::curvefs::volume::WritePart; using ::curvefs::metaserver::VolumeExtentSlice; -using ::curvefs::metaserver::VolumeExtentList; +using ::curvefs::metaserver::VolumeExtentSliceList; struct ExtentCacheOption { // preallocation size if offset ~ length is not allocated @@ -61,7 +61,7 @@ class ExtentCache { static void SetOption(const ExtentCacheOption& option); - void Build(const VolumeExtentList& extents); + void Build(const VolumeExtentSliceList& extents); void DivideForWrite(uint64_t offset, uint64_t len, @@ -81,7 +81,7 @@ class ExtentCache { bool HasDirtyExtents() const; - VolumeExtentList GetDirtyExtents(); + VolumeExtentSliceList GetDirtyExtents(); std::unordered_map> GetExtentsForTesting() const; diff --git a/curvefs/src/mds/fs_info_wrapper.cpp b/curvefs/src/mds/fs_info_wrapper.cpp index 83b1d9c209..e1fec715b6 100644 --- a/curvefs/src/mds/fs_info_wrapper.cpp +++ b/curvefs/src/mds/fs_info_wrapper.cpp @@ -1,4 +1,3 @@ - /* * Copyright (c) 2021 NetEase Inc. * @@ -50,8 +49,13 @@ FsInfoWrapper::FsInfoWrapper(const ::curvefs::mds::CreateFsRequest* request, fsInfo.set_enablesumindir(request->enablesumindir()); fsInfo.set_txsequence(0); fsInfo.set_txowner(""); + // TODO(@lixiaocui1): Currently, curveadm does not support setting recycling + // when creating a file system. It is expected to be supported in + // curveadm 3.0 if (request->has_recycletimehour()) { fsInfo.set_recycletimehour(request->recycletimehour()); + } else { + fsInfo.set_recycletimehour(1); } const auto& detail = request->fsdetail(); diff --git a/curvefs/src/mds/fs_manager.cpp b/curvefs/src/mds/fs_manager.cpp index 021bea0115..b5d67b2bf3 100644 --- a/curvefs/src/mds/fs_manager.cpp +++ b/curvefs/src/mds/fs_manager.cpp @@ -54,7 +54,7 @@ using NameLockGuard = ::curve::common::GenericNameLockGuard; bool FsManager::Init() { LOG_IF(FATAL, !fsStorage_->Init()) << "fsStorage Init fail"; s3Adapter_->Init(option_.s3AdapterOption); - auto ret = ReloadMountedFsVolumeSpace(); + auto ret = ReloadFsVolumeSpace(); if (ret != FSStatusCode::OK) { LOG(ERROR) << "Reload mounted fs volume space error"; } @@ -682,25 +682,34 @@ FSStatusCode FsManager::UmountFs(const std::string& fsName, << ", errCode = " << FSStatusCode_Name(ret); return ret; } + VLOG(3) << "FsManager delete mount point success, fsName = " << fsName + << ", mountpoint = " << mountpoint.ShortDebugString(); std::string mountpath; MountPoint2Str(mountpoint, &mountpath); DeleteClientAliveTime(mountpath); - // 3. if no mount point exist, uninit space - if (wrapper.GetFsType() == FSType::TYPE_VOLUME && - wrapper.IsMountPointEmpty()) { - auto ret = spaceManager_->RemoveVolume(wrapper.GetFsId()); - if (ret != space::SpaceOk) { - LOG(ERROR) << "UmountFs fail, uninit space fail, fsName = " + // 3. if no mount point exist, release all block groups + if (wrapper.GetFsType() == FSType::TYPE_VOLUME) { + auto volumeSpace = spaceManager_->GetVolumeSpace(wrapper.GetFsId()); + if (volumeSpace == nullptr) { + LOG(ERROR) << "handle fs mount point timeout fail, get volume " + "space fail, fsName = " << fsName - << ", mountpoint = " << mountpoint.ShortDebugString() - << ", errCode = " << space::SpaceErrCode_Name(ret); + << ", mountpoint = " << mountpoint.ShortDebugString(); return UNINIT_SPACE_ERROR; } - LOG(INFO) << "Remove volume space success, fsName = " << fsName - << ", fsId = " << wrapper.GetFsId(); + auto ret = volumeSpace->ReleaseBlockGroups(mountpath); + if (ret != space::SpaceOk) { + LOG(ERROR) + << "handle fs mount point timeout fail,release block groups " + "fail, fsName = " + << fsName << ", mountpoint = " << mountpoint.ShortDebugString(); + return SPACE_RELEASE_FAIL; + } + + VLOG(3) << "FsManager release block group for " << mountpath << " ok"; } // 4. update fs info @@ -855,7 +864,7 @@ void FsManager::RefreshSession(const RefreshSessionRequest* request, response->set_enablesumindir(wrapper.ProtoFsInfo().enablesumindir()); } -FSStatusCode FsManager::ReloadMountedFsVolumeSpace() { +FSStatusCode FsManager::ReloadFsVolumeSpace() { std::vector allfs; fsStorage_->GetAll(&allfs); @@ -865,9 +874,7 @@ FSStatusCode FsManager::ReloadMountedFsVolumeSpace() { continue; } - if (!fs.MountPoints().empty()) { - reloader.Add(fs.ProtoFsInfo()); - } + reloader.Add(fs.ProtoFsInfo()); } auto err = reloader.Wait(); diff --git a/curvefs/src/mds/fs_manager.h b/curvefs/src/mds/fs_manager.h index 03275b781b..5b20aec6c2 100644 --- a/curvefs/src/mds/fs_manager.h +++ b/curvefs/src/mds/fs_manager.h @@ -221,7 +221,7 @@ class FsManager { // set partition status to DELETING in topology bool SetPartitionToDeleting(const PartitionInfo& partition); - FSStatusCode ReloadMountedFsVolumeSpace(); + FSStatusCode ReloadFsVolumeSpace(); void GetLatestTxId(const uint32_t fsId, std::vector* txIds); diff --git a/curvefs/src/mds/heartbeat/BUILD b/curvefs/src/mds/heartbeat/BUILD index 379e3e5456..90df1fac23 100644 --- a/curvefs/src/mds/heartbeat/BUILD +++ b/curvefs/src/mds/heartbeat/BUILD @@ -27,6 +27,7 @@ cc_library( "//curvefs/proto:curvefs_topology_cc_proto", "//curvefs/proto:metaserver_cc_proto", "//curvefs/src/mds/topology:curvefs_topology", + "//curvefs/src/mds/space:curvefs_mds_space", "//external:brpc", "//external:gflags", "//external:glog", diff --git a/curvefs/src/mds/heartbeat/heartbeat_manager.cpp b/curvefs/src/mds/heartbeat/heartbeat_manager.cpp index ab54afa49b..c41864c71c 100644 --- a/curvefs/src/mds/heartbeat/heartbeat_manager.cpp +++ b/curvefs/src/mds/heartbeat/heartbeat_manager.cpp @@ -42,8 +42,9 @@ namespace mds { namespace heartbeat { HeartbeatManager::HeartbeatManager( const HeartbeatOption &option, const std::shared_ptr &topology, - const std::shared_ptr &coordinator) - : topology_(topology) { + const std::shared_ptr &coordinator, + const std::shared_ptr &spaceManager) + : topology_(topology), spaceManager_(spaceManager) { healthyChecker_ = std::make_shared(option, topology); @@ -86,24 +87,6 @@ void HeartbeatManager::Stop() { } } -void HeartbeatManager::MetaServerHealthyChecker() { - while (sleeper_.wait_for( - std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) { - healthyChecker_->CheckHeartBeatInterval(); - } -} - -void HeartbeatManager::UpdateMetaServerSpace( - const MetaServerHeartbeatRequest &request) { - MetaServerSpace space(request.spacestatus()); - TopoStatusCode ret = - topology_->UpdateMetaServerSpace(space, request.metaserverid()); - if (ret != TopoStatusCode::TOPO_OK) { - LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = " - << TopoStatusCode_Name(ret); - } -} - void HeartbeatManager::MetaServerHeartbeat( const MetaServerHeartbeatRequest &request, MetaServerHeartbeatResponse *response) { @@ -127,6 +110,32 @@ void HeartbeatManager::MetaServerHeartbeat( UpdateMetaServerSpace(request); // dealing with copysets included in the heartbeat request + Coordinate(request, response); + + // update deallocatable block group info + UpdateDeallocatableBlockGroup(request, response); +} + +void HeartbeatManager::MetaServerHealthyChecker() { + while (sleeper_.wait_for( + std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) { + healthyChecker_->CheckHeartBeatInterval(); + } +} + +void HeartbeatManager::UpdateMetaServerSpace( + const MetaServerHeartbeatRequest &request) { + MetaServerSpace space(request.spacestatus()); + TopoStatusCode ret = + topology_->UpdateMetaServerSpace(space, request.metaserverid()); + if (ret != TopoStatusCode::TOPO_OK) { + LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = " + << TopoStatusCode_Name(ret); + } +} + +void HeartbeatManager::Coordinate(const MetaServerHeartbeatRequest &request, + MetaServerHeartbeatResponse *response) { for (auto &value : request.copysetinfos()) { // convert copysetInfo from heartbeat format to topology format ::curvefs::mds::topology::CopySetInfo reportCopySetInfo; @@ -168,6 +177,40 @@ void HeartbeatManager::MetaServerHeartbeat( } } +void HeartbeatManager::UpdateDeallocatableBlockGroup( + const MetaServerHeartbeatRequest &request, + MetaServerHeartbeatResponse *response) { + uint32_t metaserverId = request.metaserverid(); + VLOG(6) << "HeartbeatManager get block group stat info from metaserver:" + << request.metaserverid() + << ", size:" << request.blockgroupstatinfos_size(); + + for (auto &info : request.blockgroupstatinfos()) { + VLOG(9) << "HeartbeatManager handle request from metaserver:" + << request.metaserverid() << ", fsid:" << info.fsid() + << ", block stat info:" << info.DebugString(); + auto volumeSpace = spaceManager_->GetVolumeSpace(info.fsid()); + if (volumeSpace == nullptr) { + LOG(ERROR) << "HeartbeatManager fsid=" << info.fsid() + << " do not have volumeSpace manager"; + response->set_statuscode(HeartbeatStatusCode::hbMetaServerFSUnkown); + return; + } + + uint64_t issued = 0; + bool hasissued = volumeSpace->UpdateDeallocatableBlockGroup( + metaserverId, info.deallocatableblockgroups(), + info.blockgroupdeallocatestatus(), &issued); + if (hasissued) { + response->mutable_issuedblockgroups()->insert( + {info.fsid(), issued}); + LOG(INFO) << "HeartbeatManager issue metaserverid=" << metaserverId + << " blockgroup, fsid=" << info.fsid() + << ", issued=" << issued; + } + } +} + HeartbeatStatusCode HeartbeatManager::CheckRequest( const MetaServerHeartbeatRequest &request) { MetaServer metaServer; @@ -203,6 +246,8 @@ HeartbeatStatusCode HeartbeatManager::CheckRequest( << metaServer.GetToken(); return HeartbeatStatusCode::hbMetaServerTokenNotMatch; } + + VLOG(6) << "HeartbeatManager get request:" << request.DebugString(); return HeartbeatStatusCode::hbOK; } diff --git a/curvefs/src/mds/heartbeat/heartbeat_manager.h b/curvefs/src/mds/heartbeat/heartbeat_manager.h index ec19585786..29eded4beb 100644 --- a/curvefs/src/mds/heartbeat/heartbeat_manager.h +++ b/curvefs/src/mds/heartbeat/heartbeat_manager.h @@ -36,6 +36,7 @@ #include "curvefs/src/mds/heartbeat/topo_updater.h" #include "curvefs/src/mds/schedule/coordinator.h" #include "curvefs/src/mds/topology/topology.h" +#include "curvefs/src/mds/space/manager.h" #include "src/common/concurrent/concurrent.h" #include "src/common/interruptible_sleeper.h" @@ -43,6 +44,7 @@ using ::curvefs::mds::topology::PoolIdType; using ::curvefs::mds::topology::CopySetIdType; using ::curvefs::mds::topology::Topology; using ::curvefs::mds::schedule::Coordinator; +using ::curvefs::mds::space::SpaceManager; using ::curve::common::Thread; using ::curve::common::Atomic; @@ -65,7 +67,8 @@ class HeartbeatManager { public: HeartbeatManager(const HeartbeatOption &option, const std::shared_ptr &topology, - const std::shared_ptr &coordinator); + const std::shared_ptr &coordinator, + const std::shared_ptr &spaceManager); ~HeartbeatManager() { Stop(); } @@ -98,6 +101,7 @@ class HeartbeatManager { MetaServerHeartbeatResponse *response); private: + FRIEND_TEST(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup); /** * @brief Background thread for heartbeat timeout inspection */ @@ -140,10 +144,18 @@ class HeartbeatManager { void UpdateMetaServerSpace(const MetaServerHeartbeatRequest &request); + void Coordinate(const MetaServerHeartbeatRequest &request, + MetaServerHeartbeatResponse *response); + + void + UpdateDeallocatableBlockGroup(const MetaServerHeartbeatRequest &request, + MetaServerHeartbeatResponse *response); + private: // Dependencies of heartbeat std::shared_ptr topology_; std::shared_ptr coordinator_; + std::shared_ptr spaceManager_; // healthyChecker_ health checker running in background thread std::shared_ptr healthyChecker_; diff --git a/curvefs/src/mds/mds.cpp b/curvefs/src/mds/mds.cpp index d5c3ff1afb..6e5d8644b5 100644 --- a/curvefs/src/mds/mds.cpp +++ b/curvefs/src/mds/mds.cpp @@ -68,6 +68,8 @@ void MDS::InitOptions(std::shared_ptr conf) { conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort); conf_->GetValueFatalIfFail("mds.server.idleTimeoutSec", &options_.idleTimeoutSec); + conf_->GetValueFatalIfFail("mds.space.calIntervalSec", + &options_.mdsSpaceCalIntervalSec); InitMetaServerOption(&options_.metaserverOptions); InitTopologyOption(&options_.topologyOptions); @@ -171,8 +173,8 @@ void MDS::Init() { space::MdsProxyManager::SetProxyOptions(options_.bsMdsProxyOptions); fsStorage_ = std::make_shared(etcdClient_); - spaceManager_ = - std::make_shared(etcdClient_, fsStorage_); + spaceManager_ = std::make_shared( + etcdClient_, fsStorage_, options_.mdsSpaceCalIntervalSec); metaserverClient_ = std::make_shared(options_.metaserverOptions); auto dlock = std::make_shared(options_.dLockOptions, etcdClient_); @@ -420,7 +422,7 @@ void MDS::InitHeartbeatManager() { heartbeatOption.mdsStartTime = steady_clock::now(); heartbeatManager_ = std::make_shared( - heartbeatOption, topology_, coordinator_); + heartbeatOption, topology_, coordinator_, spaceManager_); heartbeatManager_->Init(); } diff --git a/curvefs/src/mds/mds.h b/curvefs/src/mds/mds.h index 528a8e6579..52df06388c 100644 --- a/curvefs/src/mds/mds.h +++ b/curvefs/src/mds/mds.h @@ -90,6 +90,8 @@ struct MDSOptions { MetaserverOptions metaserverOptions; // TODO(add EtcdConf): add etcd configure + uint64_t mdsSpaceCalIntervalSec; + TopologyOption topologyOptions; HeartbeatOption heartbeatOption; ScheduleOption scheduleOption; diff --git a/curvefs/src/mds/space/BUILD b/curvefs/src/mds/space/BUILD index 81893738d3..f77c2633e4 100644 --- a/curvefs/src/mds/space/BUILD +++ b/curvefs/src/mds/space/BUILD @@ -27,6 +27,8 @@ cc_library( deps = [ "//curvefs/proto:mds_cc_proto", "//curvefs/proto:space_cc_proto", + "//curvefs/proto:metaserver_cc_proto", + "//curvefs/proto:curvefs_heartbeat_cc_proto", "//curvefs/src/mds:curvefs_mds_fs_storage", "//curvefs/src/mds/codec:fs_mds_codec", "//external:brpc", diff --git a/curvefs/src/mds/space/manager.cpp b/curvefs/src/mds/space/manager.cpp index be088e6e76..d97d38a6aa 100644 --- a/curvefs/src/mds/space/manager.cpp +++ b/curvefs/src/mds/space/manager.cpp @@ -55,12 +55,15 @@ SpaceErrCode SpaceManagerImpl::AddVolume(const FsInfo& fsInfo) { { ReadLockGuard lk(rwlock_); if (volumes_.count(fsInfo.fsid()) != 0) { - return SpaceErrCode::SpaceErrExist; + LOG(WARNING) << "Volume space already exists, fsId: " + << fsInfo.fsid(); + return SpaceOk; } } - auto space = VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(), - storage_.get(), fsStorage_.get()); + auto space = + VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(), + storage_.get(), fsStorage_.get(), calcIntervalSec_); if (!space) { LOG(ERROR) << "Create volume space failed, fsId: " << fsInfo.fsid(); diff --git a/curvefs/src/mds/space/manager.h b/curvefs/src/mds/space/manager.h index bd889f2614..695ce7e405 100644 --- a/curvefs/src/mds/space/manager.h +++ b/curvefs/src/mds/space/manager.h @@ -53,10 +53,10 @@ class SpaceManager { class SpaceManagerImpl final : public SpaceManager { public: SpaceManagerImpl( - const std::shared_ptr& kvstore, - std::shared_ptr fsStorage) + const std::shared_ptr &kvstore, + std::shared_ptr fsStorage, uint64_t calcIntervalSec) : storage_(new BlockGroupStorageImpl(kvstore)), - fsStorage_(std::move(fsStorage)) {} + fsStorage_(std::move(fsStorage)), calcIntervalSec_(calcIntervalSec) {} SpaceManagerImpl(const SpaceManagerImpl&) = delete; SpaceManagerImpl& operator=(const SpaceManagerImpl&) = delete; @@ -80,6 +80,8 @@ class SpaceManagerImpl final : public SpaceManager { curve::common::GenericNameLock namelock_; std::shared_ptr fsStorage_; + + uint64_t calcIntervalSec_; }; } // namespace space diff --git a/curvefs/src/mds/space/service.cpp b/curvefs/src/mds/space/service.cpp index 550bfa9199..6cd938dedb 100644 --- a/curvefs/src/mds/space/service.cpp +++ b/curvefs/src/mds/space/service.cpp @@ -51,16 +51,19 @@ void SpaceServiceImpl::AllocateBlockGroup( std::vector groups; auto err = space->AllocateBlockGroups(request->count(), request->owner(), &groups); - if (err != SpaceOk) { - LOG(ERROR) << "Allocate block groups failed, err: " - << SpaceErrCode_Name(err); - } else { - for (auto& group : groups) { - response->add_blockgroups()->Swap(&group); - } + response->set_status(err); + + LOG_IF(WARNING, err != SpaceOk) + << "Allocate block groups failed, err: " << SpaceErrCode_Name(err) + << ", allocated size: " << groups.size() + << ", request: " << request->ShortDebugString(); + if (err == SpaceErrEncode) { + return; } - response->set_status(err); + for (auto &group : groups) { + response->add_blockgroups()->Swap(&group); + } } void SpaceServiceImpl::AcquireBlockGroup( diff --git a/curvefs/src/mds/space/volume_space.cpp b/curvefs/src/mds/space/volume_space.cpp index 32e8a000eb..1364c13c1e 100644 --- a/curvefs/src/mds/space/volume_space.cpp +++ b/curvefs/src/mds/space/volume_space.cpp @@ -20,8 +20,7 @@ * Author: wuhanqing */ -#include "curvefs/src/mds/space/volume_space.h" - +#include #include #include @@ -39,6 +38,7 @@ #include "curvefs/src/common/fast_align.h" #include "curvefs/src/mds/fs_info_wrapper.h" #include "curvefs/src/mds/space/mds_proxy_manager.h" +#include "curvefs/src/mds/space/volume_space.h" namespace curvefs { namespace mds { @@ -65,9 +65,10 @@ BlockGroup BuildBlockGroupFromClean(uint64_t offset, } // namespace std::unique_ptr VolumeSpace::Create(uint32_t fsId, - const Volume& volume, - BlockGroupStorage* storage, - FsStorage* fsStorage) { + const Volume &volume, + BlockGroupStorage *storage, + FsStorage *fsStorage, + uint64_t calcIntervalSec) { if (!volume.has_volumesize()) { LOG(ERROR) << "Volume info doesn't have size"; return nullptr; @@ -94,7 +95,7 @@ std::unique_ptr VolumeSpace::Create(uint32_t fsId, // for allocated groups, client will send heartbeat to update usage uint64_t availableSize = 0; std::set usedGroupOffsets; - for (auto& group : groups) { + for (auto &group : groups) { usedGroupOffsets.insert(group.offset()); // availableSize += group.available(); @@ -102,11 +103,23 @@ std::unique_ptr VolumeSpace::Create(uint32_t fsId, assert(offset % blockGroupSize == 0); assert(group.size() == blockGroupSize); assert(group.bitmaplocation() == location); + assert((group.has_owner() && group.deallocating_size()) == 0); if (group.has_owner()) { + VLOG(6) << "VolumeSpace init for fsid=" << fsId + << ", blockgroup=" << group.DebugString() + << " to allocatedGroups_"; space->allocatedGroups_.emplace(offset, std::move(group)); + } else if (group.deallocating_size() || group.deallocated_size()) { + VLOG(6) << "VolumeSpace init for fsid=" << fsId + << ", blockgroup=" << group.DebugString() + << " to deallocatingGroups_"; + space->deallocatingGroups_.emplace(offset, std::move(group)); } else { + group.set_available(group.size()); + VLOG(6) << "VolumeSpace init for fsid=" << fsId + << ", blockgroup=" << group.DebugString() + << " to availableGroups_"; space->availableGroups_.emplace(offset, std::move(group)); - availableSize += group.available(); } } @@ -123,6 +136,8 @@ std::unique_ptr VolumeSpace::Create(uint32_t fsId, space->cleanGroups_ = std::move(cleanGroupOffsets); + space->calcIntervalSec_ = calcIntervalSec; + LOG(INFO) << "Init volume space success, fsId: " << fsId << ", size: " << volumeSize << ", available: " << availableSize << ", block size: " << blockSize @@ -130,8 +145,10 @@ std::unique_ptr VolumeSpace::Create(uint32_t fsId, << ", total groups: " << volumeSize / blockGroupSize << ", allocated groups: " << space->allocatedGroups_.size() << ", available groups: " << space->availableGroups_.size() + << ", deallocating groups: " << space->deallocatingGroups_.size() << ", clean groups: " << space->cleanGroups_.size(); + space->Run(); return space; } @@ -144,30 +161,38 @@ VolumeSpace::VolumeSpace(uint32_t fsId, storage_(storage), fsStorage_(fsStorage) {} + SpaceErrCode VolumeSpace::AllocateBlockGroups( uint32_t count, const std::string& owner, std::vector* blockGroups) { LockGuard lk(mtx_); auto err = AllocateBlockGroupsInternal(count, owner, blockGroups); - if (err != SpaceOk) { - LOG(WARNING) << "Allocate block groups failed, fsId: " << fsId_ - << ", err: " << SpaceErrCode_Name(err); - return err; + if (blockGroups->size() < count) { + LOG(WARNING) << "Allocate block groups not enough, fsId: " << fsId_ + << ", err: " << SpaceErrCode_Name(err) + << ", need count: " << count + << ", allocated count: " << blockGroups->size(); } - for (auto& group : *blockGroups) { - allocatedGroups_.emplace(group.offset(), group); - } + if (blockGroups->size() > 0) { + for (const auto &group : *blockGroups) { + allocatedGroups_.emplace(group.offset(), group); + VLOG(9) << "VolumeSpace fsid=" << fsId_ + << ", allocate blockgroup=" << group.DebugString() + << " to owner:" << owner; + } - err = PersistBlockGroups(*blockGroups); - if (err != SpaceOk) { - LOG(WARNING) << "Mark group allocated failed, fsId: " << fsId_ - << ", err: " << SpaceErrCode_Name(err); - return err; + // TODO(@wu-hanqing): if persist fail, we should rollback + err = PersistBlockGroups(*blockGroups); + if (err != SpaceOk) { + LOG(WARNING) << "Mark group allocated failed, fsId: " << fsId_ + << ", err: " << SpaceErrCode_Name(err); + return err; + } } - return SpaceOk; + return err; } SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal( @@ -177,6 +202,8 @@ SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal( bool extend = false; uint32_t allocated = 0; + VLOG(9) << "owner " << owner << " need allocate " << count + << " block groups"; while (allocated < count) { allocated += AllocateFromCleanGroups(count, owner, blockGroups); if (allocated >= count) { @@ -199,6 +226,8 @@ SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal( return SpaceErrNoSpace; } } else { + VLOG(9) << "only allocate " << count << " block groups to owner " + << owner; break; } } @@ -227,10 +256,27 @@ uint32_t VolumeSpace::AllocateFromAvailableGroups( uint32_t count, const std::string& owner, std::vector* groups) { + VLOG(9) << "VolumeSpace fsid=" << fsId_ + << ", allocate from available groups, count: " << count + << ", owner: " << owner + << ", available size:" << availableGroups_.size(); uint32_t allocated = 0; auto it = availableGroups_.begin(); while (allocated < count && it != availableGroups_.end()) { assert(!it->second.has_owner()); + + float usePer = 1.0 - static_cast(it->second.available()) / + static_cast(it->second.size()); + if (usePer > 0.95) { + LOG(WARNING) << "VolumeSpace fsid=" << fsId_ + << " available group=" << it->second.DebugString() + << " has no available space"; + it++; + continue; + } + VLOG(9) << "VolumeSpace fsid=" << fsId_ + << ", allocate blockgroup=" << it->second.DebugString() + << " to owner:" << owner; ++allocated; it->second.set_owner(owner); groups->push_back(std::move(it->second)); @@ -244,6 +290,7 @@ SpaceErrCode VolumeSpace::AcquireBlockGroup(uint64_t blockGroupOffset, const std::string& owner, BlockGroup* group) { LockGuard lk(mtx_); + auto err = AcquireBlockGroupInternal(blockGroupOffset, owner, group); if (err != SpaceOk) { LOG(WARNING) << "Acquire block group failed, fsId: " << fsId_ @@ -267,6 +314,21 @@ SpaceErrCode VolumeSpace::AcquireBlockGroup(uint64_t blockGroupOffset, SpaceErrCode VolumeSpace::AcquireBlockGroupInternal(uint64_t blockGroupOffset, const std::string& owner, BlockGroup* group) { + if (owner.empty()) { + // find in deallocating + auto it = deallocatingGroups_.find(blockGroupOffset); + if (it == deallocatingGroups_.end()) { + return SpaceErrNotFound; + } + + *group = it->second; + VLOG(6) << "VolumeSpace fsid=" << fsId_ + << ", recieve acquire blockgroup=" << group->DebugString() + << " request from metaserver, current block group is under " + "deallocating"; + return SpaceOk; + } + // find in availables { auto it = availableGroups_.find(blockGroupOffset); @@ -312,18 +374,39 @@ SpaceErrCode VolumeSpace::AcquireBlockGroupInternal(uint64_t blockGroupOffset, return SpaceErrNotFound; } +#define RELEASE_TO_AVAILABLE_GROUPS(GROUP) \ + do { \ + auto copy = GROUP; \ + copy.clear_owner(); \ + auto err = PersistBlockGroup(copy); \ + if (err != SpaceOk) { \ + LOG(WARNING) << "Persist block group failed, fsId: " << fsId_ \ + << ", block group offset: " << GROUP.offset() \ + << ", err: " << SpaceErrCode_Name(err); \ + return err; \ + } \ + VLOG(6) << "VolumeSpace return block group for fsid=" << fsId_ \ + << " to availableGroups:" << GROUP.DebugString(); \ + availableGroups_.emplace(GROUP.offset(), std::move(copy)); \ + } while (0) + + SpaceErrCode VolumeSpace::ReleaseBlockGroups( const std::vector& blockGroups) { LockGuard lk(mtx_); for (auto& group : blockGroups) { + VLOG(3) << "VolumeSpace fsid=" << fsId_ + << ", need release block group:" << group.DebugString(); auto it = allocatedGroups_.find(group.offset()); if (it != allocatedGroups_.end()) { if (it->second.owner() != group.owner()) { LOG(WARNING) << "Owner is not identical, block group may " "assign to others, fsId: " - << fsId_ << ", block group offset: " << group.offset(); + << fsId_ << ", block group offset: " << group.offset() + << ", record owner: " << it->second.owner() + << ", report owner: " << group.owner(); return SpaceErrConflict; } @@ -337,30 +420,49 @@ SpaceErrCode VolumeSpace::ReleaseBlockGroups( } cleanGroups_.insert(group.offset()); + VLOG(6) << "VolumeSpace fsid=" << fsId_ + << " return block group to cleanGroups:" + << group.DebugString(); } else { - auto copy = group; - copy.clear_owner(); - auto err = PersistBlockGroup(copy); - if (err != SpaceOk) { - LOG(WARNING) - << "Persist block group failed, fsId: " << fsId_ - << ", block group offset: " << group.offset() - << ", err: " << SpaceErrCode_Name(err); - return err; - } - - availableGroups_.emplace(group.offset(), std::move(copy)); + RELEASE_TO_AVAILABLE_GROUPS(group); } allocatedGroups_.erase(group.offset()); + VLOG(6) << "VolumeSpace fsid=" << fsId_ + << " erase block group from allocatedGroups:" + << group.DebugString(); + continue; } - + LOG(WARNING) << "VolumeSpace fsid=" << fsId_ + << " could not get release block gorup:" + << group.DebugString() << " in allocatedGroups_"; // and if it's not allocated, this request must be a retry request } return SpaceOk; } +SpaceErrCode VolumeSpace::ReleaseBlockGroups(const std::string &owner) { + LockGuard lk(mtx_); + + LOG(INFO) << "Release all block groups for " << owner << ", fsid=" << fsId_; + auto iter = allocatedGroups_.begin(); + while (iter != allocatedGroups_.end()) { + auto &group = iter->second; + if (group.owner() != owner) { + VLOG(9) << "VolumeSpace fsid=" << fsId_ << " expect owner:" << owner + << ", current block group:" << group.DebugString(); + iter++; + continue; + } + + RELEASE_TO_AVAILABLE_GROUPS(group); + iter = allocatedGroups_.erase(iter); + } + + return SpaceOk; +} + SpaceErrCode VolumeSpace::PersistBlockGroup(const BlockGroup& group) { return storage_->PutBlockGroup(fsId_, group.offset(), group); } @@ -447,6 +549,97 @@ void VolumeSpace::AddCleanGroups(uint64_t origin, uint64_t extended) { } } +void VolumeSpace::CalBlockGroupAvailableForDeAllocate() { + LockGuard lk(mtx_); + LockGuard statlk(statmtx_); + // check whether deallocatingGroups_ need move to availableGroups_ + auto iter = deallocatingGroups_.begin(); + while (iter != deallocatingGroups_.end()) { + if (iter->second.deallocating_size()) { + VLOG(6) << "VolumeSpace skip cal, fsId=" << fsId_ + << ", block group offset=" << iter->first + << " is under deallocating"; + + ++iter; + continue; + } + + assert(iter->second.deallocated_size() > 0); + + LOG(INFO) + << "VolumeSpace move deallocatingGroups_ to availableGroups_, " + "fsId=" + << fsId_ << ", block group offset=" << iter->first + << ", available size=" << availableGroups_.size(); + + iter->second.clear_deallocated(); + auto err = PersistBlockGroup(iter->second); + if (err != SpaceOk) { + LOG(ERROR) << "VolumeSpace put block group failed, fsId=" << fsId_ + << ", block group offset=" << iter->first + << ", err=" << SpaceErrCode_Name(err); + continue; + } + + iter->second.set_available(iter->second.size()); + availableGroups_.emplace(iter->first, std::move(iter->second)); + VLOG(9) << "VolumeSpace move deallocatingGroups_ to availableGroups_, " + "fsId=" + << fsId_ << ", block group offset=" << iter->first + << ", available size=" << availableGroups_.size(); + iter = deallocatingGroups_.erase(iter); + metric_.dealloc << 1; + } + + // check whether the cal conditions are met + if (!waitDeallocateGroups_.empty() || !deallocatingGroups_.empty() || + availableGroups_.empty() || summary_.empty()) { + VLOG(3) << "VolumeSpace wait for cal, " + "waitDeallocateGroups_ size=" + << waitDeallocateGroups_.size() + << ",deallocatingGroups_ size=" << deallocatingGroups_.size() + << ",availableGroups_ size=" << availableGroups_.size() + << ", allocatedGroups_ size=" << allocatedGroups_.size() + << ", cleanGroups_ size=" << cleanGroups_.size() + << ",summary_ size=" << summary_.size() + << ", fsid=" << fsId_; + return; + } + + // get the keys shared by availableGroups_ and summary_ + std::vector> commonKeys; + for (const auto &item : summary_) { + if (availableGroups_.count(item.first)) { + commonKeys.push_back(item); + } + } + + // sort + std::sort(commonKeys.begin(), commonKeys.end(), + [](const std::pair &a, + const std::pair &b) { + return a.second > b.second; + }); + + uint64_t size = (commonKeys.size() <= 1 ? 1 : commonKeys.size() / 2); + uint64_t selectKey = commonKeys[butil::fast_rand() % size].first; + LOG(INFO) << "VolumeSpace cal blockgroup=" << selectKey << ",fsid=" << fsId_ + << " wait for deallocate"; + + // move key from availableGroups_ to waitDeallocateGroups_ + BlockGroup selectGroup; + auto it = availableGroups_.find(selectKey); + if (it != availableGroups_.end()) { + assert(!it->second.has_owner()); + selectGroup = std::move(it->second); + selectGroup.clear_owner(); + availableGroups_.erase(it); + } + assert(waitDeallocateGroups_.count(selectKey) == 0); + waitDeallocateGroups_.emplace(selectKey, std::move(selectGroup)); + metric_.waitingDealloc << 1; +} + SpaceErrCode VolumeSpace::ExtendVolume() { if (!volume_.autoextend()) { LOG(WARNING) << "Auto extend is not supported, fsId: " << fsId_ @@ -459,34 +652,259 @@ SpaceErrCode VolumeSpace::ExtendVolume() { ExtendedSize(origin, volume_.extendfactor(), volume_.extendalignment()); LOG(INFO) << "Going to extend volume size from " << volume_.volumesize() - << " to " << extended; + << " to " << extended << ", fsid=" << fsId_; - auto* proxy = MdsProxyManager::GetInstance().GetOrCreateProxy( + auto *proxy = MdsProxyManager::GetInstance().GetOrCreateProxy( {volume_.cluster().begin(), volume_.cluster().end()}); if (proxy == nullptr) { - LOG(WARNING) << "Fail to get or create proxy"; + LOG(WARNING) << "Fail to get or create proxy, fsid=" << fsId_; return SpaceErrUnknown; } auto ret = proxy->ExtendVolume(volume_, extended); if (!ret) { - LOG(WARNING) << "Fail to extend volume"; + LOG(WARNING) << "Fail to extend volume, fsid=" << fsId_; return SpaceErrExtendVolumeError; } if (!UpdateFsInfo(origin, extended)) { - LOG(WARNING) << "Fail to update fs info"; + LOG(WARNING) << "Fail to update fs info, fsid=" << fsId_; return SpaceErrStorage; } volume_.set_volumesize(extended); AddCleanGroups(origin, extended); - LOG(INFO) << "Extended volume size from " << origin << " to " << extended; + LOG(INFO) << "Extended volume size from " << origin << " to " << extended + << ", fsid=" << fsId_; return SpaceOk; } +void VolumeSpace::Run() { + calThread_ = std::thread([this] { + while (sleeper_.wait_for(std::chrono::seconds(calcIntervalSec_))) { + CalBlockGroupAvailableForDeAllocate(); + } + }); + LOG(INFO) << "VolumeSpace start background, fsid=" << fsId_; +} + +void VolumeSpace::Stop() { + LOG(INFO) << "VolumeSpace stopping, fsid=" << fsId_; + + sleeper_.interrupt(); + if (calThread_.joinable()) { + calThread_.join(); + } + + LOG(INFO) << "VolumeSpace stopped, fsid=" << fsId_; +} + +bool VolumeSpace::UpdateDeallocatableBlockGroup( + uint32_t metaserverId, const DeallocatableBlockGroupVec &groups, + const BlockGroupDeallcateStatusMap &stats, uint64_t *issue) { + + UpdateBlockGroupDeallocatableSpace(metaserverId, groups); + + UpdateDeallocatingBlockGroup(metaserverId, stats); + + return SelectBlockGroupForDeAllocate(metaserverId, issue); +} + +void VolumeSpace::UpdateBlockGroupDeallocatableSpace( + uint32_t metaserverId, const DeallocatableBlockGroupVec &groups) { + LockGuard statlk(statmtx_); + VLOG(6) << "VolumeSpace update from metaserver:" << metaserverId + << ", fsId=" << fsId_ << ", groups size=" << groups.size(); + + // update summary_ with latest groups + std::unordered_map reportGroups; + for (auto &group : groups) { + auto offset = group.blockgroupoffset(); + auto deallocatableSize = group.deallocatablesize(); + reportGroups[offset] = deallocatableSize; + + auto iter = summary_.find(offset); + if (iter == summary_.end()) { + summary_.emplace(offset, deallocatableSize); + } else { + iter->second += group.deallocatablesize(); + } + VLOG(6) << "VolumeSpace update summary, fsId=" << fsId_ + << ", blockGroupOffset=" << offset + << ", deallocatableSize=" << group.deallocatablesize(); + } + + // remove groups from last round of reporting and record latest in + // lastUpdate_ + auto lastUpdateIter = lastUpdate_.find(metaserverId); + if (lastUpdateIter == lastUpdate_.end()) { + lastUpdate_[metaserverId] = groups; + } else { + for (auto &group : lastUpdateIter->second) { + auto offset = group.blockgroupoffset(); + auto lastDeallocatableSize = group.deallocatablesize(); + if (reportGroups.count(offset) == 0) { + continue; + } + + summary_[offset] -= lastDeallocatableSize; + if (summary_[offset] == 0) { + summary_.erase(offset); + LOG(INFO) << "VolumeSpace remove block group from summary, no " + "need deallocatable, " + "fsId=" + << fsId_ << ", blcokGroupOffset=" << offset; + } + + group.set_deallocatablesize(reportGroups[offset]); + } + } +} + +void VolumeSpace::UpdateDeallocatingBlockGroup( + uint32_t metaserverId, const BlockGroupDeallcateStatusMap &stats) { + LockGuard lk(mtx_); + + VLOG(6) << "VolumeSpace update deallocating block group from metaserver=" + << metaserverId << ", fsId=" << fsId_ + << ", stats size=" << stats.size(); + + // get completed deallocate blockgroup + std::vector doneGroups; + for (auto &stat : stats) { + auto offset = stat.first; + auto status = stat.second; + + auto iter = deallocatingGroups_.find(offset); + if (iter == deallocatingGroups_.end()) { + LOG(ERROR) << "VolumeSpace block group not found in " + "deallocatingGroups_, fsId=" + << fsId_ << ", blockGroupOffset=" << offset; + continue; + } + + VLOG(6) << "VolumeSpace get block group stat from metaserver=" + << metaserverId << ", fsId=" << fsId_ + << ", blockGroupOffset=" << offset + << ", status=" << BlockGroupDeallcateStatusCode_Name(status); + + auto alreadyDeallocated = + std::find(iter->second.deallocated().begin(), + iter->second.deallocated().end(), metaserverId); + if (status == BlockGroupDeallcateStatusCode::BGDP_DONE && + alreadyDeallocated == iter->second.deallocated().end()) { + doneGroups.emplace_back(offset); + LOG(INFO) << "VolumeSpace block group is deallocated done, fsId: " + << fsId_ << ", blcokGroupOffset: " << offset + << ", metaserverId: " << metaserverId; + } + } + + // update the metaserver from the deallocating state of the blockgroup to + // the deallocated state + for (auto offset : doneGroups) { + auto iter = deallocatingGroups_.find(offset); + assert(iter != deallocatingGroups_.end()); + + // update done metaserver to deallocated + iter->second.add_deallocated(metaserverId); + + // remove done metaserver from deallocating + auto mutableDeallocating = iter->second.mutable_deallocating(); + mutableDeallocating->erase(std::remove_if( + mutableDeallocating->begin(), + mutableDeallocating->end(), + [metaserverId](uint32_t id) { + return id == metaserverId; + }), + mutableDeallocating->end()); + + auto err = PersistBlockGroup(iter->second); + // TODO(wuhanqing): handle error, and rollback if necessary + if (err != SpaceOk) { + LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_ + << ", offset: " << offset + << ", err: " << SpaceErrCode_Name(err); + return; + } + } +} + +bool VolumeSpace::SelectBlockGroupForDeAllocate(uint32_t metaserverId, + uint64_t *issue) { + assert(issue != nullptr); + LockGuard lk(mtx_); + + // TODO(ilixiaocui): support more groups to be issued + VLOG(3) + << "VolumeSpace select block group to be deallocate for metaserverId=" + << metaserverId << ", fsId=" << fsId_ + << ", waitDeallocateGroups_size=" << waitDeallocateGroups_.size(); + + if (!waitDeallocateGroups_.empty()) { + auto iter = waitDeallocateGroups_.begin(); + auto offset = iter->first; + iter->second.add_deallocating(metaserverId); + deallocatingGroups_[offset] = std::move(iter->second); + waitDeallocateGroups_.erase(iter); + metric_.waitingDealloc << -1; + + auto err = PersistBlockGroup(deallocatingGroups_[offset]); + // TODO(wuhanqing): handle error, and rollback if necessary + if (err != SpaceOk) { + LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_ + << ", offset: " << offset + << ", err: " << SpaceErrCode_Name(err); + return false; + } + *issue = offset; + LOG(INFO) << "VolumeSpace issue block group from " + "waitDeallocateGroups_, fsId: " + << fsId_ << ", offset: " << offset + << ", to metaserverId: " << metaserverId; + return true; + } + + auto deallocatingOne = deallocatingGroups_.begin(); + if (deallocatingOne != deallocatingGroups_.end()) { + *issue = deallocatingOne->first; + + auto alreadyIssued = std::find( + deallocatingOne->second.deallocating().begin(), + deallocatingOne->second.deallocating().end(), metaserverId); + if (alreadyIssued != deallocatingOne->second.deallocating().end()) { + return true; + } + + auto alreadyDone = std::find( + deallocatingOne->second.deallocated().begin(), + deallocatingOne->second.deallocated().end(), metaserverId); + if (alreadyDone != deallocatingOne->second.deallocated().end()) { + return false; + } + + deallocatingOne->second.add_deallocating(metaserverId); + auto err = PersistBlockGroup(deallocatingOne->second); + // TODO(wuhanqing): handle error, and rollback if necessary + if (err != SpaceOk) { + LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_ + << ", offset: " << *issue + << ", err: " << SpaceErrCode_Name(err); + return false; + } + + LOG(INFO) << "VolumeSpace issue block group from " + "deallocatingGroups_, fsId: " + << fsId_ << ", offset: " << *issue + << ", to metaserverId: " << metaserverId; + return true; + } + + return false; +} + uint64_t ExtendedSize(uint64_t origin, double factor, uint64_t alignment) { return common::align_up( static_cast(std::floor(static_cast(origin) * factor)), diff --git a/curvefs/src/mds/space/volume_space.h b/curvefs/src/mds/space/volume_space.h index 95f0954999..504c05de7d 100644 --- a/curvefs/src/mds/space/volume_space.h +++ b/curvefs/src/mds/space/volume_space.h @@ -36,13 +36,26 @@ #include "curvefs/proto/common.pb.h" #include "curvefs/proto/space.pb.h" +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/proto/heartbeat.pb.h" #include "curvefs/src/mds/fs_storage.h" #include "curvefs/src/mds/space/block_group_storage.h" +#include "src/common/interruptible_sleeper.h" namespace curvefs { namespace mds { namespace space { +using ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode; +using ::curvefs::metaserver::DeallocatableBlockGroup; +using ::curve::common::InterruptibleSleeper; + +using DeallocatableBlockGroupVec = + google::protobuf::RepeatedPtrField; +using BlockGroupDeallcateStatusMap = ::google::protobuf::Map< + ::google::protobuf::uint64, + ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode>; + class AbstractVolumeSpace { public: virtual ~AbstractVolumeSpace() = default; @@ -56,8 +69,14 @@ class AbstractVolumeSpace { const std::string& owner, BlockGroup* group) = 0; - virtual SpaceErrCode ReleaseBlockGroups( - const std::vector& blockGroups) = 0; + virtual SpaceErrCode + ReleaseBlockGroups(const std::vector &blockGroups) = 0; + + virtual SpaceErrCode ReleaseBlockGroups(const std::string &owner) = 0; + + virtual bool UpdateDeallocatableBlockGroup( + uint32_t metaserverId, const DeallocatableBlockGroupVec &groups, + const BlockGroupDeallcateStatusMap &stats, uint64_t *issue) = 0; }; using ::curvefs::common::BitmapLocation; @@ -65,13 +84,13 @@ using ::curvefs::common::Volume; class VolumeSpace final : public AbstractVolumeSpace { public: - static std::unique_ptr Create(uint32_t fsId, - const Volume& volume, - BlockGroupStorage* storage, - FsStorage* fsStorage); + static std::unique_ptr + Create(uint32_t fsId, const Volume &volume, BlockGroupStorage *storage, + FsStorage *fsStorage, uint64_t calcIntervalSec); VolumeSpace(const VolumeSpace&) = delete; VolumeSpace& operator=(const VolumeSpace&) = delete; + ~VolumeSpace() { Stop(); } /** * @brief Allocate block groups @@ -94,6 +113,11 @@ class VolumeSpace final : public AbstractVolumeSpace { SpaceErrCode ReleaseBlockGroups( const std::vector& blockGroups) override; + /** + * @brief Release block groups by owner + */ + SpaceErrCode ReleaseBlockGroups(const std::string &owner) override; + /** * @brief Remove all block groups and persistent records that belong to * current volume @@ -101,11 +125,21 @@ class VolumeSpace final : public AbstractVolumeSpace { */ SpaceErrCode RemoveAllBlockGroups(); + + bool UpdateDeallocatableBlockGroup( + uint32_t metaserverId, const DeallocatableBlockGroupVec &groups, + const BlockGroupDeallcateStatusMap &stats, uint64_t *issue); + private: - VolumeSpace(uint32_t fsId, - Volume volume, - BlockGroupStorage* storage, - FsStorage* fsStorage); + VolumeSpace(uint32_t fsId, Volume volume, BlockGroupStorage *storage, + FsStorage *fsStorage); + + /** + * @brief Calculate block group that can be recycled + */ + void Run(); + + void Stop(); private: SpaceErrCode AllocateBlockGroupsInternal( @@ -131,6 +165,24 @@ class VolumeSpace final : public AbstractVolumeSpace { void AddCleanGroups(uint64_t origin, uint64_t extended); + // pick out blockgroups that can be authorized for metaserver processing + void CalBlockGroupAvailableForDeAllocate(); + + // update the deallocatable space of blockgroup + void UpdateBlockGroupDeallocatableSpace( + uint32_t metaserverId, const DeallocatableBlockGroupVec &groups); + + // update the deallocating progress reported by the metaserver + void + UpdateDeallocatingBlockGroup(uint32_t metaserverId, + const BlockGroupDeallcateStatusMap &stats); + + // check whether there is currently a blockgroup that can be recycled and + // send it to the metaserver + bool SelectBlockGroupForDeAllocate(uint32_t metaserverId, uint64_t *issue); + + FRIEND_TEST(VolumeSpaceTest, Test_CalBlockGroupAvailableForDeAllocate); + private: // persist block group to backend storage SpaceErrCode PersistBlockGroup(const BlockGroup& group); @@ -164,6 +216,13 @@ class VolumeSpace final : public AbstractVolumeSpace { // 3. clean // these block groups' space is never used, and they can be allocated to // other clients. + // 4. waitDeallocate + // these block groups wait for being deallocated by + // metaservers, and they can not allocate to clients. + // 5. deallocating + // these block groups are being deallocated by metaservers, and they can + // not allocate to clients. + // these block groups are not persisted into storage. // key is block group offset std::unordered_map allocatedGroups_; @@ -174,9 +233,51 @@ class VolumeSpace final : public AbstractVolumeSpace { // stores clean block groups' offset std::unordered_set cleanGroups_; + // key is block group offset + std::unordered_map waitDeallocateGroups_; + + // key is block group offset + std::unordered_map deallocatingGroups_; + + mutable bthread::Mutex statmtx_; + // Summarize the deallocatable space of the blockgroup in this volume + // reported by all metaservers + // + // The metaserver reports the full amount of information instead of + // incremental information, so adding this data to the statistical data + // requires subtracting the last data + // + // - lastupdate_ + // metaserver report last time + // - summary_ + // summary add metaserver report info and remove lastupdate info + + // key is metaserver id + std::unordered_map lastUpdate_; + + // key is block group offset, value is deallocatable size + std::unordered_map summary_; + BlockGroupStorage* storage_; FsStorage* fsStorage_; + + // TODO(@ilixiaocui): may need use thread pool, if there are many volumes + std::thread calThread_; + InterruptibleSleeper sleeper_; + int64_t calcIntervalSec_; + + private: + struct Metric { + bvar::Adder dealloc; + bvar::Adder waitingDealloc; + + Metric() + : dealloc("mds_volume_space_dealloc"), + waitingDealloc("mds_volume_space_wait_dealloc") {} + }; + + Metric metric_; }; // Calculate extended size based on origin with factor, and the result size is diff --git a/curvefs/src/metaserver/BUILD b/curvefs/src/metaserver/BUILD index e390509a44..04c90247dd 100644 --- a/curvefs/src/metaserver/BUILD +++ b/curvefs/src/metaserver/BUILD @@ -29,6 +29,10 @@ cc_library( ["storage/*.cpp"], ) + glob( ["streaming/*.cpp"], + ) + glob( + ["space/*.cpp"], + ) + glob( + ["mds/*.cpp"], ), hdrs = glob( ["*.h"], @@ -38,6 +42,10 @@ cc_library( ["storage/*.h"], ) + glob( ["streaming/*.h"], + ) + glob( + ["space/*.h"], + ) + glob( + ["mds/*.h"], ), copts = CURVE_DEFAULT_COPTS, visibility = ["//visibility:public"], @@ -51,6 +59,8 @@ cc_library( "//curvefs/proto:mds_cc_proto", "//curvefs/src/common:curvefs_common", "//curvefs/src/metaserver/common:fs_metaserver_common", + "//curvefs/src/volume:volume", + "//curvefs/src/client:fuse_client_lib", "//external:braft", "//src/common:curve_common", "//src/fs:lfs", @@ -61,6 +71,7 @@ cc_library( "@com_google_absl//absl/container:btree", "@com_google_absl//absl/memory", "@com_google_absl//absl/utility", + "@com_google_protobuf//:protobuf", "//external:gtest", ], ) diff --git a/curvefs/src/metaserver/common/types.h b/curvefs/src/metaserver/common/types.h index 614bab60bc..076fd74fee 100644 --- a/curvefs/src/metaserver/common/types.h +++ b/curvefs/src/metaserver/common/types.h @@ -24,6 +24,9 @@ #define CURVEFS_SRC_METASERVER_COMMON_TYPES_H_ #include +#include +#include +#include namespace curvefs { namespace metaserver { @@ -32,6 +35,14 @@ using PoolId = uint32_t; using CopysetId = uint32_t; using PartitionId = uint32_t; +inline std::string StringToHex(const std::string &str) { + std::stringstream ss; + ss << std::hex << std::setfill('0'); + for (const auto &c : str) { + ss << std::setw(2) << static_cast(static_cast(c)); + } + return ss.str(); +} } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 3360bc2b4c..4b11c268b9 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include "absl/cleanup/cleanup.h" #include "absl/memory/memory.h" @@ -630,24 +632,157 @@ void CopysetNode::ListPeers(std::vector* peers) const { } // if copyset is loading, return false; -// if copyset is not loading, and metastore returns fales, retry; +// if copyset is not loading, and metastore returns false, retry; // if copyset is not loading, and metastore returns true, return true and get // partition info list success. bool CopysetNode::GetPartitionInfoList( - std::list *partitionInfoList) { + std::list *partitionInfoList) { + if (IsLoading()) { + LOG(INFO) << "Copyset is loading, return empty partition list"; + return false; + } + uint32_t retryCount = 0; - while (true) { - if (IsLoading()) { - LOG(INFO) << "Copyset is loading, return empty partition list"; - return false; - } + do { bool ret = metaStore_->GetPartitionInfoList(partitionInfoList); - if (ret) { - return true; + if (!ret) { + LOG(WARNING) + << "Copyset is not loading, but GetPartitionInfoList fail," + << " retryCount = " << retryCount++; + continue; + } + + return true; + } while (true); +} + +bool CopysetNode::GetBlockStatInfo( + std::map *blockStatInfoMap) { + if (IsLoading()) { + LOG(INFO) + << "CopysetNode copyset is loading, return empty block stat info"; + return false; + } + + uint32_t retryCount = 0; + uint32_t blockGroupNum = 0; + do { + std::map> partitionSnap; + if (!metaStore_->GetPartitionSnap(&partitionSnap)) { + LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = " + << ++retryCount; + continue; + } + + for (const auto &item : partitionSnap) { + auto partition = item.second; + VLOG(6) << "CopysetNode get block stat info from partition=" + << partition->GetPartitionId() + << ", fsId=" << partition->GetFsId(); + if (!AggregateBlockStatInfo(partition, blockStatInfoMap, + &blockGroupNum)) { + continue; + } } - LOG(WARNING) << "Copyset is not loading, but GetPartitionInfoList fail," - << " retryCount = " << retryCount++; + + return true; + } while (true); +} + +// NOTE: +// If the file system size is 1PB, the block size is 128MB +// - The maximum number of blockgroups in the file system is 838,8608. +// +// The deallocatable space of the current blockgroup is counted according to +// the granularity of the partition. Now the maximum number of partitions on +// each copyset is configured to 128 +// - In extreme cases, the number of statistical fragments of blockgroups in +// each copyset is 838,8608*128≈10billion.The number of fragments of the +// blockgroup that metaserver needs to carry in a heartbeat will be too large +// +// Therefore, it is necessary to combine the information of these fragments to +// calculate. +// +// TODO(ilixiaocui): need more stat optimization +bool CopysetNode::AggregateBlockStatInfo( + const std::shared_ptr &partition, + std::map *blockStatInfoMap, + uint32_t *blockGroupNum) { + uint32_t fsId = partition->GetFsId(); + uint64_t partitionId = partition->GetPartitionId(); + + // get block group info in partition + std::vector deallocatableBlockGroupVec; + if (MetaStatusCode::OK != + partition->GetAllBlockGroup(&deallocatableBlockGroupVec)) { + LOG(WARNING) << "CopysetNode get all blockgroup fail, partitionId= " + << partitionId << ", fsId=" << fsId; + return false; } + + auto &blockGroupStatInfo = (*blockStatInfoMap)[fsId]; + if (!blockGroupStatInfo.has_fsid()) { + blockGroupStatInfo.set_fsid(fsId); + } + +#define LIMITBLICKGROUPNUM 8192 + // combine the information of the same blockgroup + auto blockGroups = blockGroupStatInfo.mutable_deallocatableblockgroups(); + std::unordered_map blockGroupMap; + for (auto &blockGroup : *blockGroups) { + blockGroupMap[blockGroup.blockgroupoffset()] = &blockGroup; + } + + for (auto &blockGroup : deallocatableBlockGroupVec) { + auto it = blockGroupMap.find(blockGroup.blockgroupoffset()); + if (it != blockGroupMap.end()) { + it->second->set_deallocatablesize(it->second->deallocatablesize() + + blockGroup.deallocatablesize()); + } else { + blockGroup.clear_inodeidlist(); + blockGroup.clear_inodeidunderdeallocate(); + blockGroups->Add()->MergeFrom(blockGroup); + if (++(*blockGroupNum) > LIMITBLICKGROUPNUM) { + LOG(WARNING) << "CopysetNode get blockgroup num over limit, " + "blockGroupNum = " + << *blockGroupNum; + break; + } + } + VLOG(6) << "CopysetNode get block group info, fsId = " << fsId + << ", block info:" << blockGroup.DebugString(); + } + + VLOG(6) << "CopysetNode get block group num:" << *blockGroupNum; + + return true; +} + +void CopysetNode::Deallocate(uint64_t fsId, uint64_t blockGroupOffset) { + if (IsLoading()) { + LOG(INFO) + << "CopysetNode copyset is loading, return empty block stat info"; + return; + } + + uint32_t retryCount = 0; + do { + std::map> partitionSnap; + if (!metaStore_->GetPartitionSnap(&partitionSnap)) { + LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = " + << ++retryCount; + continue; + } + + for (const auto &item : partitionSnap) { + auto partition = item.second; + if (partition->GetFsId() != fsId) { + continue; + } + item.second->SetVolumeDeallocate(fsId, blockGroupOffset); + } + return; + } while (true); } void CopysetNode::OnConfChangeComplete() { diff --git a/curvefs/src/metaserver/copyset/copyset_node.h b/curvefs/src/metaserver/copyset/copyset_node.h index d37c07b0bb..c971a0d569 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.h +++ b/curvefs/src/metaserver/copyset/copyset_node.h @@ -24,11 +24,13 @@ #define CURVEFS_SRC_METASERVER_COPYSET_COPYSET_NODE_H_ #include +#include #include #include #include #include +#include #include "curvefs/src/metaserver/common/types.h" #include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h" @@ -38,6 +40,7 @@ #include "curvefs/src/metaserver/copyset/metric.h" #include "curvefs/src/metaserver/copyset/raft_node.h" #include "curvefs/src/metaserver/metastore.h" +#include "curvefs/proto/heartbeat.pb.h" namespace curvefs { namespace metaserver { @@ -47,6 +50,7 @@ using ::braft::PeerId; using ::curvefs::common::Peer; using ::curvefs::metaserver::MetaStore; using ::curve::mds::heartbeat::ConfigChangeType; +using ::curvefs::mds::heartbeat::BlockGroupStatInfo; class CopysetNodeManager; @@ -89,13 +93,13 @@ class CopysetNode : public braft::StateMachine { */ virtual bool IsLeaseExpired(const braft::LeaderLeaseStatus &lease_status) const; // NOLINT - PoolId GetPoolId() const; + virtual PoolId GetPoolId() const; - const braft::PeerId& GetPeerId() const; + virtual const braft::PeerId& GetPeerId() const; - CopysetId GetCopysetId() const; + virtual CopysetId GetCopysetId() const; - PeerId GetLeaderId() const; + virtual PeerId GetLeaderId() const; MetaStore* GetMetaStore() const; @@ -154,7 +158,7 @@ class CopysetNode : public braft::StateMachine { virtual void RemovePeer(const Peer& peer, braft::Closure* done = nullptr); virtual void ChangePeers(const std::vector& newPeers, braft::Closure* done = nullptr); - void GetConfChange(ConfigChangeType* type, Peer* alterPeer); + virtual void GetConfChange(ConfigChangeType* type, Peer* alterPeer); void OnConfChangeComplete(); private: @@ -193,9 +197,15 @@ class CopysetNode : public braft::StateMachine { public: // for heartbeat - bool GetPartitionInfoList(std::list *partitionInfoList); + virtual bool + GetPartitionInfoList(std::list *partitionInfoList); - bool IsLoading() const; + virtual bool IsLoading() const; + + virtual bool + GetBlockStatInfo(std::map *blockStatInfoMap); + + virtual void Deallocate(uint64_t fsId, uint64_t blockGroupOffset); private: void InitRaftNodeOptions(); @@ -203,6 +213,13 @@ class CopysetNode : public braft::StateMachine { bool FetchLeaderStatus(const braft::PeerId& peerId, braft::NodeStatus* leaderStatus); + bool AggregateBlockStatInfo( + const std::shared_ptr &partition, + std::map *blockStatInfoMap, + uint32_t *blockGroupNum); + + FRIEND_TEST(CopysetNodeBlockGroupTest, Test_AggregateBlockStatInfo); + private: const PoolId poolId_; const CopysetId copysetId_; diff --git a/curvefs/src/metaserver/copyset/copyset_node_manager.cpp b/curvefs/src/metaserver/copyset/copyset_node_manager.cpp index e6d492f88b..a3ceeb5b04 100644 --- a/curvefs/src/metaserver/copyset/copyset_node_manager.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node_manager.cpp @@ -246,7 +246,7 @@ bool CopysetNodeManager::CreateCopysetNode(PoolId poolId, CopysetId copysetId, } void CopysetNodeManager::GetAllCopysets( - std::vector* nodes) const { + std::vector *nodes) const { nodes->clear(); ReadLockGuard lock(lock_); for (auto& copyset : copysets_) { diff --git a/curvefs/src/metaserver/copyset/copyset_node_manager.h b/curvefs/src/metaserver/copyset/copyset_node_manager.h index 12e838ac9f..41b58dde67 100644 --- a/curvefs/src/metaserver/copyset/copyset_node_manager.h +++ b/curvefs/src/metaserver/copyset/copyset_node_manager.h @@ -74,7 +74,7 @@ class CopysetNodeManager { virtual bool PurgeCopysetNode(PoolId poolId, CopysetId copysetId); - void GetAllCopysets(std::vector* nodes) const; + virtual void GetAllCopysets(std::vector *nodes) const; virtual bool IsLoadFinished() const; diff --git a/curvefs/src/metaserver/copyset/meta_operator.cpp b/curvefs/src/metaserver/copyset/meta_operator.cpp index 12affc5458..82fdf802bc 100644 --- a/curvefs/src/metaserver/copyset/meta_operator.cpp +++ b/curvefs/src/metaserver/copyset/meta_operator.cpp @@ -187,6 +187,7 @@ OPERATOR_ON_APPLY(CreatePartition); OPERATOR_ON_APPLY(DeletePartition); OPERATOR_ON_APPLY(PrepareRenameTx); OPERATOR_ON_APPLY(UpdateVolumeExtent); +OPERATOR_ON_APPLY(UpdateDeallocatableBlockGroup); #undef OPERATOR_ON_APPLY @@ -265,7 +266,7 @@ void GetVolumeExtentOperator::OnApply(int64_t index, } // in streaming mode, swap slices out and send them by streaming - VolumeExtentList extents; + VolumeExtentSliceList extents; response->mutable_slices()->Swap(&extents); response->clear_slices(); @@ -312,6 +313,7 @@ OPERATOR_ON_APPLY_FROM_LOG(CreatePartition); OPERATOR_ON_APPLY_FROM_LOG(DeletePartition); OPERATOR_ON_APPLY_FROM_LOG(PrepareRenameTx); OPERATOR_ON_APPLY_FROM_LOG(UpdateVolumeExtent); +OPERATOR_ON_APPLY_FROM_LOG(UpdateDeallocatableBlockGroup); #undef OPERATOR_ON_APPLY_FROM_LOG @@ -370,6 +372,7 @@ OPERATOR_REDIRECT(DeletePartition); OPERATOR_REDIRECT(PrepareRenameTx); OPERATOR_REDIRECT(GetVolumeExtent); OPERATOR_REDIRECT(UpdateVolumeExtent); +OPERATOR_REDIRECT(UpdateDeallocatableBlockGroup); #undef OPERATOR_REDIRECT @@ -396,6 +399,7 @@ OPERATOR_ON_FAILED(DeletePartition); OPERATOR_ON_FAILED(PrepareRenameTx); OPERATOR_ON_FAILED(GetVolumeExtent); OPERATOR_ON_FAILED(UpdateVolumeExtent); +OPERATOR_ON_FAILED(UpdateDeallocatableBlockGroup); #undef OPERATOR_ON_FAILED @@ -421,6 +425,8 @@ OPERATOR_HASH_CODE(PrepareRenameTx); OPERATOR_HASH_CODE(DeletePartition); OPERATOR_HASH_CODE(GetVolumeExtent); OPERATOR_HASH_CODE(UpdateVolumeExtent); +OPERATOR_HASH_CODE(UpdateDeallocatableBlockGroup); + #undef OPERATOR_HASH_CODE @@ -458,6 +464,7 @@ OPERATOR_TYPE(CreatePartition); OPERATOR_TYPE(DeletePartition); OPERATOR_TYPE(GetVolumeExtent); OPERATOR_TYPE(UpdateVolumeExtent); +OPERATOR_TYPE(UpdateDeallocatableBlockGroup); #undef OPERATOR_TYPE diff --git a/curvefs/src/metaserver/copyset/meta_operator.h b/curvefs/src/metaserver/copyset/meta_operator.h index f5e21f097a..dca819f9e8 100644 --- a/curvefs/src/metaserver/copyset/meta_operator.h +++ b/curvefs/src/metaserver/copyset/meta_operator.h @@ -537,6 +537,25 @@ class UpdateVolumeExtentOperator : public MetaOperator { void OnFailed(MetaStatusCode code) override; }; +class UpdateDeallocatableBlockGroupOperator : public MetaOperator { + public: + using MetaOperator::MetaOperator; + + void OnApply(int64_t index, google::protobuf::Closure *done, + uint64_t startTimeUs) override; + + void OnApplyFromLog(uint64_t startTimeUs) override; + + uint64_t HashCode() const override; + + OperatorType GetOperatorType() const override; + + private: + void Redirect() override; + + void OnFailed(MetaStatusCode code) override; +}; + } // namespace copyset } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/copyset/operator_type.cpp b/curvefs/src/metaserver/copyset/operator_type.cpp index e29286772a..adcee67671 100644 --- a/curvefs/src/metaserver/copyset/operator_type.cpp +++ b/curvefs/src/metaserver/copyset/operator_type.cpp @@ -66,6 +66,8 @@ const char* OperatorTypeName(OperatorType type) { return "GetVolumeExtent"; case OperatorType::UpdateVolumeExtent: return "UpdateVolumeExtent"; + case OperatorType::UpdateDeallocatableBlockGroup: + return "UpdateDeallocatableBlockGroup"; // Add new case before `OperatorType::OperatorTypeMax` case OperatorType::OperatorTypeMax: break; diff --git a/curvefs/src/metaserver/copyset/operator_type.h b/curvefs/src/metaserver/copyset/operator_type.h index bdd905c158..c2e54c44bf 100644 --- a/curvefs/src/metaserver/copyset/operator_type.h +++ b/curvefs/src/metaserver/copyset/operator_type.h @@ -52,6 +52,8 @@ enum class OperatorType : uint32_t { GetVolumeExtent = 15, UpdateVolumeExtent = 16, CreateManageInode = 17, + UpdateDeallocatableBlockGroup = 18, + // NOTE: // Add new operator before `OperatorTypeMax` // And DO NOT recorder or delete previous types diff --git a/curvefs/src/metaserver/copyset/raft_log_codec.cpp b/curvefs/src/metaserver/copyset/raft_log_codec.cpp index 5bed069313..d70799f25c 100644 --- a/curvefs/src/metaserver/copyset/raft_log_codec.cpp +++ b/curvefs/src/metaserver/copyset/raft_log_codec.cpp @@ -165,6 +165,10 @@ std::unique_ptr RaftLogCodec::Decode(CopysetNode* node, return ParseFromRaftLog(node, type, meta); + case OperatorType::UpdateDeallocatableBlockGroup: + return ParseFromRaftLog( + node, type, meta); // Add new case before `OperatorType::OperatorTypeMax` case OperatorType::OperatorTypeMax: break; diff --git a/curvefs/src/metaserver/heartbeat.cpp b/curvefs/src/metaserver/heartbeat.cpp index de42a148d9..0c2bba9a93 100644 --- a/curvefs/src/metaserver/heartbeat.cpp +++ b/curvefs/src/metaserver/heartbeat.cpp @@ -40,19 +40,21 @@ #include "curvefs/src/metaserver/copyset/utils.h" #include "curvefs/src/metaserver/storage/storage.h" #include "curvefs/src/metaserver/resource_statistic.h" +#include "curvefs/src/metaserver/space/volume_deallocate_manager.h" namespace curvefs { namespace metaserver { using ::curve::mds::heartbeat::ConfigChangeType; using ::curvefs::mds::heartbeat::ConfigChangeInfo; +using ::curvefs::mds::heartbeat::CopySetInfo; using ::curvefs::metaserver::copyset::CopysetService_Stub; using ::curvefs::metaserver::copyset::ToGroupIdString; namespace { int GatherCopysetConfChange(CopysetNode* node, ConfigChangeInfo* info) { - ConfigChangeType type; + ConfigChangeType type = ConfigChangeType::NONE; Peer peer; node->GetConfChange(&type, &peer); @@ -161,7 +163,7 @@ int Heartbeat::Fini() { return 0; } -void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info, +void Heartbeat::BuildCopysetInfo(CopySetInfo *info, CopysetNode *copyset) { int ret; PoolId poolId = copyset->GetPoolId(); @@ -213,6 +215,12 @@ void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info, } } +void Heartbeat::BuildBlockGroupStatInfo(CopysetNode *copyset, + BlockGroupStatInfoMap *blockGroupStatInfoMap) { + copyset->GetBlockStatInfo(blockGroupStatInfoMap); +} + + // TODO(@Wine93): now we use memory storage, so we gather disk usage bytes // which only has raft related capacity. If we use rocksdb storage, maybe // we should need more flexible strategy. @@ -255,16 +263,50 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) { req->set_copysetcount(copysets.size()); int leaders = 0; - + BlockGroupStatInfoMap blockGroupStatInfoMap; for (auto copyset : copysets) { - curvefs::mds::heartbeat::CopySetInfo *info = req->add_copysetinfos(); - + // build copyset info + CopySetInfo *info = req->add_copysetinfos(); BuildCopysetInfo(info, copyset); if (copyset->IsLeaderTerm()) { + // build block group info + VLOG(6) << "Heartbeat build block group stat info for copyset:" + << info->DebugString(); + BuildBlockGroupStatInfo(copyset, &blockGroupStatInfoMap); ++leaders; } } + + // get deallocate task status + for (auto &item : blockGroupStatInfoMap) { + auto iterTask = taskExecutor_->deallocTask_.find(item.first); + if (iterTask != taskExecutor_->deallocTask_.end()) { + uint32_t fsId = iterTask->first; + uint64_t blockGroupOffset = iterTask->second; + + bool doing = VolumeDeallocateManager::GetInstance().HasDeallocate(); + auto status = item.second.mutable_blockgroupdeallocatestatus(); + if (doing) { + status->insert( + {blockGroupOffset, + BlockGroupDeallcateStatusCode::BGDP_PROCESSING}); + } else { + status->insert({blockGroupOffset, + BlockGroupDeallcateStatusCode::BGDP_DONE}); + taskExecutor_->deallocTask_.erase(iterTask); + } + + VLOG(6) << "Heartbeat find fsId=" << fsId + << " in deallocTask, blockgroupOffset=" << blockGroupOffset + << ", status=" << status; + } + + VLOG(6) << "Heartbeat find fsId=" << item.first + << " not in deallocTask"; + *req->add_blockgroupstatinfos() = std::move(item.second); + } + req->set_leadercount(leaders); MetaServerSpaceStatus* status = req->mutable_spacestatus(); @@ -277,21 +319,7 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) { } void Heartbeat::DumpHeartbeatRequest(const HeartbeatRequest& request) { - VLOG(6) << "Heartbeat request: Metaserver ID: " << request.metaserverid() - << ", IP = " << request.ip() << ", port = " << request.port() - << ", copyset count = " << request.copysetcount() - << ", leader count = " << request.leadercount() - << ", diskThresholdByte = " - << request.spacestatus().diskthresholdbyte() - << ", diskCopysetMinRequireByte = " - << request.spacestatus().diskcopysetminrequirebyte() - << ", diskUsedByte = " - << request.spacestatus().diskusedbyte() - << ", memoryThresholdByte = " - << request.spacestatus().memorythresholdbyte() - << ", memoryCopySetMinRequireByte = " - << request.spacestatus().memorycopysetminrequirebyte() - << ", memoryUsedByte = " << request.spacestatus().memoryusedbyte(); + VLOG(6) << "Heartbeat reuqest: " << request.DebugString(); for (int i = 0; i < request.copysetinfos_size(); i++) { const curvefs::mds::heartbeat::CopySetInfo &info = @@ -314,9 +342,14 @@ void Heartbeat::DumpHeartbeatResponse(const HeartbeatResponse &response) { VLOG(3) << "Received heartbeat response, statusCode = " << response.statuscode(); - for (auto& conf : response.needupdatecopysets()) { + for (auto &conf : response.needupdatecopysets()) { VLOG(3) << "need update copyset: " << conf.ShortDebugString(); } + + for (auto &issue : response.issuedblockgroups()) { + VLOG(3) << "issued block group fsid=" << issue.first + << ", blockgroupoffset=" << issue.second; + } } int Heartbeat::SendHeartbeat(const HeartbeatRequest &request, @@ -406,10 +439,33 @@ HeartbeatTaskExecutor::HeartbeatTaskExecutor(CopysetNodeManager* mgr, const butil::EndPoint& endpoint) : copysetMgr_(mgr), ep_(endpoint) {} -void HeartbeatTaskExecutor::ExecTasks(const HeartbeatResponse& response) { - for (auto& conf : response.needupdatecopysets()) { +void HeartbeatTaskExecutor::ExecTasks(const HeartbeatResponse &response) { + for (auto &conf : response.needupdatecopysets()) { ExecOneTask(conf); } + + std::vector copysets; + copysetMgr_->GetAllCopysets(©sets); + for (auto &issue : response.issuedblockgroups()) { + auto iter = deallocTask_.find(issue.first); + if (iter != deallocTask_.end()) { + VLOG(6) << "HeartbeatTaskExecutor dealloc task fsid=" << issue.first + << ", blockgroupoffset=" << issue.second << " is excuting"; + assert(iter->second == issue.second); + continue; + } + + for (auto ©set : copysets) { + if (copyset->IsLeaderTerm()) { + copyset->Deallocate(issue.first, issue.second); + VLOG(6) << "HeartbeatTaskExecutor issue dealloc task fsid=" + << issue.first << ", blockgroupoffset=" << issue.second + << " to copyset " << copyset->Name(); + } + } + + deallocTask_.emplace(issue.first, issue.second); + } } void HeartbeatTaskExecutor::ExecOneTask(const CopySetConf& conf) { diff --git a/curvefs/src/metaserver/heartbeat.h b/curvefs/src/metaserver/heartbeat.h index fbcd8dbe5f..2e07cb804a 100644 --- a/curvefs/src/metaserver/heartbeat.h +++ b/curvefs/src/metaserver/heartbeat.h @@ -25,6 +25,7 @@ #include // NodeImpl #include +#include #include #include @@ -37,23 +38,26 @@ #include "curvefs/src/metaserver/copyset/copyset_node_manager.h" #include "src/common/concurrent/concurrent.h" #include "src/common/wait_interval.h" - -using ::curve::common::Thread; -using ::curvefs::metaserver::copyset::CopysetNode; +#include "absl/types/optional.h" namespace curvefs { namespace metaserver { +using curve::common::Thread; +using curve::fs::LocalFileSystem; +using curvefs::common::Peer; +using curvefs::mds::heartbeat::BlockGroupStatInfo; +using curvefs::mds::heartbeat::CopySetConf; +using curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode; +using curvefs::metaserver::copyset::CopysetNode; +using curvefs::metaserver::copyset::CopysetNodeManager; -using ::curve::fs::LocalFileSystem; using HeartbeatRequest = curvefs::mds::heartbeat::MetaServerHeartbeatRequest; using HeartbeatResponse = curvefs::mds::heartbeat::MetaServerHeartbeatResponse; using MetaServerSpaceStatus = curvefs::mds::heartbeat::MetaServerSpaceStatus; -using ::curvefs::mds::heartbeat::CopySetConf; -using TaskStatus = butil::Status; using CopysetNodePtr = std::shared_ptr; -using curvefs::metaserver::copyset::CopysetNodeManager; -using curvefs::common::Peer; using PeerId = braft::PeerId; +using TaskStatus = butil::Status; +using BlockGroupStatInfoMap = std::map; class ResourceCollector; @@ -104,6 +108,8 @@ class Heartbeat { int Run(); private: + FRIEND_TEST(HeartbeatTest, Test_BuildRequest); + /** * @brief stop heartbeat subsystem * @return 0:success; not 0: fail @@ -118,6 +124,9 @@ class Heartbeat { void BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo* info, CopysetNode* copyset); + void BuildBlockGroupStatInfo(CopysetNode *copyset, + BlockGroupStatInfoMap *blockGroupStatInfoMap); + int BuildRequest(HeartbeatRequest* request); int SendHeartbeat(const HeartbeatRequest& request, @@ -136,11 +145,14 @@ class Heartbeat { bool GetMetaserverSpaceStatus(MetaServerSpaceStatus* status, uint64_t ncopysets); + // Handle heartbeat and send recyclable BlockGroup requests + void DeallocateBolckGroup(const HeartbeatResponse &response); + private: friend class HeartbeatTest; private: - Thread hbThread_; + Thread hbThread_; std::atomic toStop_; @@ -175,6 +187,11 @@ class HeartbeatTaskExecutor { void ExecTasks(const HeartbeatResponse& response); + // for unit test + void SetDeallocTask(uint32_t fsid, uint64_t offset) { + deallocTask_[fsid] = offset; + } + private: void ExecOneTask(const CopySetConf& conf); @@ -187,8 +204,13 @@ class HeartbeatTaskExecutor { bool NeedPurge(const CopySetConf& conf); private: + friend class Heartbeat; + CopysetNodeManager* copysetMgr_; butil::EndPoint ep_; + + // key is fsid, value is blockgroupOffset + std::map deallocTask_; }; } // namespace metaserver diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 096fcd857c..be56544034 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -342,8 +342,12 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) { needUpdate = true; } + bool fileNeedDeallocate = + (needAddTrash && (FsFileType::TYPE_FILE == old.type())); + bool s3NeedTrash = (needAddTrash && (FsFileType::TYPE_S3 == old.type())); + if (needUpdate) { - ret = inodeStorage_->Update(old); + ret = inodeStorage_->Update(old, fileNeedDeallocate); if (ret != MetaStatusCode::OK) { LOG(ERROR) << "UpdateInode fail, " << request.ShortDebugString() << ", ret: " << MetaStatusCode_Name(ret); @@ -351,7 +355,7 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) { } } - if (needAddTrash) { + if (s3NeedTrash) { trash_->Add(old.fsid(), old.inodeid(), old.dtime()); --(*type2InodeNum_)[old.type()]; } @@ -556,7 +560,7 @@ MetaStatusCode InodeManager::UpdateVolumeExtentSlice( MetaStatusCode InodeManager::UpdateVolumeExtent( uint32_t fsId, uint64_t inodeId, - const VolumeExtentList &extents) { + const VolumeExtentSliceList &extents) { VLOG(6) << "UpdateInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId; NameLockGuard guard(inodeLock_, GetInodeLockName(fsId, inodeId)); @@ -578,7 +582,7 @@ MetaStatusCode InodeManager::GetVolumeExtent( uint32_t fsId, uint64_t inodeId, const std::vector &slices, - VolumeExtentList *extents) { + VolumeExtentSliceList *extents) { VLOG(6) << "GetInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId; if (slices.empty()) { diff --git a/curvefs/src/metaserver/inode_manager.h b/curvefs/src/metaserver/inode_manager.h index 0bca465989..eedc3c807d 100644 --- a/curvefs/src/metaserver/inode_manager.h +++ b/curvefs/src/metaserver/inode_manager.h @@ -113,7 +113,7 @@ class InodeManager { // Update one or more volume extent slice MetaStatusCode UpdateVolumeExtent(uint32_t fsId, uint64_t inodeId, - const VolumeExtentList &extents); + const VolumeExtentSliceList &extents); // Update only one volume extent slice MetaStatusCode UpdateVolumeExtentSlice(uint32_t fsId, @@ -123,7 +123,7 @@ class InodeManager { MetaStatusCode GetVolumeExtent(uint32_t fsId, uint64_t inodeId, const std::vector &slices, - VolumeExtentList *extents); + VolumeExtentSliceList *extents); private: void GenerateInodeInternal(uint64_t inodeId, const InodeParam ¶m, diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index da175f5ba2..3dae80bba7 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -20,18 +20,23 @@ * Author: chenwei */ +#include + #include #include #include #include #include +#include #include "src/common/concurrent/rw_lock.h" #include "src/common/string_util.h" #include "curvefs/proto/metaserver.pb.h" +#include "curvefs/proto/common.pb.h" #include "curvefs/src/metaserver/storage/status.h" #include "curvefs/src/metaserver/inode_storage.h" #include "curvefs/src/metaserver/storage/converter.h" +#include "curvefs/src/metaserver/common/types.h" namespace curvefs { namespace metaserver { @@ -48,8 +53,8 @@ using ::curvefs::metaserver::storage::Prefix4ChunkIndexS3ChunkInfoList; using ::curvefs::metaserver::storage::Prefix4InodeS3ChunkInfoList; using ::curvefs::metaserver::storage::Prefix4AllInode; using ::curvefs::metaserver::storage::Key4InodeAuxInfo; - -using S3ChunkInfoMap = google::protobuf::Map; +using ::curvefs::metaserver::storage::Key4DeallocatableBlockGroup; +using ::curvefs::metaserver::storage::Prefix4AllDeallocatableBlockGroup; InodeStorage::InodeStorage(std::shared_ptr kvStorage, std::shared_ptr nameGenerator, @@ -59,8 +64,11 @@ InodeStorage::InodeStorage(std::shared_ptr kvStorage, table4S3ChunkInfo_(nameGenerator->GetS3ChunkInfoTableName()), table4VolumeExtent_(nameGenerator->GetVolumeExtentTableName()), table4InodeAuxInfo_(nameGenerator->GetInodeAuxInfoTableName()), - nInode_(nInode), - conv_() {} + table4DeallocatableInode_( + nameGenerator->GetDeallocatableInodeTableName()), + table4DeallocatableBlockGroup_( + nameGenerator->GetDeallocatableBlockGroupTableName()), + nInode_(nInode), conv_() {} MetaStatusCode InodeStorage::Insert(const Inode& inode) { WriteLockGuard lg(rwLock_); @@ -181,18 +189,53 @@ MetaStatusCode InodeStorage::Delete(const Key4Inode& key) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } -MetaStatusCode InodeStorage::Update(const Inode& inode) { +MetaStatusCode InodeStorage::Update(const Inode &inode, bool inodeDeallocate) { WriteLockGuard lg(rwLock_); Key4Inode key(inode.fsid(), inode.inodeid()); std::string skey = conv_.SerializeToString(key); - Status s = kvStorage_->HSet(table4Inode_, skey, inode); + // only update inodes + if (!inodeDeallocate) { + Status s = kvStorage_->HSet(table4Inode_, skey, inode); + if (s.ok()) { + return MetaStatusCode::OK; + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + // update inode and update deallocatable inode list + google::protobuf::Empty value; + auto txn = kvStorage_->BeginTransaction(); + if (nullptr == txn) { + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + std::string step = "update inode " + key.SerializeToString(); + + Status s = txn->HSet(table4Inode_, skey, inode); if (s.ok()) { - return MetaStatusCode::OK; + s = txn->HSet(table4DeallocatableInode_, skey, value); + step = "add inode " + key.SerializeToString() + + " to inode deallocatable list"; } - return MetaStatusCode::STORAGE_INTERNAL_ERROR; + + if (!s.ok()) { + LOG(ERROR) << "txn is failed in " << step; + if (!txn->Rollback().ok()) { + LOG(ERROR) << "rollback transaction failed, inode=" + << key.SerializeToString(); + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + } else if (!txn->Commit().ok()) { + LOG(ERROR) << "commit transaction failed, inode=" + << key.SerializeToString(); + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + return MetaStatusCode::OK; } + std::shared_ptr InodeStorage::GetAllInode() { ReadLockGuard lg(rwLock_); std::string sprefix = conv_.SerializeToString(Prefix4AllInode()); @@ -527,9 +570,9 @@ MetaStatusCode InodeStorage::UpdateVolumeExtentSlice( : MetaStatusCode::STORAGE_INTERNAL_ERROR; } -MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId, - uint64_t inodeId, - VolumeExtentList* extents) { +MetaStatusCode +InodeStorage::GetAllVolumeExtent(uint32_t fsId, uint64_t inodeId, + VolumeExtentSliceList *extents) { ReadLockGuard guard(rwLock_); auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId}); auto iter = kvStorage_->SSeek(table4VolumeExtent_, key); @@ -552,6 +595,13 @@ MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId, return MetaStatusCode::OK; } +std::shared_ptr InodeStorage::GetAllVolumeExtent(uint32_t fsId, + uint64_t inodeId) { + ReadLockGuard guard(rwLock_); + auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId}); + return kvStorage_->SSeek(table4VolumeExtent_, key); +} + MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId, uint64_t inodeId, uint64_t offset, @@ -571,5 +621,200 @@ MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::GetAllBlockGroup( + std::vector *deallocatableBlockGroupVec) { + auto iter = kvStorage_->HGetAll(table4DeallocatableBlockGroup_); + if (iter->Status() != 0) { + LOG(ERROR) << "InodeStorage failed to get iterator for all " + "deallocatable block group"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + uint32_t count = 0; + DeallocatableBlockGroup deallocatbleBlockGroup; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (!conv_.ParseFromString(iter->Value(), &deallocatbleBlockGroup)) { + LOG(ERROR) << "InodeStorage failed to parse deallocatable block " + "group"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + deallocatableBlockGroupVec->emplace_back( + std::move(deallocatbleBlockGroup)); + count++; + } + + return count > 0 ? MetaStatusCode::OK : MetaStatusCode::NOT_FOUND; +} + +MetaStatusCode InodeStorage::UpdateDeallocatableBlockGroup( + uint32_t fsId, const DeallocatableBlockGroupVec &update) { + auto txn = kvStorage_->BeginTransaction(); + + MetaStatusCode st = MetaStatusCode::OK; + std::string step; + + for (auto &item : update) { + Key4DeallocatableBlockGroup key(fsId, item.blockgroupoffset()); + std::string skey(key.SerializeToString()); + + DeallocatableBlockGroup out; + auto s = txn->HGet(table4DeallocatableBlockGroup_, skey, &out); + + if (!s.ok() && !s.IsNotFound()) { + step = "get deallocatable group skey=" + skey + " failed"; + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + break; + } + + if (item.has_increase()) { + // for first increase call, set blockgroupoffset + if (s.IsNotFound()) { + out.set_blockgroupoffset(item.blockgroupoffset()); + } + st = Increase(txn, fsId, item.increase(), &out); + } else if (item.has_decrease()) { + st = Decrease(item.decrease(), &out); + } else if (item.has_mark()) { + st = Mark(item.mark(), &out); + } + + s = txn->HSet(table4DeallocatableBlockGroup_, skey, out); + if (!s.ok()) { + step = "update deallocatable group skey=" + skey + " failed"; + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + break; + } + } + + if (st != MetaStatusCode::OK) { + LOG(ERROR) << "UpdateDeallocatableBlockGroup txn is failed at " << step; + if (!txn->Rollback().ok()) { + LOG(ERROR) << "UpdateDeallocatableBlockGroup rollback transaction " + "failed, fsId=" + << fsId; + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + } else if (!txn->Commit().ok()) { + LOG(ERROR) + << "UpdateDeallocatableBlockGroup commit transaction failed, fsId=" + << fsId; + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + return st; +} + +MetaStatusCode +InodeStorage::Increase(Transaction txn, uint32_t fsId, + const IncreaseDeallocatableBlockGroup &increase, + DeallocatableBlockGroup *out) { + MetaStatusCode st = MetaStatusCode::OK; + + // update DeallocatableBlockGroup + VLOG(6) << "InodeStorage handle increase=" << increase.DebugString(); + + uint64_t oldSize = + out->has_deallocatablesize() ? out->deallocatablesize() : 0; + out->set_deallocatablesize(oldSize + increase.increasedeallocatablesize()); + out->mutable_inodeidlist()->MergeFrom(increase.inodeidlistadd()); + std::set unique_elements(out->inodeidlist().begin(), + out->inodeidlist().end()); + out->mutable_inodeidlist()->Clear(); + for (auto &elem : unique_elements) { + out->mutable_inodeidlist()->Add(elem); + } + + VLOG(6) << "InodeStorage handle increase set out=" + << out->DebugString(); + + // remove related inode in table4DeallocatableInode_ + for (auto &inodeid : increase.inodeidlistadd()) { + auto s = txn->HDel( + table4DeallocatableInode_, + conv_.SerializeToString(Key4Inode{fsId, inodeid})); + if (!s.ok()) { + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + VLOG(6) << "InodeStorage delete inodeid=" << inodeid << " from " + << table4DeallocatableInode_ << " fail"; + break; + } + + VLOG(6) << "InodeStorage delete inodeid=" << inodeid << " from " + << StringToHex(table4DeallocatableInode_) << " success"; + } + + return st; +} + +MetaStatusCode +InodeStorage::Decrease(const DecreaseDeallocatableBlockGroup &decrease, + DeallocatableBlockGroup *out) { + VLOG(6) << "InodeStorage handle increase=" << decrease.DebugString(); + if (!out->IsInitialized() || !out->has_deallocatablesize()) { + LOG(ERROR) + << "UpdateDeallocatableBlockGroup record missing required fields"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + uint64_t oldSize = out->deallocatablesize(); + if (oldSize < decrease.decreasedeallocatablesize()) { + LOG(ERROR) << "UpdateDeallocatableBlockGroup decrease size is too big, " + "oldSize=" + << oldSize + << ", decreasesize=" << decrease.decreasedeallocatablesize(); + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + // update dallocatablesize + out->set_deallocatablesize(oldSize - decrease.decreasedeallocatablesize()); + + // update deallocatableinode list + auto inodeidlist = out->mutable_inodeidunderdeallocate(); + inodeidlist->erase( + std::remove_if(inodeidlist->begin(), inodeidlist->end(), + [&decrease](uint64_t inodeid) { + auto search = decrease.inodeddeallocated(); + return std::find(search.begin(), search.end(), + inodeid) != search.end(); + }), + inodeidlist->end()); + + VLOG(6) << "InodeStorage handle decrease ok, and set out=" + << out->DebugString(); + + return MetaStatusCode::OK; +} + +MetaStatusCode InodeStorage::Mark(const MarkDeallocatableBlockGroup &mark, + DeallocatableBlockGroup *out) { + MetaStatusCode st = MetaStatusCode::OK; + + VLOG(6) << "InodeStorage handle mark=" << mark.DebugString(); + if (!out->IsInitialized() || !out->has_deallocatablesize()) { + LOG(ERROR) + << "UpdateDeallocatableBlockGroup record missing required fields"; + st = MetaStatusCode::STORAGE_INTERNAL_ERROR; + } else { + // update inodeunderdeallocate + out->mutable_inodeidunderdeallocate()->MergeFrom( + mark.inodeidunderdeallocate()); + + // update inodeidlist + auto inodeidlist = out->mutable_inodeidlist(); + inodeidlist->erase( + std::remove_if(inodeidlist->begin(), inodeidlist->end(), + [&mark](uint64_t inodeid) { + auto search = mark.inodeidunderdeallocate(); + return std::find(search.begin(), search.end(), + inodeid) != search.end(); + }), + inodeidlist->end()); + VLOG(6) << "InodeStorage handle mark ok, and set out=" + << out->DebugString(); + } + return st; +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index fb878120a0..a92c6b3650 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -30,6 +30,7 @@ #include #include #include +#include #include "absl/container/btree_set.h" #include "absl/container/btree_map.h" @@ -49,8 +50,12 @@ using ::curvefs::metaserver::storage::StorageTransaction; using ::curvefs::metaserver::storage::Key4Inode; using ::curvefs::metaserver::storage::Converter; using ::curvefs::metaserver::storage::NameGenerator; + using S3ChunkInfoMap = google::protobuf::Map; +using DeallocatableBlockGroupVec = + google::protobuf::RepeatedPtrField; using Transaction = std::shared_ptr; + class InodeStorage { public: InodeStorage(std::shared_ptr kvStorage, @@ -98,9 +103,10 @@ class InodeStorage { /** * @brief update inode from storage * @param[in] inode: the inode want to update + * @param[in] inodeDeallocate: Whether the inode needs to deallocate space * @return If inode not exist, return NOT_FOUND; else replace and return OK */ - MetaStatusCode Update(const Inode& inode); + MetaStatusCode Update(const Inode& inode, bool inodeDeallocate = false); std::shared_ptr GetAllInode(); @@ -141,19 +147,24 @@ class InodeStorage { MetaStatusCode GetAllVolumeExtent(uint32_t fsId, uint64_t inodeId, - VolumeExtentList* extents); + VolumeExtentSliceList* extents); + + std::shared_ptr GetAllVolumeExtent(uint32_t fsId, + uint64_t inodeId); MetaStatusCode GetVolumeExtentByOffset(uint32_t fsId, uint64_t inodeId, uint64_t offset, VolumeExtentSlice* slice); - MetaStatusCode AddS3ChunkInfoList( - std::shared_ptr txn, - uint32_t fsId, - uint64_t inodeId, - uint64_t chunkIndex, - const S3ChunkInfoList* list2add); + // use the transaction to delete {inodes} in the deallocatable_inode_list + // and update the statistics of each item of blockgroup_list + MetaStatusCode + UpdateDeallocatableBlockGroup(uint32_t fsId, + const DeallocatableBlockGroupVec &update); + + MetaStatusCode GetAllBlockGroup( + std::vector *deallocatableBlockGroupVec); private: MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId, @@ -162,12 +173,25 @@ class InodeStorage { uint64_t GetInodeS3MetaSize(uint32_t fsId, uint64_t inodeId); - MetaStatusCode DelS3ChunkInfoList( - std::shared_ptr txn, - uint32_t fsId, - uint64_t inodeId, - uint64_t chunkIndex, - const S3ChunkInfoList* list2del); + MetaStatusCode DelS3ChunkInfoList(Transaction txn, + uint32_t fsId, uint64_t inodeId, + uint64_t chunkIndex, + const S3ChunkInfoList *list2del); + + MetaStatusCode AddS3ChunkInfoList(Transaction txn, + uint32_t fsId, uint64_t inodeId, + uint64_t chunkIndex, + const S3ChunkInfoList *list2add); + + MetaStatusCode Increase(Transaction txn, uint32_t fsId, + const IncreaseDeallocatableBlockGroup &increase, + DeallocatableBlockGroup *out); + + MetaStatusCode Decrease(const DecreaseDeallocatableBlockGroup &decrease, + DeallocatableBlockGroup *out); + + MetaStatusCode Mark(const MarkDeallocatableBlockGroup &mark, + DeallocatableBlockGroup *out); private: // FIXME: please remove this lock, because we has locked each inode @@ -179,6 +203,9 @@ class InodeStorage { std::string table4S3ChunkInfo_; std::string table4VolumeExtent_; std::string table4InodeAuxInfo_; + std::string table4DeallocatableBlockGroup_; + std::string table4DeallocatableInode_; + size_t nInode_; Converter conv_; }; diff --git a/curvefs/src/metaserver/mds/fsinfo_manager.cpp b/curvefs/src/metaserver/mds/fsinfo_manager.cpp new file mode 100644 index 0000000000..25b2c5ca9d --- /dev/null +++ b/curvefs/src/metaserver/mds/fsinfo_manager.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Mar 22 10:39:52 CST 2023 + * Author: lixiaocui + */ + +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" + +namespace curvefs { +namespace metaserver { +bool FsInfoManager::GetFsInfo(uint32_t fsId, FsInfo *fsInfo) { + std::lock_guard lock(mtx_); + auto iter = fsInfoMap_.find(fsId); + if (iter == fsInfoMap_.end()) { + auto ret = mdsClient_->GetFsInfo(fsId, fsInfo); + if (ret != FSStatusCode::OK) { + if (FSStatusCode::NOT_FOUND == ret) { + LOG(ERROR) << "The fsName not exist, fsId = " << fsId; + return false; + } else { + LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret + << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) + << ", fsId = " << fsId; + return false; + } + } + fsInfoMap_.insert({fsId, *fsInfo}); + } else { + *fsInfo = iter->second; + } + + return true; +} +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/mds/fsinfo_manager.h b/curvefs/src/metaserver/mds/fsinfo_manager.h new file mode 100644 index 0000000000..0cf6fb84e8 --- /dev/null +++ b/curvefs/src/metaserver/mds/fsinfo_manager.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Mar 22 10:40:08 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_ +#define CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_ + +#include +#include +#include +#include "curvefs/src/client/rpcclient/mds_client.h" + +namespace curvefs { +namespace metaserver { + +using ::curvefs::client::rpcclient::MdsClient; + +class FsInfoManager { + public: + static FsInfoManager &GetInstance() { + static FsInfoManager instance_; + return instance_; + } + + void SetMdsClient(std::shared_ptr mdsClient) { + mdsClient_ = mdsClient; + } + + bool GetFsInfo(uint32_t fsId, FsInfo *fsInfo); + + private: + std::shared_ptr mdsClient_; + std::map fsInfoMap_; + + std::mutex mtx_; +}; +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_ diff --git a/curvefs/src/metaserver/metacli_manager.cpp b/curvefs/src/metaserver/metacli_manager.cpp new file mode 100644 index 0000000000..4c7f1533cf --- /dev/null +++ b/curvefs/src/metaserver/metacli_manager.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Fri Jun 30 10:55:44 CST 2023 + * Author: lixiaocui + */ + +#include "curvefs/src/metaserver/metacli_manager.h" + +namespace curvefs { +namespace metaserver { + +using curvefs::client::rpcclient::MetaServerClientImpl; + +std::shared_ptr MetaCliManager::GetMetaCli(uint32_t fsId) { + std::lock_guard lock(mtx_); + auto out = metaCliMap_.find(fsId); + + if (out == metaCliMap_.end()) { + auto metaCli = std::make_shared(); + auto metaCache = std::make_shared(); + + metaCache->Init(opt_.metaCacheOpt, opt_.cli2Cli, opt_.mdsCli); + metaCli->Init(opt_.executorOpt, opt_.internalOpt, metaCache, + opt_.channelManager); + metaCliMap_.insert({fsId, metaCli}); + + return metaCli; + } else { + return out->second; + } +} +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/metacli_manager.h b/curvefs/src/metaserver/metacli_manager.h new file mode 100644 index 0000000000..e12b80b0b6 --- /dev/null +++ b/curvefs/src/metaserver/metacli_manager.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Fri Jun 30 10:56:51 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_ +#define CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_ + +#include +#include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/client/rpcclient/metaserver_client.h" + +namespace curvefs { +namespace metaserver { + +using curvefs::client::rpcclient::ChannelManager; +using curvefs::client::rpcclient::Cli2Client; +using curvefs::client::rpcclient::MdsClient; +using curvefs::client::rpcclient::MetaCache; +using curvefs::client::rpcclient::MetaServerClient; + +struct MetaCliManagerOpt { + MetaCacheOpt metaCacheOpt; + ExcutorOpt executorOpt; + ExcutorOpt internalOpt; + + std::shared_ptr cli2Cli; + std::shared_ptr mdsCli; + std::shared_ptr> channelManager; +}; + +class MetaCliManager { + public: + static MetaCliManager &GetInstance() { + static MetaCliManager instance_; + return instance_; + } + + void Init(MetaCliManagerOpt &&Opt) { + opt_ = std::move(Opt); + } + + std::shared_ptr GetMetaCli(uint32_t fsId); + + private: + std::mutex mtx_; + std::map> metaCliMap_; + + MetaCliManagerOpt opt_; +}; +} // namespace metaserver +} // namespace curvefs +#endif // CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_ diff --git a/curvefs/src/metaserver/metaserver.cpp b/curvefs/src/metaserver/metaserver.cpp index 9e10a3d1d5..8762da05fe 100644 --- a/curvefs/src/metaserver/metaserver.cpp +++ b/curvefs/src/metaserver/metaserver.cpp @@ -38,6 +38,7 @@ #include "curvefs/src/metaserver/trash_manager.h" #include "curvefs/src/metaserver/storage/storage.h" #include "curvefs/src/metaserver/storage/rocksdb_perf.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "src/common/crc32.h" #include "src/common/curve_version.h" #include "src/common/s3_adapter.h" @@ -46,6 +47,7 @@ #include "src/common/uri_parser.h" #include "curvefs/src/metaserver/resource_statistic.h" #include "src/fs/ext4_filesystem_impl.h" +#include "curvefs/src/metaserver/metacli_manager.h" namespace braft { @@ -160,6 +162,16 @@ void Metaserver::InitRecycleManagerOption( &recycleManagerOption->scanLimit)); } +void Metaserver::InitVolumeDeallocateOption( + VolumeDeallocateWorkerQueueOption *queueOpt, + VolumeDeallocateExecuteOption *execOpt) { + conf_->GetValueFatalIfFail("volume.deallocate.enable", &queueOpt->enable); + conf_->GetValueFatalIfFail("volume.deallocate.workerNum", + &queueOpt->workerNum); + conf_->GetValueFatalIfFail("volume.deallocate.batchClean", + &execOpt->batchClean); +} + void InitExcutorOption(const std::shared_ptr& conf, ExcutorOpt *opts, bool internal) { if (internal) { @@ -208,6 +220,8 @@ void Metaserver::Init() { mdsClient_ = std::make_shared(); mdsClient_->Init(mdsOptions_, mdsBase_); + FsInfoManager::GetInstance().SetMdsClient(mdsClient_); + // init metaserver client for recycle InitMetaClient(); @@ -245,6 +259,20 @@ void Metaserver::Init() { S3CompactManager::GetInstance().Init(conf_); + VolumeSpaceManagerOptions spaceManagerOpt; + spaceManagerOpt.mdsClient = mdsClient_; + conf_->GetValueFatalIfFail("volume.sdk.confPath", + &spaceManagerOpt.deviceOpt.configPath); + auto volumeSpaceMgr = std::make_shared(); + volumeSpaceMgr->Init(spaceManagerOpt); + + VolumeDeallocateWorkerQueueOption queueOpt; + VolumeDeallocateExecuteOption executeOpt; + executeOpt.metaClient = metaClient_; + executeOpt.volumeSpaceManager = std::move(volumeSpaceMgr); + InitVolumeDeallocateOption(&queueOpt, &executeOpt); + VolumeDeallocateManager::GetInstance().Init(queueOpt, executeOpt); + PartitionCleanOption partitionCleanOption; InitPartitionOption(s3Adaptor_, mdsClient_, &partitionCleanOption); PartitionCleanManager::GetInstance().Init(partitionCleanOption); @@ -266,6 +294,15 @@ void Metaserver::InitMetaClient() { InitExcutorOption(conf_, &excutorOpt, false); InitExcutorOption(conf_, &internalOpt, true); metaClient_->Init(excutorOpt, internalOpt, metaCache, channelManager); + + MetaCliManagerOpt opt; + opt.metaCacheOpt = std::move(metaCacheOpt); + opt.executorOpt = std::move(excutorOpt); + opt.internalOpt = std::move(internalOpt); + opt.cli2Cli = cli2Client; + opt.mdsCli = mdsClient_; + opt.channelManager = channelManager; + MetaCliManager::GetInstance().Init(std::move(opt)); } void Metaserver::GetMetaserverDataByLoadOrRegister() { @@ -494,6 +531,9 @@ void Metaserver::Run() { LOG_IF(FATAL, S3CompactManager::GetInstance().Run() != 0); running_ = true; + // start volume deallocate manager + VolumeDeallocateManager::GetInstance().Run(); + // start copyset node manager LOG_IF(FATAL, !copysetNodeManager_->Start()) << "Failed to start copyset node manager"; @@ -528,6 +568,7 @@ void Metaserver::Stop() { s3Adaptor_ = nullptr; S3CompactManager::GetInstance().Stop(); + VolumeDeallocateManager::GetInstance().Stop(); LOG(INFO) << "MetaServer stopped success"; } diff --git a/curvefs/src/metaserver/metaserver.h b/curvefs/src/metaserver/metaserver.h index 60b7e23581..75eb6e332b 100644 --- a/curvefs/src/metaserver/metaserver.h +++ b/curvefs/src/metaserver/metaserver.h @@ -45,6 +45,9 @@ #include "src/fs/local_filesystem.h" #include "curvefs/src/metaserver/resource_statistic.h" #include "curvefs/src/metaserver/recycle_manager.h" +#include "curvefs/src/metaserver/space/volume_deallocate_manager.h" +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" +#include "curvefs/src/metaserver/space/volume_space_manager.h" namespace curvefs { namespace metaserver { @@ -96,6 +99,9 @@ class Metaserver { PartitionCleanOption* partitionCleanOption); void InitRecycleManagerOption( RecycleManagerOption* recycleManagerOption); + + void InitVolumeDeallocateOption(VolumeDeallocateWorkerQueueOption *queueOpt, + VolumeDeallocateExecuteOption *execOpt); void GetMetaserverDataByLoadOrRegister(); int PersistMetaserverMeta(std::string path, MetaServerMetadata* metadata); int LoadMetaserverMeta(const std::string& metaFilePath, diff --git a/curvefs/src/metaserver/metaserver_service.cpp b/curvefs/src/metaserver/metaserver_service.cpp index 48966ed89e..6cc3d9c914 100644 --- a/curvefs/src/metaserver/metaserver_service.cpp +++ b/curvefs/src/metaserver/metaserver_service.cpp @@ -52,6 +52,7 @@ using ::curvefs::metaserver::copyset::DeletePartitionOperator; using ::curvefs::metaserver::copyset::PrepareRenameTxOperator; using ::curvefs::metaserver::copyset::GetVolumeExtentOperator; using ::curvefs::metaserver::copyset::UpdateVolumeExtentOperator; +using ::curvefs::metaserver::copyset::UpdateDeallocatableBlockGroupOperator; namespace { @@ -298,5 +299,16 @@ void MetaServerServiceImpl::UpdateVolumeExtent( request->copysetid()); } +void MetaServerServiceImpl::UpdateDeallocatableBlockGroup( + ::google::protobuf::RpcController *controller, + const UpdateDeallocatableBlockGroupRequest *request, + UpdateDeallocatableBlockGroupResponse *response, + ::google::protobuf::Closure *done) { + OperatorHelper helper(copysetNodeManager_, inflightThrottle_); + helper.operator()( + controller, request, response, done, request->poolid(), + request->copysetid()); +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/metaserver_service.h b/curvefs/src/metaserver/metaserver_service.h index 3dbf175dde..7e19e20b20 100644 --- a/curvefs/src/metaserver/metaserver_service.h +++ b/curvefs/src/metaserver/metaserver_service.h @@ -123,6 +123,12 @@ class MetaServerServiceImpl : public MetaServerService { UpdateVolumeExtentResponse* response, ::google::protobuf::Closure* done) override; + void UpdateDeallocatableBlockGroup( + ::google::protobuf::RpcController *controller, + const UpdateDeallocatableBlockGroupRequest *request, + UpdateDeallocatableBlockGroupResponse *response, + ::google::protobuf::Closure *done) override; + private: CopysetNodeManager* copysetNodeManager_; InflightThrottle* inflightThrottle_; diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index 381afd96eb..d927ca8c37 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -204,6 +204,7 @@ bool MetaStoreImpl::ClearInternal() { for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) { TrashManager::GetInstance().Remove(it->first); it->second->CancelS3Compact(); + it->second->CancelVolumeDeallocate(); PartitionCleanManager::GetInstance().Remove(it->first); if (!it->second->Clear()) { @@ -276,6 +277,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request, TrashManager::GetInstance().Remove(partitionId); RecycleManager::GetInstance().Remove(partitionId); it->second->CancelS3Compact(); + it->second->CancelVolumeDeallocate(); PartitionCleanManager::GetInstance().Remove(partitionId); partitionMap_.erase(it); response->set_statuscode(MetaStatusCode::OK); @@ -294,6 +296,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request, TrashManager::GetInstance().Remove(partitionId); RecycleManager::GetInstance().Remove(partitionId); it->second->CancelS3Compact(); + it->second->CancelVolumeDeallocate(); } else { LOG(INFO) << "DeletePartition, partition is already deleting" << ", partitionId = " << partitionId; @@ -316,27 +319,46 @@ bool MetaStoreImpl::GetPartitionInfoList( } rwLock_.Unlock(); return true; - } else { - LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get" - " the rwLock_"; - return false; } + LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get" + " the rwLock_"; + return false; +} + +bool MetaStoreImpl::GetPartitionSnap( + std::map> *partitionSnap) { + int ret = rwLock_.TryRDLock(); + if (ret == 0) { + *partitionSnap = partitionMap_; + rwLock_.Unlock(); + return true; + } + + LOG(WARNING) << "MetaStoreImpl get partition snap fail, it fail to get" + " the rwLock_"; + return false; } std::shared_ptr MetaStoreImpl::GetStreamServer() { return streamServer_; } +#define GET_PARTITION_OR_RETURN(PARTITION) \ + PARTITION = GetPartition(request->partitionid()); \ + if (PARTITION == nullptr) { \ + MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; \ + response->set_statuscode(status); \ + return status; \ + } + + // dentry MetaStatusCode MetaStoreImpl::CreateDentry(const CreateDentryRequest *request, CreateDentryResponse *response) { ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); + MetaStatusCode status = partition->CreateDentry(request->dentry()); response->set_statuscode(status); return status; @@ -349,12 +371,8 @@ MetaStatusCode MetaStoreImpl::GetDentry(const GetDentryRequest *request, const auto &name = request->name(); auto txId = request->txid(); ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); // handle by partition Dentry dentry; @@ -378,12 +396,8 @@ MetaStatusCode MetaStoreImpl::DeleteDentry(const DeleteDentryRequest *request, std::string name = request->name(); auto txId = request->txid(); ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); // handle by partition Dentry dentry; @@ -404,12 +418,8 @@ MetaStatusCode MetaStoreImpl::ListDentry(const ListDentryRequest *request, uint64_t parentInodeId = request->dirinodeid(); auto txId = request->txid(); ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); // handle by partition Dentry dentry; @@ -440,16 +450,12 @@ MetaStoreImpl::PrepareRenameTx(const PrepareRenameTxRequest *request, PrepareRenameTxResponse *response) { ReadLockGuard readLockGuard(rwLock_); MetaStatusCode rc; - auto partitionId = request->partitionid(); - auto partition = GetPartition(partitionId); - if (nullptr == partition) { - rc = MetaStatusCode::PARTITION_NOT_FOUND; - } else { - std::vector dentrys{request->dentrys().begin(), - request->dentrys().end()}; - rc = partition->HandleRenameTx(dentrys); - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); + std::vector dentrys{request->dentrys().begin(), + request->dentrys().end()}; + rc = partition->HandleRenameTx(dentrys); response->set_statuscode(rc); return rc; } @@ -487,12 +493,9 @@ MetaStatusCode MetaStoreImpl::CreateInode(const CreateInodeRequest *request, } ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); + MetaStatusCode status = partition->CreateInode(param, response->mutable_inode()); response->set_statuscode(status); @@ -521,12 +524,8 @@ MetaStoreImpl::CreateRootInode(const CreateRootInodeRequest *request, } ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); MetaStatusCode status = partition->CreateRootInode(param); response->set_statuscode(status); @@ -560,12 +559,8 @@ MetaStoreImpl::CreateManageInode(const CreateManageInodeRequest *request, } ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); MetaStatusCode status = partition->CreateManageInode( param, request->managetype(), response->mutable_inode()); @@ -597,12 +592,8 @@ MetaStatusCode MetaStoreImpl::GetInode(const GetInodeRequest *request, uint64_t inodeId = request->inodeid(); ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); Inode *inode = response->mutable_inode(); MetaStatusCode rc = partition->GetInode(fsId, inodeId, inode); @@ -633,12 +624,8 @@ MetaStatusCode MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request, BatchGetInodeAttrResponse *response) { ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); uint32_t fsId = request->fsid(); MetaStatusCode status = MetaStatusCode::OK; @@ -658,12 +645,8 @@ MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request, MetaStatusCode MetaStoreImpl::BatchGetXAttr(const BatchGetXAttrRequest *request, BatchGetXAttrResponse *response) { ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); uint32_t fsId = request->fsid(); MetaStatusCode status = MetaStatusCode::OK; @@ -686,12 +669,8 @@ MetaStatusCode MetaStoreImpl::DeleteInode(const DeleteInodeRequest *request, uint64_t inodeId = request->inodeid(); ReadLockGuard readLockGuard(rwLock_); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); MetaStatusCode status = partition->DeleteInode(fsId, inodeId); response->set_statuscode(status); @@ -702,12 +681,8 @@ MetaStatusCode MetaStoreImpl::UpdateInode(const UpdateInodeRequest *request, UpdateInodeResponse *response) { ReadLockGuard readLockGuard(rwLock_); VLOG(9) << "UpdateInode inode " << request->inodeid(); - std::shared_ptr partition = GetPartition(request->partitionid()); - if (partition == nullptr) { - MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(status); - return status; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); MetaStatusCode status = partition->UpdateInode(*request); response->set_statuscode(status); @@ -720,12 +695,8 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo( std::shared_ptr *iterator) { MetaStatusCode rc; ReadLockGuard readLockGuard(rwLock_); - auto partition = GetPartition(request->partitionid()); - if (nullptr == partition) { - rc = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(rc); - return rc; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); uint32_t fsId = request->fsid(); uint64_t inodeId = request->inodeid(); @@ -792,12 +763,8 @@ MetaStatusCode MetaStoreImpl::GetVolumeExtent(const GetVolumeExtentRequest *request, GetVolumeExtentResponse *response) { ReadLockGuard guard(rwLock_); - auto partition = GetPartition(request->partitionid()); - if (!partition) { - auto st = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(st); - return st; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); std::vector slices(request->sliceoffsets().begin(), request->sliceoffsets().end()); @@ -815,12 +782,8 @@ MetaStatusCode MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request, UpdateVolumeExtentResponse *response) { ReadLockGuard guard(rwLock_); - auto partition = GetPartition(request->partitionid()); - if (!partition) { - auto st = MetaStatusCode::PARTITION_NOT_FOUND; - response->set_statuscode(st); - return st; - } + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); VLOG(9) << "UpdateVolumeExtent, request: " << request->ShortDebugString(); @@ -830,6 +793,22 @@ MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request, return st; } +MetaStatusCode MetaStoreImpl::UpdateDeallocatableBlockGroup( + const UpdateDeallocatableBlockGroupRequest *request, + UpdateDeallocatableBlockGroupResponse *response) { + ReadLockGuard guard(rwLock_); + std::shared_ptr partition; + GET_PARTITION_OR_RETURN(partition); + + VLOG(9) << "UpdateDeallocatableBlockGroup, request: " + << request->ShortDebugString(); + + auto st = partition->UpdateDeallocatableBlockGroup(*request); + response->set_statuscode(st); + + return st; +} + bool MetaStoreImpl::InitStorage() { if (storageOptions_.type == "memory") { kvStorage_ = std::make_shared(storageOptions_); diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index cc941c665b..0c8061d2f5 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -86,6 +86,22 @@ using S3ChunkInfoMap = google::protobuf::Map; using ::curvefs::metaserver::storage::StorageOptions; +// Dentry and inode related data will be stored in kvstorage +// +// 1. When kvstorage is rocksdb, the structure of the data is as follows: +// (table is related with partitionID) +// column +// | +// (table1 table2 table3) +// | | | +// inode inodedealloc blockGroup-with-inodedealloc +// +// column +// | +// (table1 table2 table3) +// | | | +// dentry s3chunkinfo volumnextent + class MetaStore { public: MetaStore() = default; @@ -107,6 +123,9 @@ class MetaStore { virtual bool GetPartitionInfoList( std::list *partitionInfoList) = 0; + virtual bool GetPartitionSnap( + std::map> *partitionSnap) = 0; + virtual std::shared_ptr GetStreamServer() = 0; // dentry @@ -170,6 +189,10 @@ class MetaStore { virtual MetaStatusCode UpdateVolumeExtent( const UpdateVolumeExtentRequest* request, UpdateVolumeExtentResponse* response) = 0; + + virtual MetaStatusCode UpdateDeallocatableBlockGroup( + const UpdateDeallocatableBlockGroupRequest *request, + UpdateDeallocatableBlockGroupResponse *response) = 0; }; class MetaStoreImpl : public MetaStore { @@ -193,6 +216,9 @@ class MetaStoreImpl : public MetaStore { bool GetPartitionInfoList( std::list *partitionInfoList) override; + bool GetPartitionSnap( + std::map> *partitionSnap) override; + std::shared_ptr GetStreamServer() override; // dentry @@ -255,6 +281,11 @@ class MetaStoreImpl : public MetaStore { const UpdateVolumeExtentRequest* request, UpdateVolumeExtentResponse* response) override; + // block group + MetaStatusCode UpdateDeallocatableBlockGroup( + const UpdateDeallocatableBlockGroupRequest *request, + UpdateDeallocatableBlockGroupResponse *response) override; + private: FRIEND_TEST(MetastoreTest, partition); FRIEND_TEST(MetastoreTest, test_inode); diff --git a/curvefs/src/metaserver/metastore_fstream.cpp b/curvefs/src/metaserver/metastore_fstream.cpp index edf0cb8c32..02f4292007 100644 --- a/curvefs/src/metaserver/metastore_fstream.cpp +++ b/curvefs/src/metaserver/metastore_fstream.cpp @@ -80,8 +80,7 @@ bool MetaStoreFStream::LoadPartition(uint32_t partitionId, return false; } - LOG(INFO) << "Load partition, partition id: " << partitionId - << ", partition info: " << partitionInfo.ShortDebugString(); + LOG(INFO) << "Load partition info: " << partitionInfo.ShortDebugString(); // FIXME: partitionId is always 0 in some unittest, // maybe this problem also exist in production code diff --git a/curvefs/src/metaserver/partition.cpp b/curvefs/src/metaserver/partition.cpp index d2e46c2b7d..1cb4266759 100644 --- a/curvefs/src/metaserver/partition.cpp +++ b/curvefs/src/metaserver/partition.cpp @@ -33,6 +33,10 @@ #include "curvefs/src/metaserver/trash_manager.h" #include "curvefs/src/metaserver/storage/converter.h" #include "curvefs/src/metaserver/s3compact.h" +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" +#include "curvefs/src/metaserver/space/volume_deallocate_manager.h" +#include "curvefs/src/metaserver/copyset/copyset_node_manager.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" namespace curvefs { namespace metaserver { @@ -41,28 +45,30 @@ using ::curvefs::metaserver::storage::NameGenerator; Partition::Partition(PartitionInfo partition, std::shared_ptr kvStorage, - bool startCompact) { + bool startCompact, bool startVolumeDeallocate) { assert(partition.start() <= partition.end()); partitionInfo_ = std::move(partition); + uint32_t partitionId = partitionInfo_.partitionid(); + + kvStorage_ = kvStorage; + nameGen_ = std::make_shared(partitionId); uint64_t nInode = 0; uint64_t nDentry = 0; - uint32_t partitionId = partitionInfo_.partitionid(); if (partitionInfo_.has_inodenum()) { nInode = partitionInfo_.inodenum(); } if (partitionInfo_.has_dentrynum()) { nDentry = partitionInfo_.dentrynum(); } - auto tableName = std::make_shared(partitionId); - inodeStorage_ = std::make_shared( - kvStorage, tableName, nInode); - dentryStorage_ = std::make_shared( - kvStorage, tableName, nDentry); + inodeStorage_ = + std::make_shared(kvStorage_, nameGen_, nInode); + dentryStorage_ = + std::make_shared(kvStorage_, nameGen_, nDentry); - trash_ = std::make_shared(inodeStorage_); + auto trash = std::make_shared(inodeStorage_); inodeManager_ = std::make_shared( - inodeStorage_, trash_, partitionInfo_.mutable_filetype2inodenum()); + inodeStorage_, trash, partitionInfo_.mutable_filetype2inodenum()); txManager_ = std::make_shared(dentryStorage_); dentryManager_ = std::make_shared(dentryStorage_, txManager_); @@ -72,21 +78,39 @@ Partition::Partition(PartitionInfo partition, } if (partitionInfo_.status() != PartitionStatus::DELETING) { - TrashManager::GetInstance().Add(partitionInfo_.partitionid(), trash_); + TrashManager::GetInstance().Add(partitionInfo_.partitionid(), trash); if (startCompact) { StartS3Compact(); } + if (startVolumeDeallocate) { + StartVolumeDeallocate(); + } } } +#define PRECHECK(fsId, inodeId) \ + do { \ + if (!IsInodeBelongs((fsId), (inodeId))) { \ + return MetaStatusCode::PARTITION_ID_MISSMATCH; \ + } \ + if (GetStatus() == PartitionStatus::DELETING) { \ + return MetaStatusCode::PARTITION_DELETING; \ + } \ + } while (0) + +#define PRECHECK_FSID(fsId) \ + do { \ + if (!IsInodeBelongs((fsId))) { \ + return MetaStatusCode::PARTITION_ID_MISSMATCH; \ + } \ + if (GetStatus() == PartitionStatus::DELETING) { \ + return MetaStatusCode::PARTITION_DELETING; \ + } \ + } while (0) + MetaStatusCode Partition::CreateDentry(const Dentry& dentry) { - if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } + PRECHECK(dentry.fsid(), dentry.parentinodeid()); - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } MetaStatusCode ret = dentryManager_->CreateDentry(dentry); if (MetaStatusCode::OK == ret) { if (dentry.has_type()) { @@ -107,11 +131,8 @@ MetaStatusCode Partition::CreateDentry(const Dentry& dentry) { MetaStatusCode Partition::LoadDentry(const DentryVec& vec, bool merge) { auto dentry = vec.dentrys(0); - if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } else if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } + + PRECHECK(dentry.fsid(), dentry.parentinodeid()); MetaStatusCode rc = dentryManager_->CreateDentry(vec, merge); if (rc == MetaStatusCode::OK || @@ -122,9 +143,7 @@ MetaStatusCode Partition::LoadDentry(const DentryVec& vec, bool merge) { } MetaStatusCode Partition::DeleteDentry(const Dentry& dentry) { - if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } + PRECHECK(dentry.fsid(), dentry.parentinodeid()); MetaStatusCode ret = dentryManager_->DeleteDentry(dentry); if (MetaStatusCode::OK == ret) { @@ -143,14 +162,7 @@ MetaStatusCode Partition::DeleteDentry(const Dentry& dentry) { } MetaStatusCode Partition::GetDentry(Dentry* dentry) { - if (!IsInodeBelongs(dentry->fsid(), dentry->parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK(dentry->fsid(), dentry->parentinodeid()); return dentryManager_->GetDentry(dentry); } @@ -158,14 +170,7 @@ MetaStatusCode Partition::ListDentry(const Dentry& dentry, std::vector* dentrys, uint32_t limit, bool onlyDir) { - if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK(dentry.fsid(), dentry.parentinodeid()); return dentryManager_->ListDentry(dentry, dentrys, limit, onlyDir); } @@ -174,14 +179,8 @@ void Partition::ClearDentry() { } MetaStatusCode Partition::HandleRenameTx(const std::vector& dentrys) { - for (const auto& it : dentrys) { - if (!IsInodeBelongs(it.fsid(), it.parentinodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; + for (const auto &it : dentrys) { + PRECHECK(it.fsid(), it.parentinodeid()); } return dentryManager_->HandleRenameTx(dentrys); @@ -196,10 +195,6 @@ bool Partition::InsertPendingTx(const PrepareRenameTxRequest& pendingTx) { } } - if (GetStatus() == PartitionStatus::DELETING) { - return false; - } - auto renameTx = RenameTx(dentrys, dentryStorage_); return txManager_->InsertPendingTx(renameTx); } @@ -247,74 +242,42 @@ MetaStatusCode Partition::CreateInode(const InodeParam ¶m, } MetaStatusCode Partition::CreateRootInode(const InodeParam ¶m) { - if (!IsInodeBelongs(param.fsId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK_FSID(param.fsId); return inodeManager_->CreateRootInode(param); } MetaStatusCode Partition::CreateManageInode(const InodeParam ¶m, ManageInodeType manageType, Inode* inode) { - if (!IsInodeBelongs(param.fsId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK_FSID(param.fsId); return inodeManager_->CreateManageInode(param, manageType, inode); } MetaStatusCode Partition::GetInode(uint32_t fsId, uint64_t inodeId, Inode* inode) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - + PRECHECK(fsId, inodeId); return inodeManager_->GetInode(fsId, inodeId, inode); } MetaStatusCode Partition::GetInodeAttr(uint32_t fsId, uint64_t inodeId, InodeAttr* attr) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - + PRECHECK(fsId, inodeId); return inodeManager_->GetInodeAttr(fsId, inodeId, attr); } MetaStatusCode Partition::GetXAttr(uint32_t fsId, uint64_t inodeId, XAttr* xattr) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - + PRECHECK(fsId, inodeId); return inodeManager_->GetXAttr(fsId, inodeId, xattr); } MetaStatusCode Partition::DeleteInode(uint32_t fsId, uint64_t inodeId) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } + PRECHECK(fsId, inodeId); return inodeManager_->DeleteInode(fsId, inodeId); } MetaStatusCode Partition::UpdateInode(const UpdateInodeRequest& request) { - if (!IsInodeBelongs(request.fsid(), request.inodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - - if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK(request.fsid(), request.inodeid()); return inodeManager_->UpdateInode(request); } @@ -325,12 +288,7 @@ MetaStatusCode Partition::GetOrModifyS3ChunkInfo( const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap, std::shared_ptr* iterator) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } else if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } - + PRECHECK(fsId, inodeId); return inodeManager_->GetOrModifyS3ChunkInfo( fsId, inodeId, map2add, map2del, returnS3ChunkInfoMap, iterator); } @@ -339,19 +297,12 @@ MetaStatusCode Partition::PaddingInodeS3ChunkInfo(int32_t fsId, uint64_t inodeId, S3ChunkInfoMap* m, uint64_t limit) { - if (!IsInodeBelongs(fsId, inodeId)) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } else if (GetStatus() == PartitionStatus::DELETING) { - return MetaStatusCode::PARTITION_DELETING; - } + PRECHECK(fsId, inodeId); return inodeManager_->PaddingInodeS3ChunkInfo(fsId, inodeId, m, limit); } MetaStatusCode Partition::InsertInode(const Inode& inode) { - if (!IsInodeBelongs(inode.fsid(), inode.inodeid())) { - return MetaStatusCode::PARTITION_ID_MISSMATCH; - } - + PRECHECK(inode.fsid(), inode.inodeid()); return inodeManager_->InsertInode(inode); } @@ -374,7 +325,7 @@ bool Partition::IsDeletable() { return true; } -bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) { +bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) const { if (fsId != partitionInfo_.fsid()) { LOG(WARNING) << "partition fsid mismatch, fsId = " << fsId << ", inodeId = " << inodeId @@ -394,7 +345,7 @@ bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) { return true; } -bool Partition::IsInodeBelongs(uint32_t fsId) { +bool Partition::IsInodeBelongs(uint32_t fsId) const { if (fsId != partitionInfo_.fsid()) { return false; } @@ -402,10 +353,6 @@ bool Partition::IsInodeBelongs(uint32_t fsId) { return true; } -uint32_t Partition::GetPartitionId() const { - return partitionInfo_.partitionid(); -} - PartitionInfo Partition::GetPartitionInfo() { partitionInfo_.set_inodenum(GetInodeNum()); partitionInfo_.set_dentrynum(GetDentryNum()); @@ -481,19 +428,9 @@ std::string Partition::GetDentryTablename() { return oss.str(); } -#define PRECHECK(fsId, inodeId) \ - do { \ - if (!IsInodeBelongs((fsId), (inodeId))) { \ - return MetaStatusCode::PARTITION_ID_MISSMATCH; \ - } \ - if (GetStatus() == PartitionStatus::DELETING) { \ - return MetaStatusCode::PARTITION_DELETING; \ - } \ - } while (0) - -MetaStatusCode Partition::UpdateVolumeExtent(uint32_t fsId, - uint64_t inodeId, - const VolumeExtentList& extents) { +MetaStatusCode +Partition::UpdateVolumeExtent(uint32_t fsId, uint64_t inodeId, + const VolumeExtentSliceList &extents) { PRECHECK(fsId, inodeId); return inodeManager_->UpdateVolumeExtent(fsId, inodeId, extents); } @@ -509,11 +446,23 @@ MetaStatusCode Partition::UpdateVolumeExtentSlice( MetaStatusCode Partition::GetVolumeExtent(uint32_t fsId, uint64_t inodeId, const std::vector& slices, - VolumeExtentList* extents) { + VolumeExtentSliceList* extents) { PRECHECK(fsId, inodeId); return inodeManager_->GetVolumeExtent(fsId, inodeId, slices, extents); } +MetaStatusCode Partition::UpdateDeallocatableBlockGroup( + const UpdateDeallocatableBlockGroupRequest &request) { + PRECHECK_FSID(request.fsid()); + return inodeStorage_->UpdateDeallocatableBlockGroup(request.fsid(), + request.update()); +} + +MetaStatusCode Partition::GetAllBlockGroup( + std::vector *deallocatableBlockGroupVec) { + return inodeStorage_->GetAllBlockGroup(deallocatableBlockGroupVec); +} + void Partition::StartS3Compact() { S3CompactManager::GetInstance().Register( S3Compact{inodeManager_, partitionInfo_}); @@ -523,5 +472,57 @@ void Partition::CancelS3Compact() { S3CompactManager::GetInstance().Cancel(partitionInfo_.partitionid()); } +void Partition::StartVolumeDeallocate() { + FsInfo fsInfo; + bool ok = + FsInfoManager::GetInstance().GetFsInfo(partitionInfo_.fsid(), &fsInfo); + if (!ok) { + LOG(ERROR) + << "Partition start volume deallocate fail, get fsinfo fail. fsid=" + << partitionInfo_.fsid(); + return; + } + + if (!fsInfo.detail().has_volume()) { + LOG(INFO) << "Partition not belong to volume, do not need start " + "deallocate. partitionInfo=" + << partitionInfo_.DebugString(); + return; + } + + VolumeDeallocateCalOption calOpt; + calOpt.kvStorage = kvStorage_; + calOpt.inodeStorage = inodeStorage_; + calOpt.nameGen = nameGen_; + auto copysetNode = + copyset::CopysetNodeManager::GetInstance().GetSharedCopysetNode( + partitionInfo_.poolid(), partitionInfo_.copysetid()); + if (copysetNode == nullptr) { + LOG(ERROR) << "Partition get copyset node failed. poolid=" + << partitionInfo_.poolid() + << ", copysetid=" << partitionInfo_.copysetid(); + return; + } + + InodeVolumeSpaceDeallocate task(partitionInfo_.fsid(), + partitionInfo_.partitionid(), copysetNode); + task.Init(calOpt); + + VolumeDeallocateManager::GetInstance().Register(std::move(task)); + + VLOG(3) << "Partition start volume deallocate success. partitionInfo=" + << partitionInfo_.DebugString(); +} + +void Partition::CancelVolumeDeallocate() { + VolumeDeallocateManager::GetInstance().Cancel(partitionInfo_.partitionid()); +} + +void Partition::SetVolumeDeallocate(uint64_t fsId, uint64_t blockGroupOffset) { + assert(fsId == partitionInfo_.fsid()); + VolumeDeallocateManager::GetInstance().Deallocate( + partitionInfo_.partitionid(), blockGroupOffset); +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/partition.h b/curvefs/src/metaserver/partition.h index 5a316e2cd2..7d307b3a39 100644 --- a/curvefs/src/metaserver/partition.h +++ b/curvefs/src/metaserver/partition.h @@ -52,7 +52,8 @@ class Partition { public: Partition(PartitionInfo partition, std::shared_ptr kvStorage, - bool startCompact = true); + bool startCompact = true, bool startVolumeDeallocate = true); + Partition() = default; // dentry MetaStatusCode CreateDentry(const Dentry& dentry); @@ -111,7 +112,7 @@ class Partition { MetaStatusCode UpdateVolumeExtent(uint32_t fsId, uint64_t inodeId, - const VolumeExtentList& extents); + const VolumeExtentSliceList& extents); MetaStatusCode UpdateVolumeExtentSlice(uint32_t fsId, uint64_t inodeId, @@ -120,7 +121,14 @@ class Partition { MetaStatusCode GetVolumeExtent(uint32_t fsId, uint64_t inodeId, const std::vector& slices, - VolumeExtentList* extents); + VolumeExtentSliceList* extents); + + + MetaStatusCode UpdateDeallocatableBlockGroup( + const UpdateDeallocatableBlockGroupRequest &request); + + virtual MetaStatusCode GetAllBlockGroup( + std::vector *deallocatableBlockGroupVec); MetaStatusCode InsertInode(const Inode& inode); @@ -130,18 +138,20 @@ class Partition { bool IsDeletable(); // check if fsid matchs and inode range belongs to this partition - bool IsInodeBelongs(uint32_t fsId, uint64_t inodeId); + bool IsInodeBelongs(uint32_t fsId, uint64_t inodeId) const; // check if fsid match this partition - bool IsInodeBelongs(uint32_t fsId); + bool IsInodeBelongs(uint32_t fsId) const; - uint32_t GetPartitionId() const; + virtual uint32_t GetPartitionId() const { + return partitionInfo_.partitionid(); + } - uint32_t GetPoolId() { return partitionInfo_.poolid(); } + virtual uint32_t GetPoolId() const { return partitionInfo_.poolid(); } - uint32_t GetCopySetId() { return partitionInfo_.copysetid(); } + virtual uint32_t GetCopySetId() const { return partitionInfo_.copysetid(); } - uint32_t GetFsId() { return partitionInfo_.fsid(); } + virtual uint32_t GetFsId() const { return partitionInfo_.fsid(); } PartitionInfo GetPartitionInfo(); @@ -165,6 +175,12 @@ class Partition { void CancelS3Compact(); + void StartVolumeDeallocate(); + + void CancelVolumeDeallocate(); + + void SetVolumeDeallocate(uint64_t fsId, uint64_t blockGroupOffset); + std::string GetInodeTablename(); std::string GetDentryTablename(); @@ -190,10 +206,13 @@ class Partition { } private: + std::shared_ptr kvStorage_; + std::shared_ptr nameGen_; + std::shared_ptr inodeStorage_; std::shared_ptr dentryStorage_; + std::shared_ptr inodeManager_; - std::shared_ptr trash_; std::shared_ptr dentryManager_; std::shared_ptr txManager_; diff --git a/curvefs/src/metaserver/partition_cleaner.cpp b/curvefs/src/metaserver/partition_cleaner.cpp index 99960ca169..5b65caee40 100644 --- a/curvefs/src/metaserver/partition_cleaner.cpp +++ b/curvefs/src/metaserver/partition_cleaner.cpp @@ -19,10 +19,10 @@ * @Date: 2021-12-15 10:54:37 * @Author: chenwei */ -#include "curvefs/src/metaserver/partition_cleaner.h" #include - +#include "curvefs/src/metaserver/partition_cleaner.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "curvefs/src/metaserver/copyset/meta_operator.h" namespace curvefs { @@ -87,26 +87,15 @@ MetaStatusCode PartitionCleaner::CleanDataAndDeleteInode(const Inode& inode) { // TODO(cw123) : consider FsFileType::TYPE_FILE if (FsFileType::TYPE_S3 == inode.type()) { // get s3info from mds - FsInfo fsInfo; - if (fsInfoMap_.find(inode.fsid()) == fsInfoMap_.end()) { - auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo); - if (ret != FSStatusCode::OK) { - if (FSStatusCode::NOT_FOUND == ret) { - LOG(ERROR) - << "The fsName not exist, fsId = " << inode.fsid(); - return MetaStatusCode::S3_DELETE_ERR; - } else { - LOG(ERROR) - << "GetFsInfo failed, FSStatusCode = " << ret - << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) - << ", fsId = " << inode.fsid(); - return MetaStatusCode::S3_DELETE_ERR; - } - } - fsInfoMap_.insert({inode.fsid(), fsInfo}); - } else { - fsInfo = fsInfoMap_.find(inode.fsid())->second; - } + FsInfo fsInfo; + auto ret = + FsInfoManager::GetInstance().GetFsInfo(inode.fsid(), &fsInfo); + if (!ret) { + LOG(ERROR) << "PartitionCleaner get fsinfo failed, fsId = " + << inode.fsid(); + return MetaStatusCode::S3_DELETE_ERR; + } + const auto& s3Info = fsInfo.detail().s3info(); // reinit s3 adaptor S3ClientAdaptorOption clientAdaptorOption; diff --git a/curvefs/src/metaserver/partition_cleaner.h b/curvefs/src/metaserver/partition_cleaner.h index 0717a53f00..3abc899518 100644 --- a/curvefs/src/metaserver/partition_cleaner.h +++ b/curvefs/src/metaserver/partition_cleaner.h @@ -83,7 +83,6 @@ class PartitionCleaner { std::shared_ptr mdsClient_; bool isStop_; uint32_t inodeDeletePeriodMs_; - std::unordered_map fsInfoMap_; }; class PartitionCleanerClosure : public google::protobuf::Closure { diff --git a/curvefs/src/metaserver/space/inode_volume_space_deallocate.cpp b/curvefs/src/metaserver/space/inode_volume_space_deallocate.cpp new file mode 100644 index 0000000000..b1e98077e1 --- /dev/null +++ b/curvefs/src/metaserver/space/inode_volume_space_deallocate.cpp @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Mar 22 10:38:14 CST 2023 + * Author: lixiaocui + */ + +#include +#include +#include +#include +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" +#include "curvefs/src/client/async_request_closure.h" + +using curvefs::client::UpdateVolumeExtentClosure; +using curvefs::client::CURVEFS_ERROR; + +namespace curvefs { +namespace metaserver { + +void InodeVolumeSpaceDeallocate::CalDeallocatableSpace() { + // get all deallocatable inode + auto table = calOpt_.nameGen->GetDeallocatableInodeTableName(); + VLOG(3) << "InodeVolumeSpaceDeallocate cal deallocatable space for table=" + << StringToHex(table) << ", fsid=" << fsId_ + << ", partitionId=" << partitionId_; + + auto iter = calOpt_.kvStorage->HGetAll(table); + if (iter->Status() != 0) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate failed to get iterator for " + "all deallocatable indoe" + << ", fsid=" << fsId_ << ", partitionId=" << partitionId_; + return; + } + + // key is volume offset + DeallocatableBlockGroupMap increase; + uint64_t typicalInode = 0; + for (iter->SeekToFirst();; iter->Next()) { + // update according to metaserver client + if (increase.size() >= executeOpt_.batchClean || + (!iter->Valid() && increase.size() > 0)) { + MetaStatusCode st = metaCli_->UpdateDeallocatableBlockGroup( + fsId_, typicalInode, &increase); + if (st != MetaStatusCode::OK) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate update " + << "deallocatable block group fail" + << ", fsid=" << fsId_ + << ", partitionId=" << partitionId_ + << ", typicalInode=" << typicalInode; + } else { + LOG(INFO) << "InodeVolumeSpaceDeallocate cal success this " + "round, fsId=" + << fsId_ << ", partitionId=" << partitionId_ + << ", typicalInode=" << typicalInode + << ", blockGroupSize=" << blockGroupSize_; + } + increase.clear(); + } + + if (!iter->Valid()) { + break; + } + + VLOG(9) << "InodeVolumeSpaceDeallocate cal spac for inode=" + << iter->Key() << ", fsId=" << fsId_ + << ", partitionId=" << partitionId_; + + // parse deallocatable inode key + Key4Inode key; + bool ok = calOpt_.conv.ParseFromString(iter->Key(), &key); + if (!ok) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate parse inode key from " + << iter->Key() << " fail, fsId=" << fsId_ + << ", partitionId=" << partitionId_ + << ", blockGroupSize=" << blockGroupSize_; + continue; + } else { + typicalInode = key.inodeId; + } + + // fill IncreaseDeallocatableBlockGroup + ok = DeallocatableSpaceForInode(key, &increase); + if (!ok) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate cal space for inode=" + << key.inodeId << " fail, fsId=" << fsId_ + << ", partitionId=" << partitionId_ + << ", typicalInode=" << typicalInode + << ", blockGroupSize=" << blockGroupSize_; + continue; + } + } +} + +MetaStatusCode +InodeVolumeSpaceDeallocate::DeallocateOneBlockGroup(uint64_t blockGroupOffset) { + // get block group + auto skey = calOpt_.conv.SerializeToString( + Key4DeallocatableBlockGroup{fsId_, blockGroupOffset}); + + DeallocatableBlockGroup out; + auto s = calOpt_.kvStorage->HGet( + calOpt_.nameGen->GetDeallocatableBlockGroupTableName(), skey, &out); + if (s.IsNotFound()) { + LOG(INFO) << "InodeVolumeSpaceDeallocate do not record deallocatable " + "blockgroup, " + "fsId=" + << fsId_ << ", blockGroupOffset=" << blockGroupOffset; + return MetaStatusCode::OK; + } + if (!s.ok()) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate deallocate blockgroup fail, " + "fsId=" + << fsId_ << ", blockGroupOffset=" << blockGroupOffset + << ", partitionId=" << partitionId_ + << ", status=" << s.ToString(); + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + + VLOG(3) << "InodeVolumeSpaceDeallocate begin to deallocate blockgroup: " + << blockGroupOffset << ", fsId=" << fsId_ + << ", partitionId=" << partitionId_; + + // batch processing of inodelists involving space deallocatable + DeallocatableBlockGroupMap mark; + auto &onemark = mark[blockGroupOffset]; + onemark.set_blockgroupoffset(blockGroupOffset); + + auto iter = out.inodeidlist().begin(); + auto boundary = out.inodeidlist().end(); + while (iter != boundary) { + onemark.mutable_mark()->add_inodeidunderdeallocate(*iter); + iter++; + + auto size = onemark.mark().inodeidunderdeallocate_size(); + if (size >= executeOpt_.batchClean || (iter == boundary && size > 0)) { + VLOG(3) << "InodeVolumeSpaceDeallocate process specify inode:" + << onemark.DebugString(); + ProcessSepcifyInodeList(blockGroupOffset, &mark); + onemark.mutable_mark()->clear_inodeidunderdeallocate(); + } + } + + return MetaStatusCode::OK; +} + +bool InodeVolumeSpaceDeallocate::DeallocatableSpaceForInode( + const Key4Inode &key, DeallocatableBlockGroupMap *increaseMap) { + + // Calculate the deallocatable space of the blockgroup corresponding to the + // extent in the inode + auto iter = calOpt_.inodeStorage->GetAllVolumeExtent(key.fsId, key.inodeId); + + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + VolumeExtentSlice slice; + if (!slice.ParseFromString(iter->Value())) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate parse VolumeExtentSlice " + "failed, fsId=" + << key.fsId << ", inodeId=" << key.inodeId; + return false; + } + + VLOG(9) << "InodeVolumeSpaceDeallocate deallocate space for inode=" + << key.inodeId << ", table=" + << StringToHex( + calOpt_.nameGen->GetDeallocatableInodeTableName()) + << ", extent=" << slice.DebugString(); + + DeallocatbleSpaceForVolumeExtent(slice, key, increaseMap); + } + + return true; +} + +void InodeVolumeSpaceDeallocate::DeallocatbleSpaceForVolumeExtent( + const VolumeExtentSlice &slice, const Key4Inode &key, + DeallocatableBlockGroupMap *increaseMap) { + std::unordered_set relatedBlockGroup; + + for (const auto &next : slice.extents()) { + // | BlockGroup | BlockGroup | + // ----------- ----------- + // | 4k bitmap | | 4k bitmap | + // ----------- ----------- + // by default, 4K bitmap is not pre-allocated, so the extent will not + // cross block group. + auto blockGroupOffset = + (next.volumeoffset() / blockGroupSize_) * blockGroupSize_; + + auto &exist = (*increaseMap)[blockGroupOffset]; + auto increase = exist.mutable_increase(); + + if (relatedBlockGroup.count(blockGroupOffset)) { + auto oldSize = increase->increasedeallocatablesize(); + increase->set_increasedeallocatablesize(oldSize + next.length()); + } else { + relatedBlockGroup.emplace(blockGroupOffset); + exist.set_blockgroupoffset(blockGroupOffset); + increase->set_increasedeallocatablesize(next.length()); + } + } + + for (const auto &item : relatedBlockGroup) { + auto &exist = (*increaseMap)[item]; + exist.mutable_increase()->add_inodeidlistadd(key.inodeId); + VLOG(6) << "InodeVolumeSpaceDeallocate deallocatable space for inode:" + << key.SerializeToString() + << ", related block group:" << item << ", increase size:" + << exist.increase().increasedeallocatablesize(); + } +} + +// NOTE: +// 1. update the status of DeallocatableBlockGroup: add inodeunderdeallocate +// and remove related inode to inodeidlist +// +// 2. dallocate the space of the block according to the fetched inode +// +// 3. update the deallocatable status of DeallocatableBlockGroup: decrease +// deallocatableSize +void InodeVolumeSpaceDeallocate::ProcessSepcifyInodeList( + uint64_t blockGroupOffset, DeallocatableBlockGroupMap *markMap) { + + // 1 + uint64_t typicalInode = + (*markMap)[blockGroupOffset].mark().inodeidunderdeallocate(0); + + MetaStatusCode st = + metaCli_->UpdateDeallocatableBlockGroup(fsId_, typicalInode, markMap); + if (st != MetaStatusCode::OK) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate mark inodelist to be " + "deallocatable failed, fsId=" + << fsId_ << ", partitionId=" << partitionId_ + << ", blockGroupOffset=" << blockGroupOffset; + return; + } + auto &onemark = (*markMap)[blockGroupOffset]; + + // 2 + uint64_t decreaseSize = 0; + auto inodeUnderDeallocate = + onemark.mutable_mark()->mutable_inodeidunderdeallocate(); + bool ok = + DeallocateInode(blockGroupOffset, *inodeUnderDeallocate, &decreaseSize); + if (!ok) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate an error occurred while " + "actually deallocate the volume space, fsId=" + << fsId_ << ", blockGroupOffset=" << blockGroupOffset; + return; + } + + // 3 + DeallocatableBlockGroupMap decrease; + auto &onedecrease = decrease[blockGroupOffset]; + onedecrease.set_blockgroupoffset(blockGroupOffset); + onedecrease.mutable_decrease()->set_decreasedeallocatablesize(decreaseSize); + onedecrease.mutable_decrease()->mutable_inodeddeallocated()->Swap( + inodeUnderDeallocate); + + st = + metaCli_->UpdateDeallocatableBlockGroup(fsId_, typicalInode, &decrease); + if (st != MetaStatusCode::OK) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate update deallocatable size " + "failed, fsId=" + << fsId_ << ", partitionId=" << partitionId_ + << ", blockGroupOffset=" << blockGroupOffset; + return; + } +} + +// NOTE: +// 1. update inode extent +// 2. deallocate the corresponding space in volume +// +// According to this order, one is that the corresponding volume space will not +// be deleted repeatedly, but there is a disadvantage, when the process hangs +// up, this part of the space will not be recovered. And may need to be +// deallocate by means of fscheck, etc. +bool InodeVolumeSpaceDeallocate::DeallocateInode(uint64_t blockGroupOffset, + const Uint64Vec &inodelist, + uint64_t *decrease) { + std::vector deallocatableVolumeSpace; + + for (const auto &inodeId : inodelist) { + VolumeExtentSliceList sliceList; + MetaStatusCode st = calOpt_.inodeStorage->GetAllVolumeExtent( + fsId_, inodeId, &sliceList); + if (st != MetaStatusCode::OK) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate get inode extent " + "failed, fsId=" + << fsId_ << "partitionId=" << partitionId_ + << ", inodeId=" << inodeId; + return false; + } + VLOG(6) << "InodeVolumeSpaceDeallocate will update inode:" << inodeId + << ", blockGroupOffset:" << blockGroupOffset + << ", sliceList:" << sliceList.DebugString(); + + // 1 + UpdateDeallocateInodeExtentSlice(blockGroupOffset, decrease, &sliceList, + &deallocatableVolumeSpace); + UpdateVolumeExtentClosure closure(nullptr, true); + metaCli_->AsyncUpdateVolumeExtent(fsId_, inodeId, sliceList, &closure); + if (CURVEFS_ERROR::OK != closure.Wait()) { + LOG(ERROR) << "InodeVolumeSpaceDeallocate update inode:" << inodeId + << ", blockGroupOffset:" << blockGroupOffset + << ", to sliceList:" << sliceList.DebugString() + << " failed"; + return false; + } + VLOG(6) << "InodeVolumeSpaceDeallocate update inode:" << inodeId + << ", blockGroupOffset:" << blockGroupOffset + << ", to sliceList:" << sliceList.DebugString(); + } + + // 2 + bool ok = executeOpt_.volumeSpaceManager->DeallocVolumeSpace( + fsId_, blockGroupOffset, deallocatableVolumeSpace); + if (!ok) { + // TODO(ilixiaocui): this part of the non-recyclable space should be + // included in the metric statistics + LOG(ERROR) << "InodeVolumeSpaceDeallocate deallocate volume space " + "failed, fsId=" + << fsId_ << ", partitionId=" << partitionId_ + << ", blockGroupOffset=" << blockGroupOffset << " fail"; + } + + return ok; +} + +void InodeVolumeSpaceDeallocate::UpdateDeallocateInodeExtentSlice( + uint64_t blockGroupOffset, uint64_t *decrease, + VolumeExtentSliceList *sliceList, + std::vector *deallocatableVolumeSpace) { + uint64_t boundary = + (blockGroupOffset / blockGroupSize_ + 1) * blockGroupSize_; + + for (auto &slice : *sliceList->mutable_slices()) { + auto extents = slice.mutable_extents(); + extents->erase( + std::remove_if( + extents->begin(), extents->end(), + [&](const VolumeExtent &extent) { + // by default, 4K bitmap is not pre-allocated, so the extent + // will not cross block group. + bool res = (extent.volumeoffset() >= blockGroupOffset) && + (extent.volumeoffset() < boundary); + if (res) { + *decrease += extent.length(); + deallocatableVolumeSpace->emplace_back( + extent.volumeoffset(), extent.length()); + VLOG(6) << "InodeVolumeSpaceDeallocate cal " + "deallocatable volume space, volumeoffset:" + << extent.volumeoffset() + << ", length:" << extent.length(); + } + return res; + }), + extents->end()); + } +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/space/inode_volume_space_deallocate.h b/curvefs/src/metaserver/space/inode_volume_space_deallocate.h new file mode 100644 index 0000000000..0a26038d5c --- /dev/null +++ b/curvefs/src/metaserver/space/inode_volume_space_deallocate.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Mar 22 10:39:04 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_ +#define CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_ + +#include + +#include +#include +#include +#include +#include + +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/proto/common.pb.h" +#include "curvefs/src/metaserver/storage/converter.h" +#include "curvefs/src/metaserver/storage/storage.h" +#include "curvefs/src/metaserver/inode_storage.h" +#include "curvefs/src/metaserver/space/volume_space_manager.h" +#include "curvefs/src/client/rpcclient/metaserver_client.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/metaserver/copyset/copyset_node_manager.h" +#include "curvefs/src/metaserver/metacli_manager.h" + +namespace curvefs { +namespace metaserver { + +using curvefs::client::rpcclient::MetaServerClient; +using curvefs::common::BlockGroupID; +using curvefs::metaserver::storage::Converter; +using curvefs::metaserver::storage::Key4DeallocatableBlockGroup; +using curvefs::metaserver::storage::Key4Inode; +using curvefs::metaserver::storage::KVStorage; +using curvefs::metaserver::storage::Prefix4InodeVolumeExtent; +using curvefs::volume::Extent; + +using Uint64Vec = google::protobuf::RepeatedField; +using DeallocatableBlockGroupMap = std::map; + +struct VolumeDeallocateCalOption { + Converter conv; + std::shared_ptr nameGen; + std::shared_ptr kvStorage; + std::shared_ptr inodeStorage; +}; + +struct VolumeDeallocateExecuteOption { + std::shared_ptr volumeSpaceManager; + std::shared_ptr metaClient; + + uint32_t batchClean; +}; + +class InodeVolumeSpaceDeallocate { + public: + InodeVolumeSpaceDeallocate( + uint64_t fsId, uint32_t partitionId, + std::shared_ptr copysetNode) + : fsId_(fsId), partitionId_(partitionId), copysetNode_(copysetNode) { + metaCli_ = MetaCliManager::GetInstance().GetMetaCli(fsId_); + } + + void Init(VolumeDeallocateCalOption calOpt) { calOpt_ = std::move(calOpt); } + + int Init(const VolumeDeallocateExecuteOption &executeOpt) { + executeOpt_.volumeSpaceManager = executeOpt.volumeSpaceManager; + executeOpt_.metaClient = executeOpt.metaClient; + executeOpt_.batchClean = executeOpt.batchClean; + + blockGroupSize_ = + executeOpt_.volumeSpaceManager->GetBlockGroupSize(fsId_); + if (blockGroupSize_ <= 0) { + return -1; + } + + LOG(INFO) << "InodeVolumeSpaceDeallocate init, fsid=" << fsId_ + << ", partitionId=" << partitionId_ + << ", blockGroupSize=" << blockGroupSize_; + return 0; + } + + // used to traverse the list of inodes to be deleted, and count the + // deallocatable space of BlockGroup + void CalDeallocatableSpace(); + + // Return the reclaimable space of each BlockGroup + // deallocatablespace: key=str(fsid+offset) value=(deallocatabe space size) + void + GetDeallocatableSpace(std::map *deallocatablespaces); + + // DeAllocate the space of the specified BlockGroup + MetaStatusCode DeallocateOneBlockGroup(uint64_t blockGroupOffset); + + uint32_t GetPartitionID() const { return partitionId_; } + + void SetCanceled() { canceled_ = true; } + + bool IsCanceled() const { return canceled_; } + + bool HasDeallocateTask() { + return waitingBlockGroupOffset_.has_value(); + } + + uint64_t GetDeallocateTask() { + return waitingBlockGroupOffset_.value(); + } + + void ResetDeallocateTask() { + waitingBlockGroupOffset_.reset(); + } + + void SetDeallocateTask(uint64_t blockGroupOffset) { + waitingBlockGroupOffset_ = blockGroupOffset; + LOG(INFO) << "InodeVolumeSpaceDeallocate set deallocate task, " + << "blockGroupOffset=" << blockGroupOffset; + } + + bool CanStart() { + bool leader = copysetNode_->IsLeaderTerm(); + if (!leader) { + LOG(WARNING) << "InodeVolumeSpaceDeallocate copyset=" + << copysetNode_->Name() << " is not leader, skip"; + } + return leader; + } + + private: + friend class InodeVolumeSpaceDeallocateTest; + FRIEND_TEST(InodeVolumeSpaceDeallocateTest, + Test_DeallocatableSpaceForInode); + FRIEND_TEST(InodeVolumeSpaceDeallocateTest, Test_ProcessSepcifyInodeList); + FRIEND_TEST(InodeVolumeSpaceDeallocateTest, Test_DeallocateInode); + FRIEND_TEST(InodeVolumeSpaceDeallocateTest, + Test_UpdateDeallocateInodeExtentSlice); + + bool DeallocatableSpaceForInode(const Key4Inode &key, + DeallocatableBlockGroupMap *increaseMap); + + void + DeallocatbleSpaceForVolumeExtent(const VolumeExtentSlice &slice, + const Key4Inode &key, + DeallocatableBlockGroupMap *increaseMap); + + void ProcessSepcifyInodeList(uint64_t blockGroupOffset, + DeallocatableBlockGroupMap *markMap); + bool DeallocateInode(uint64_t blockGroupOffset, const Uint64Vec &inodelist, + uint64_t *decrease); + + private: + void UpdateDeallocateInodeExtentSlice( + uint64_t blockGroupOffset, uint64_t *decrease, + VolumeExtentSliceList *sliceList, + std::vector *deallocatableVolumeSpace); + + private: + VolumeDeallocateCalOption calOpt_; + VolumeDeallocateExecuteOption executeOpt_; + + uint64_t fsId_; + uint32_t partitionId_; + uint64_t blockGroupSize_; + + absl::optional waitingBlockGroupOffset_; + + bool canceled_{false}; + + std::shared_ptr copysetNode_; + std::shared_ptr metaCli_; +}; +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_ diff --git a/curvefs/src/metaserver/space/volume_deallocate_manager.cpp b/curvefs/src/metaserver/space/volume_deallocate_manager.cpp new file mode 100644 index 0000000000..50832a0e4e --- /dev/null +++ b/curvefs/src/metaserver/space/volume_deallocate_manager.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 15:46:58 CST 2023 + * Author: lixiaocui + */ + +#include +#include "curvefs/src/metaserver/space/volume_deallocate_manager.h" + +namespace curvefs { +namespace metaserver { +void VolumeDeallocateManager::Register(InodeVolumeSpaceDeallocate task) { + if (task.Init(executeOpt_) != 0) { + return; + } + + { + std::lock_guard lk(workerCtx_.mtx); + workerCtx_.allTasks.emplace_back(std::move(task)); + } + + workerCtx_.cond.notify_one(); +} + +void VolumeDeallocateManager::Cancel(uint32_t partitionId) { + std::lock_guard lk(workerCtx_.mtx); + + // cancle from tasking + auto it = workerCtx_.tasking.find(partitionId); + if (it != workerCtx_.tasking.end()) { + it->second->Cancel(partitionId); + LOG(INFO) << "VolumeDeallocateManager cancel task, partitionId=" + << partitionId; + return; + } + + // cancel from waiting + workerCtx_.allTasks.remove_if([partitionId]( + const InodeVolumeSpaceDeallocate &task) { + bool ret = (task.GetPartitionID() == partitionId); + if (ret) { + LOG(INFO) + << "VolumeDeallocateManager cancel waiting task, partitionId=" + << partitionId; + } + return ret; + }); +} + +void VolumeDeallocateManager::Deallocate(uint32_t partitionId, + uint64_t blockGroupOffset) { + std::lock_guard lk(workerCtx_.mtx); + + // set from waiting + { + auto it = + std::find_if(workerCtx_.allTasks.begin(), workerCtx_.allTasks.end(), + [partitionId](const InodeVolumeSpaceDeallocate &task) { + return task.GetPartitionID() == partitionId; + }); + if (it != workerCtx_.allTasks.end()) { + it->SetDeallocateTask(blockGroupOffset); + return; + } + } + + // set from tasking + { + auto it = workerCtx_.tasking.find(partitionId); + if (it != workerCtx_.tasking.end()) { + it->second->SetDeallocate(blockGroupOffset); + return; + } + } + + LOG(ERROR) << "VolumeDeallocateManager do not find deallocate handler for " + "partitionId=" + << partitionId; +} + +bool VolumeDeallocateManager::HasDeallocate() { + std::lock_guard lk(workerCtx_.mtx); + + // view from waiting + for (auto &task : workerCtx_.allTasks) { + if (task.HasDeallocateTask()) { + return true; + } + } + + // view from tasking + for (auto &task : workerCtx_.tasking) { + if (task.second->HasDeallocate()) { + return true; + } + } + + return false; +} + +void VolumeDeallocateManager::Run() { + if (!workerOpt_.enable) { + LOG(INFO) << "VolumeDeallocateManager not enable"; + return; + } + + if (!workerCtx_.running.exchange(true)) { + for (uint16_t i = 0; i < workerOpt_.workerNum; i++) { + workers_.emplace_back( + absl::make_unique(&workerCtx_)); + workers_.back()->Run(); + } + LOG(INFO) << "VolumeDeallocateManager start all workers ok"; + } else { + LOG(INFO) << "VolumeDeallocateManager already run"; + } +} + +void VolumeDeallocateManager::Stop() { + if (!workerCtx_.running.exchange(false)) { + LOG(INFO) << "VolumeDeallocateManager has no worker running, skip"; + return; + } + + workerCtx_.cond.notify_all(); + for (auto &worker : workers_) { + worker->Stop(); + } + LOG(INFO) << "VolumeDeallocateManager stop all workers ok"; +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/space/volume_deallocate_manager.h b/curvefs/src/metaserver/space/volume_deallocate_manager.h new file mode 100644 index 0000000000..edcd17b933 --- /dev/null +++ b/curvefs/src/metaserver/space/volume_deallocate_manager.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 15:36:29 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_ +#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_ + +#include +#include + +#include "curvefs/src/metaserver/space/volume_deallocate_worker.h" +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" + +namespace curvefs { +namespace metaserver { + +struct VolumeDeallocateWorkerQueueOption { + bool enable = false; + uint32_t workerNum = 1; +}; + +class VolumeDeallocateManager { + public: + static VolumeDeallocateManager &GetInstance() { + static VolumeDeallocateManager instance; + return instance; + } + + void Init(const VolumeDeallocateWorkerQueueOption &option, + const VolumeDeallocateExecuteOption &executeOpt) { + workerOpt_.enable = option.enable; + workerOpt_.workerNum = option.workerNum; + + executeOpt_.volumeSpaceManager = executeOpt.volumeSpaceManager; + executeOpt_.metaClient = executeOpt.metaClient; + executeOpt_.batchClean = executeOpt.batchClean; + } + + void Register(InodeVolumeSpaceDeallocate task); + void Cancel(uint32_t partitionId); + void Deallocate(uint32_t partitioId, uint64_t blockGroupOffset); + bool HasDeallocate(); + + void Run(); + void Stop(); + + + + private: + VolumeDeallocateManager() = default; + ~VolumeDeallocateManager() = default; + + VolumeDeallocateWorkerQueueOption workerOpt_; + VolumeDeallocateExecuteOption executeOpt_; + + VolumeDeallocateWorkerContext workerCtx_; + std::vector> workers_; +}; + +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_ diff --git a/curvefs/src/metaserver/space/volume_deallocate_worker.cpp b/curvefs/src/metaserver/space/volume_deallocate_worker.cpp new file mode 100644 index 0000000000..eae2a0a9fc --- /dev/null +++ b/curvefs/src/metaserver/space/volume_deallocate_worker.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 16:52:55 CST 2023 + * Author: lixiaocui + */ + +#include "absl/cleanup/cleanup.h" + +#include "curvefs/src/common/threading.h" +#include "curvefs/src/metaserver/space/volume_deallocate_worker.h" + +namespace curvefs { +namespace metaserver { +void VolumeDeallocateWorker::Run() { + calWorkingThread_ = + std::thread(&VolumeDeallocateWorker::DeallocatetWorker, this); +} + +void VolumeDeallocateWorker::Stop() { + sleeper_.interrupt(); + + if (calWorkingThread_.joinable()) { + calWorkingThread_.join(); + } + + LOG(INFO) << "VolumeDeallocateWorker stopped"; +} + +void VolumeDeallocateWorker::Cancel(uint32_t partitionId) { + assert(current_->GetPartitionID() == partitionId); + current_->SetCanceled(); + sleeper_.interrupt(); +} + +void VolumeDeallocateWorker::SetDeallocate(uint64_t blockGroupOffset) { + current_->SetDeallocateTask(blockGroupOffset); +} + +bool VolumeDeallocateWorker::HasDeallocate() { + return current_->HasDeallocateTask(); +} + +bool VolumeDeallocateWorker::WaitDeallocate() { + // wait for task arrive + std::unique_lock lock(ctx_->mtx); + if (ctx_->allTasks.empty()) { + VLOG(6) << "VolumeDeallocateWorker task empty, wait"; + ctx_->cond.wait( + lock, [this] { return !ctx_->allTasks.empty() || !ctx_->running; }); + } + + // cancel by manager + if (!ctx_->running) { + LOG(INFO) << "VolumeDeallocateWorker running was canceled"; + return false; + } + + // wait ok + current_ = std::move(ctx_->allTasks.front()); + ctx_->allTasks.pop_front(); + ctx_->tasking.emplace(current_->GetPartitionID(), this); + + VLOG(1) << "VolumeDeallocateWorker get task, partitionId = " + << current_->GetPartitionID(); + + sleeper_.init(); + return true; +} + +void VolumeDeallocateWorker::Cleanup() { + std::lock_guard lock(ctx_->mtx); + + ctx_->tasking.erase(current_->GetPartitionID()); + + if (current_->IsCanceled()) { + LOG(INFO) << "VolumeDeallocateWorker task canceled, partitionId=" + << current_->GetPartitionID(); + return; + } + + ctx_->allTasks.emplace_back(std::move(current_.value())); + ctx_->cond.notify_one(); + + VLOG(3) << "VolumeDeallocateWorker current tasks num:" + << ctx_->allTasks.size(); + current_.reset(); +} + +void VolumeDeallocateWorker::DeallocatetWorker() { + common::SetThreadName("CalVolumeDeallocateWorker"); + while (ctx_->running) { + sleeper_.wait_for(std::chrono::seconds(10)); + + if (!WaitDeallocate()) { + continue; + } + + CHECK(current_.has_value()); + VLOG(1) << "VolumeDeallocateWorker Going to statistic deallocatable " + "space for block group"; + + auto cleanup = absl::MakeCleanup([&, this] { Cleanup(); }); + + if (!current_->CanStart()) { + if (current_->HasDeallocateTask()) { + current_->ResetDeallocateTask(); + } + continue; + } + + if (current_->HasDeallocateTask()) { + current_->DeallocateOneBlockGroup(current_->GetDeallocateTask()); + VLOG(1) << "VolumeDeallocateWorker deallocate for " + "partitionId=" + << current_->GetPartitionID() << " ok"; + current_->ResetDeallocateTask(); + } else { + current_->CalDeallocatableSpace(); + VLOG(1) << "VolumeDeallocateWorker cal deallocatable space for " + "partitionId=" + << current_->GetPartitionID() << " ok"; + } + } + + LOG(INFO) << "VolumeDeallocateWorker stopped"; +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/space/volume_deallocate_worker.h b/curvefs/src/metaserver/space/volume_deallocate_worker.h new file mode 100644 index 0000000000..a759a115f3 --- /dev/null +++ b/curvefs/src/metaserver/space/volume_deallocate_worker.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 16:13:06 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_ +#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_ + +#include +#include +#include +#include + +#include "absl/types/optional.h" + +#include "src/common/interruptible_sleeper.h" +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" + +namespace curvefs { +namespace metaserver { + +class VolumeDeallocateWorker; + +struct VolumeDeallocateWorkerContext { + std::atomic running{false}; + + std::mutex mtx; + std::condition_variable cond; + std::list allTasks; + std::unordered_map tasking; +}; + +class VolumeDeallocateWorker { + public: + explicit VolumeDeallocateWorker(VolumeDeallocateWorkerContext *context) + : ctx_(context) {} + + void Run(); + void Stop(); + void Cancel(uint32_t partitionId); + void SetDeallocate(uint64_t blockGroupOffset); + bool HasDeallocate(); + + private: + bool WaitDeallocate(); + void Cleanup(); + void DeallocatetWorker(); + void ExecuteDeallocateWorker(); + + private: + VolumeDeallocateWorkerContext *ctx_; + std::thread calWorkingThread_; + + // current cal + absl::optional current_; + + curve::common::InterruptibleSleeper sleeper_; +}; + +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_ diff --git a/curvefs/src/metaserver/space/volume_space_manager.cpp b/curvefs/src/metaserver/space/volume_space_manager.cpp new file mode 100644 index 0000000000..af9e5cb3bd --- /dev/null +++ b/curvefs/src/metaserver/space/volume_space_manager.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Mar 28 19:55:27 CST 2023 + * Author: lixiaocui + */ + +#include "curvefs/src/metaserver/space/volume_space_manager.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" + +namespace curvefs { +namespace metaserver { + +using curvefs::volume::SpaceManagerImpl; + +void VolumeSpaceManager::Uninit() { + std::lock_guard lock(mutex_); + for (auto &item : spaceManagers_) { + item.second->Shutdown(); + } +} + +void VolumeSpaceManager::Destroy(uint32_t fsId) { + std::lock_guard lock(mutex_); + auto it = spaceManagers_.find(fsId); + if (it != spaceManagers_.end()) { + spaceManagers_.erase(it); + } +} + +uint64_t VolumeSpaceManager::GetBlockGroupSize(uint32_t fsId) { + auto spaceManager = GetSpaceManager(fsId); + if (spaceManager == nullptr) { + LOG(ERROR) << "VolumeSpaceManager get block group size failed, could " + "not get space manager, fsId " + << fsId << " not exist"; + return 0; + } + return spaceManager->GetBlockGroupSize(); +} + +bool VolumeSpaceManager::DeallocVolumeSpace( + uint32_t fsId, uint64_t blockGroupOffset, + const std::vector &deallocatableVolumeSpace) { + auto spaceManager = GetSpaceManager(fsId); + if (spaceManager == nullptr) { + LOG(ERROR) << "VolumeSpaceManager dealloc volume space failed, could " + "not get space manager, fsId " + << fsId << " not exist"; + return false; + } + return spaceManager->DeAlloc(blockGroupOffset, deallocatableVolumeSpace); +} + +std::shared_ptr +VolumeSpaceManager::GetSpaceManager(uint32_t fsId) { + std::lock_guard lock(mutex_); + auto it = spaceManagers_.find(fsId); + if (it == spaceManagers_.end()) { + if (!GenerateSpaceManager(fsId)) { + return nullptr; + } + } + return spaceManagers_[fsId]; +} + +bool VolumeSpaceManager::GenerateSpaceManager(uint32_t fsId) { + FsInfo fsInfo; + bool ret = FsInfoManager::GetInstance().GetFsInfo(fsId, &fsInfo); + if (!ret) { + LOG(ERROR) << "VolumeSpaceManager generate space manager failed, fsId " + << fsId << " not exist"; + return false; + } + + // init block device client + auto blockDeviceClient = std::make_shared(); + ret = blockDeviceClient->Init(options_.deviceOpt); + if (!ret) { + LOG(ERROR) << "VolumeSpaceManager init block device failed"; + return false; + } + const auto &vol = fsInfo.detail().volume(); + const auto &volName = vol.volumename(); + const auto &user = vol.user(); + ret = blockDeviceClient->Open(volName, user); + if (!ret) { + LOG(ERROR) << "VolumeSpaceManager open failed, volName=" << volName + << ", user=" << user; + return false; + } + + // init space manager option + SpaceManagerOption spaceManagerOpt; + spaceManagerOpt.blockGroupManagerOption.fsId = fsInfo.fsid(); + spaceManagerOpt.blockGroupManagerOption.blockGroupSize = + fsInfo.detail().volume().blockgroupsize(); + spaceManagerOpt.blockGroupManagerOption.blockSize = + fsInfo.detail().volume().blocksize(); + spaceManagerOpt.allocatorOption.type = "bitmap"; + + spaceManagers_[fsId] = std::make_shared( + spaceManagerOpt, options_.mdsClient, blockDeviceClient); + + return true; +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/src/metaserver/space/volume_space_manager.h b/curvefs/src/metaserver/space/volume_space_manager.h new file mode 100644 index 0000000000..ff3bc5f6f0 --- /dev/null +++ b/curvefs/src/metaserver/space/volume_space_manager.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Fri Mar 24 17:09:28 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_ +#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_ + +#include +#include +#include +#include + +#include "curvefs/src/volume/space_manager.h" + +namespace curvefs { +namespace metaserver { +using curvefs::client::rpcclient::MdsClient; +using curvefs::volume::BlockDeviceClientOptions; +using curvefs::volume::BlockDeviceClientImpl; +using curvefs::volume::Extent; +using curvefs::volume::SpaceManager; +using curvefs::volume::SpaceManagerOption; + +struct VolumeSpaceManagerOptions { + std::shared_ptr mdsClient; + BlockDeviceClientOptions deviceOpt; +}; + +class VolumeSpaceManager { + public: + void Init(VolumeSpaceManagerOptions options) { options_ = options; } + + void Uninit(); + + virtual bool + DeallocVolumeSpace(uint32_t fsId, uint64_t volumeOffset, + const std::vector &deallocatableVolumeSpace); + + virtual void Destroy(uint32_t fsId); + + virtual uint64_t GetBlockGroupSize(uint32_t fsId); + + private: + std::shared_ptr GetSpaceManager(uint32_t fsId); + + bool GenerateSpaceManager(uint32_t fsId); + + private: + std::mutex mutex_; + std::map> spaceManagers_; + VolumeSpaceManagerOptions options_; +}; + +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_ diff --git a/curvefs/src/metaserver/storage/converter.cpp b/curvefs/src/metaserver/storage/converter.cpp index ebcfc999ee..39a18dc3b0 100644 --- a/curvefs/src/metaserver/storage/converter.cpp +++ b/curvefs/src/metaserver/storage/converter.cpp @@ -54,6 +54,10 @@ static bool CompareType(const std::string& str, KEY_TYPE keyType) { NameGenerator::NameGenerator(uint32_t partitionId) : tableName4Inode_(Format(kTypeInode, partitionId)), + tableName4DeallocatableIndoe_( + Format(kTypeDeallocatableInode, partitionId)), + tableName4DeallocatableBlockGroup_( + Format(kTypeDeallocatableBlockGroup, partitionId)), tableName4S3ChunkInfo_(Format(kTypeS3ChunkInfo, partitionId)), tableName4Dentry_(Format(kTypeDentry, partitionId)), tableName4VolumeExtent_(Format(kTypeVolumeExtent, partitionId)), @@ -63,6 +67,14 @@ std::string NameGenerator::GetInodeTableName() const { return tableName4Inode_; } +std::string NameGenerator::GetDeallocatableInodeTableName() const { + return tableName4DeallocatableIndoe_; +} + +std::string NameGenerator::GetDeallocatableBlockGroupTableName() const { + return tableName4DeallocatableBlockGroup_; +} + std::string NameGenerator::GetS3ChunkInfoTableName() const { return tableName4S3ChunkInfo_; } @@ -340,6 +352,28 @@ bool Key4InodeAuxInfo::ParseFromString(const std::string& value) { StringToUl(items[1], &fsId) && StringToUll(items[2], &inodeId); } +std::string Key4DeallocatableBlockGroup::SerializeToString() const { + return absl::StrCat(keyType_, kDelimiter, fsId, kDelimiter, volumeOffset); +} + +bool Key4DeallocatableBlockGroup::ParseFromString(const std::string &value) { + std::vector items; + SplitString(value, kDelimiter, &items); + return items.size() == 3 && CompareType(items[0], keyType_) && + StringToUl(items[1], &fsId) && StringToUll(items[2], &volumeOffset); +} + +std::string Prefix4AllDeallocatableBlockGroup::SerializeToString() const { + return absl::StrCat(keyType_, ":"); +} + +bool Prefix4AllDeallocatableBlockGroup::ParseFromString( + const std::string &value) { + std::vector items; + SplitString(value, ":", &items); + return items.size() == 1 && CompareType(items[0], keyType_); +} + std::string Converter::SerializeToString(const StorageKey& key) { return key.SerializeToString(); } @@ -352,7 +386,6 @@ bool Converter::SerializeToString(const google::protobuf::Message& entry, } return entry.SerializeToString(value); } - } // namespace storage } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/storage/converter.h b/curvefs/src/metaserver/storage/converter.h index 0042db91eb..8d29471322 100644 --- a/curvefs/src/metaserver/storage/converter.h +++ b/curvefs/src/metaserver/storage/converter.h @@ -43,6 +43,9 @@ enum KEY_TYPE : unsigned char { kTypeDentry = 3, kTypeVolumeExtent = 4, kTypeInodeAuxInfo = 5, + kTypeBlockGroup = 6, + kTypeDeallocatableBlockGroup = 7, + kTypeDeallocatableInode = 8, }; // NOTE: you must generate all table name by NameGenerator class for @@ -54,6 +57,8 @@ class NameGenerator { std::string GetInodeTableName() const; + std::string GetDeallocatableInodeTableName() const; + std::string GetS3ChunkInfoTableName() const; std::string GetDentryTableName() const; @@ -62,6 +67,10 @@ class NameGenerator { std::string GetInodeAuxInfoTableName() const; + std::string GetBlockGroupStatisticTableName() const; + + std::string GetDeallocatableBlockGroupTableName() const; + static size_t GetFixedLength(); private: @@ -69,6 +78,8 @@ class NameGenerator { private: std::string tableName4Inode_; + std::string tableName4DeallocatableIndoe_; + std::string tableName4DeallocatableBlockGroup_; std::string tableName4S3ChunkInfo_; std::string tableName4Dentry_; std::string tableName4VolumeExtent_; @@ -84,7 +95,7 @@ class StorageKey { }; /* rules for key serialization: - * Key4Inode : kTypeInode:fsId:InodeId + * Key4Inode : kTypeInode:fsId:inodeId * Prefix4AllInode : kTypeInode: * Key4S3ChunkInfoList : kTypeS3ChunkInfo:fsId:inodeId:chunkIndex:firstChunkId:lastChunkId // NOLINT * Prefix4ChunkIndexS3ChunkInfoList : kTypeS3ChunkInfo:fsId:inodeId:chunkIndex: // NOLINT @@ -93,10 +104,12 @@ class StorageKey { * Key4Dentry : kTypeDentry:parentInodeId:name * Prefix4SameParentDentry : kTypeDentry:parentInodeId: * Prefix4AllDentry : kTypeDentry: - * Key4VolumeExtentSlice : kTypeExtent:fsId:InodeId:SliceOffset - * Prefix4InodeVolumeExtent : kTypeExtent:fsId:InodeId: + * Key4VolumeExtentSlice : kTypeExtent:fsId:inodeId:sliceOffset + * Prefix4InodeVolumeExtent : kTypeExtent:fsId:inodeId: * Prefix4AllVolumeExtent : kTypeExtent: * Key4InodeAuxInfo : kTypeInodeAuxInfo:fsId:inodeId + * Key4DeallocatableBlockGroup : kTypeBlockGroup:fsId:volumeOffset + * Prefix4AllDeallocatableBlockGroup: kTypeBlockGroup: */ class Key4Inode : public StorageKey { @@ -324,6 +337,37 @@ class Key4InodeAuxInfo : public StorageKey { static constexpr KEY_TYPE keyType_ = kTypeInodeAuxInfo; }; +class Key4DeallocatableBlockGroup : public StorageKey { + public: + Key4DeallocatableBlockGroup() = default; + + Key4DeallocatableBlockGroup(uint32_t fsId, uint64_t volumeOffset) + : fsId(fsId), volumeOffset(volumeOffset) {} + + std::string SerializeToString() const override; + + bool ParseFromString(const std::string &value) override; + + public: + uint32_t fsId; + uint64_t volumeOffset; + + private: + static constexpr KEY_TYPE keyType_ = kTypeDeallocatableBlockGroup; +}; + +class Prefix4AllDeallocatableBlockGroup : public StorageKey { + public: + Prefix4AllDeallocatableBlockGroup() = default; + + std::string SerializeToString() const override; + + bool ParseFromString(const std::string &value) override; + + public: + static const KEY_TYPE keyType_ = kTypeDeallocatableBlockGroup; +}; + // converter class Converter { public: diff --git a/curvefs/src/metaserver/storage/iterator.h b/curvefs/src/metaserver/storage/iterator.h index cd6e7393fe..a481bb89c0 100644 --- a/curvefs/src/metaserver/storage/iterator.h +++ b/curvefs/src/metaserver/storage/iterator.h @@ -72,6 +72,7 @@ class MergeIterator : public Iterator { for (const auto &child : children_) { size += child->Size(); } + return size; } diff --git a/curvefs/src/metaserver/streaming_utils.cpp b/curvefs/src/metaserver/streaming_utils.cpp index 4051fddd30..1684c53dc7 100644 --- a/curvefs/src/metaserver/streaming_utils.cpp +++ b/curvefs/src/metaserver/streaming_utils.cpp @@ -31,7 +31,7 @@ namespace curvefs { namespace metaserver { MetaStatusCode StreamingSendVolumeExtent(StreamConnection* connection, - const VolumeExtentList& extents) { + const VolumeExtentSliceList& extents) { VLOG(9) << "StreamingSendVolumeExtent, extents: " << extents.ShortDebugString(); for (const auto& slice : extents.slices()) { diff --git a/curvefs/src/metaserver/streaming_utils.h b/curvefs/src/metaserver/streaming_utils.h index fc97e7bc8f..2b32b47383 100644 --- a/curvefs/src/metaserver/streaming_utils.h +++ b/curvefs/src/metaserver/streaming_utils.h @@ -32,7 +32,7 @@ namespace metaserver { using ::curvefs::common::StreamConnection; MetaStatusCode StreamingSendVolumeExtent(StreamConnection* connection, - const VolumeExtentList& extents); + const VolumeExtentSliceList& extents); } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/volume/allocator.cpp b/curvefs/src/volume/allocator.cpp index 0ef437b583..67ae69be29 100644 --- a/curvefs/src/volume/allocator.cpp +++ b/curvefs/src/volume/allocator.cpp @@ -32,6 +32,7 @@ namespace volume { std::unique_ptr Allocator::Create(const std::string& type, const AllocatorOption& option) { + // TODO(@all): define bit map as a attribute of message Volume if (type == "bitmap") { return absl::make_unique(option.bitmapAllocatorOption); } else { diff --git a/curvefs/src/volume/block_group_loader.h b/curvefs/src/volume/block_group_loader.h index 22d1848631..0cf06fbf58 100644 --- a/curvefs/src/volume/block_group_loader.h +++ b/curvefs/src/volume/block_group_loader.h @@ -36,6 +36,7 @@ namespace curvefs { namespace volume { using ::curvefs::common::BitmapLocation; +using ::curvefs::mds::space::BlockGroup; class BlockDeviceClient; @@ -48,20 +49,15 @@ struct AllocatorAndBitmapUpdater { // load bitmap for each block group class BlockGroupBitmapLoader { public: - BlockGroupBitmapLoader(BlockDeviceClient* client, - uint32_t blockSize, - uint64_t offset, - uint64_t blockGroupSize, - BitmapLocation location, - bool clean, - const AllocatorOption& option) - : blockDev_(client), - offset_(offset), - blockGroupSize_(blockGroupSize), - blockSize_(blockSize), - bitmapLocation_(location), - clean_(clean), - allocatorOption_(option) {} + BlockGroupBitmapLoader(BlockDeviceClient *client, uint32_t blockSize, + const AllocatorOption &option, + const BlockGroup &blockGroup) + : blockDev_(client), blockSize_(blockSize), allocatorOption_(option) { + offset_ = blockGroup.offset(); + blockGroupSize_ = blockGroup.size(); + bitmapLocation_ = blockGroup.bitmaplocation(); + clean_ = (blockGroup.size() == blockGroup.available()); + } BlockGroupBitmapLoader(const BlockGroupBitmapLoader&) = delete; BlockGroupBitmapLoader& operator=(const BlockGroupBitmapLoader&) = delete; diff --git a/curvefs/src/volume/block_group_manager.cpp b/curvefs/src/volume/block_group_manager.cpp index 17760fcdaf..3249316a6f 100644 --- a/curvefs/src/volume/block_group_manager.cpp +++ b/curvefs/src/volume/block_group_manager.cpp @@ -20,14 +20,12 @@ * Author: wuhanqing */ -#include "curvefs/src/volume/block_group_manager.h" - #include - #include - #include "absl/memory/memory.h" #include "curvefs/proto/space.pb.h" +#include "curvefs/src/volume/block_group_manager.h" +#include "curvefs/src/volume/space_manager.h" namespace curvefs { namespace volume { @@ -35,6 +33,7 @@ namespace volume { using ::curve::common::Bitmap; using ::curve::common::BITMAP_UNIT_SIZE; using ::curve::common::BitRange; +using ::curve::common::WriteLockGuard; using ::curvefs::common::BitmapLocation; using ::curvefs::mds::space::BlockGroup; using ::curvefs::mds::space::SpaceErrCode_Name; @@ -56,24 +55,25 @@ bool BlockGroupManagerImpl::AllocateBlockGroup( std::vector groups; auto err = mdsClient_->AllocateVolumeBlockGroup( option_.fsId, option_.blockGroupAllocateOnce, option_.owner, &groups); - if (err != SpaceErrCode::SpaceOk) { - LOG(ERROR) << "Allocate volume block group failed, err: " - << SpaceErrCode_Name(err); - return false; - } else if (groups.empty()) { + + LOG_IF(ERROR, err != SpaceErrCode::SpaceOk) + << "Allocate volume block group failed, err: " + << SpaceErrCode_Name(err); + + if (groups.empty()) { LOG(ERROR) << "Allocate volume block group failed, no block group allocated"; return false; } - for (const auto& group : groups) { + WriteLockGuard lk(groupsLock_); + for (auto& group : groups) { VLOG(9) << "load group: " << group.ShortDebugString(); AllocatorAndBitmapUpdater res; res.blockGroupOffset = group.offset(); - BlockGroupBitmapLoader loader( - blockDeviceClient_.get(), option_.blockSize, group.offset(), - group.size(), group.bitmaplocation(), - group.size() == group.available(), allocatorOption_); + BlockGroupBitmapLoader loader(blockDeviceClient_.get(), + option_.blockSize, allocatorOption_, + group); auto ret = loader.Load(&res); if (!ret) { LOG(ERROR) << "Create allocator for block group failed"; @@ -81,13 +81,84 @@ bool BlockGroupManagerImpl::AllocateBlockGroup( } else { out->push_back(std::move(res)); } + + groups_.emplace_back(std::move(group)); + } + + return true; +} + +bool BlockGroupManagerImpl::AcquireBlockGroup(uint64_t blockGroupOffset, + AllocatorAndBitmapUpdater *out) { + BlockGroup group; + SpaceErrCode err = mdsClient_->AcquireVolumeBlockGroup( + option_.fsId, blockGroupOffset, option_.owner, &group); + if (err != SpaceErrCode::SpaceOk) { + LOG(WARNING) << "Acquire volume block group failed, err: " + << SpaceErrCode_Name(err); + return false; + } + + AllocatorAndBitmapUpdater res; + res.blockGroupOffset = group.offset(); + BlockGroupBitmapLoader loader(blockDeviceClient_.get(), option_.blockSize, + allocatorOption_, group); + auto ret = loader.Load(&res); + if (!ret) { + LOG(ERROR) << "Create allocator for block group failed"; + return false; + } + + *out = std::move(res); + return true; +} + + +bool BlockGroupManagerImpl::ReleaseBlockGroup(uint64_t blockGroupOffset, + uint64_t available) { + WriteLockGuard lk(groupsLock_); + auto iter = std::find_if(groups_.begin(), groups_.end(), + [blockGroupOffset](const BlockGroup &group) { + return group.offset() == blockGroupOffset; + }); + if (iter == groups_.end()) { + LOG(ERROR) << "BlockGroupManagerImpl find group block: " + << blockGroupOffset << " failed"; + return false; } + iter->set_available(available); + + SpaceErrCode err = mdsClient_->ReleaseVolumeBlockGroup( + option_.fsId, option_.owner, {*iter}); + if (err != SpaceErrCode::SpaceOk) { + LOG(WARNING) << "BlockGroupManagerImpl release block group: " + << blockGroupOffset + << "failed, err: " << SpaceErrCode_Name(err) + << ", blockGroupOffset: " << blockGroupOffset; + return false; + } + + groups_.erase(iter); + LOG(INFO) << "BlockGroupManagerImpl release one volume block group=" + << blockGroupOffset << " successfully"; return true; } -// TODO(wuhanqing): implement this function bool BlockGroupManagerImpl::ReleaseAllBlockGroups() { + WriteLockGuard lk(groupsLock_); + SpaceErrCode err = mdsClient_->ReleaseVolumeBlockGroup( + option_.fsId, option_.owner, groups_); + if (err != SpaceErrCode::SpaceOk) { + LOG(WARNING) << "BlockGroupManagerImpl release all volume block groups " + "failed, err: " + << SpaceErrCode_Name(err) << ""; + return false; + } + + groups_.clear(); + LOG(INFO) << "BlockGroupManagerImpl release all volume block groups " + "successfully"; return true; } diff --git a/curvefs/src/volume/block_group_manager.h b/curvefs/src/volume/block_group_manager.h index 5809ddce02..08f7a26997 100644 --- a/curvefs/src/volume/block_group_manager.h +++ b/curvefs/src/volume/block_group_manager.h @@ -49,6 +49,12 @@ class BlockGroupManager { std::vector* out) = 0; virtual bool ReleaseAllBlockGroups() = 0; + + virtual bool ReleaseBlockGroup(uint64_t blockGroupOffset, + uint64_t available) = 0; + + virtual bool AcquireBlockGroup(uint64_t blockGroupOffset, + AllocatorAndBitmapUpdater *out) = 0; }; class BlockGroupManagerImpl final : public BlockGroupManager { @@ -62,12 +68,16 @@ class BlockGroupManagerImpl final : public BlockGroupManager { bool AllocateBlockGroup( std::vector* out) override; - void AcquireBlockGroup(); + bool AcquireBlockGroup(uint64_t blockGroupOffset, + AllocatorAndBitmapUpdater *out) override; void AllocateBlockGroupAsync(); bool ReleaseAllBlockGroups() override; + bool ReleaseBlockGroup(uint64_t blockGroupOffset, + uint64_t available) override; + private: SpaceManager* spaceManager_; std::shared_ptr mdsClient_; @@ -75,6 +85,9 @@ class BlockGroupManagerImpl final : public BlockGroupManager { BlockGroupManagerOption option_; AllocatorOption allocatorOption_; + + curve::common::RWLock groupsLock_; + std::vector groups_; }; } // namespace volume diff --git a/curvefs/src/volume/option.h b/curvefs/src/volume/option.h index 01d6754253..9845dfda77 100644 --- a/curvefs/src/volume/option.h +++ b/curvefs/src/volume/option.h @@ -54,6 +54,9 @@ struct AllocatorOption { struct SpaceManagerOption { AllocatorOption allocatorOption; BlockGroupManagerOption blockGroupManagerOption; + + double threshold{1.0}; + uint64_t releaseInterSec{300}; }; } // namespace volume diff --git a/curvefs/src/volume/space_manager.cpp b/curvefs/src/volume/space_manager.cpp index 6df5933487..bee28a0654 100644 --- a/curvefs/src/volume/space_manager.cpp +++ b/curvefs/src/volume/space_manager.cpp @@ -41,20 +41,17 @@ using ::curve::common::ReadLockGuard; using ::curve::common::WriteLockGuard; SpaceManagerImpl::SpaceManagerImpl( - const SpaceManagerOption& option, - const std::shared_ptr& mdsClient, - const std::shared_ptr& blockDev) - : totalBytes_(0), - availableBytes_(0), + const SpaceManagerOption &option, + const std::shared_ptr &mdsClient, + const std::shared_ptr &blockDev) + : totalBytes_(0), availableBytes_(0), blockSize_(option.blockGroupManagerOption.blockSize), blockGroupSize_(option.blockGroupManagerOption.blockGroupSize), - blockGroupManager_( - new BlockGroupManagerImpl(this, - mdsClient, - blockDev, - option.blockGroupManagerOption, - option.allocatorOption)), - allocating_(false) {} + blockGroupManager_(new BlockGroupManagerImpl( + this, mdsClient, blockDev, option.blockGroupManagerOption, + option.allocatorOption)), + allocating_(false), threshold_(option.threshold), + releaseInterSec_(option.releaseInterSec) {} bool SpaceManagerImpl::Alloc(uint32_t size, const AllocateHint& hint, @@ -88,7 +85,7 @@ bool SpaceManagerImpl::Alloc(uint32_t size, left -= allocated; } - auto ret = UpdateBitmap(*extents); + auto ret = UpdateBitmap(*extents, BlockGroupBitmapUpdater::Set); if (!ret) { LOG(ERROR) << "Update bitmap failed"; metric_.errorCount << 1; @@ -104,9 +101,34 @@ bool SpaceManagerImpl::Alloc(uint32_t size, return true; } -bool SpaceManagerImpl::DeAlloc(const std::vector& extents) { - // TODO(wuhanqing): fix - (void)extents; +bool SpaceManagerImpl::DeAlloc(uint64_t blockGroupOffset, + const std::vector &extents) { + butil::Timer timer; + timer.start(); + + if (!AcquireBlockGroup(blockGroupOffset)) { + LOG(WARNING) << "Acquire block group failed, block group offset: " + << blockGroupOffset; + return false; + } + + bool ret = UpdateBitmap(extents, BlockGroupBitmapUpdater::Op::Clear); + if (!ret) { + LOG(ERROR) << "DeAlloc update bitmap failed"; + metric_.errorCount << 1; + return false; + } + + uint64_t size = 0; + for (const auto &ext : extents) { + size += ext.len; + } + + timer.stop(); + metric_.deallocLatency << timer.u_elapsed(); + metric_.allocSize << -size; + + VLOG(9) << "Dealloc success, " << extents; return true; } @@ -139,6 +161,11 @@ int64_t SpaceManagerImpl::AllocInternal(int64_t size, const AllocateHint& hint, std::vector* exts) { ReadLockGuard lk(allocatorsLock_); + if (allocators_.empty()) { + VLOG(6) << "space manager has no allocator"; + return 0; + } + int64_t left = size; auto it = FindAllocator(hint); const auto beginIt = it; @@ -162,13 +189,14 @@ int64_t SpaceManagerImpl::AllocInternal(int64_t size, return size - left; } -bool SpaceManagerImpl::UpdateBitmap(const std::vector& exts) { +bool SpaceManagerImpl::UpdateBitmap(const std::vector &exts, + BlockGroupBitmapUpdater::Op op) { ReadLockGuard lk(updatersLock_); std::unordered_set dirty; for (const auto& ext : exts) { BlockGroupBitmapUpdater* updater = FindBitmapUpdater(ext); - updater->Update(ext, BlockGroupBitmapUpdater::Set); + updater->Update(ext, op); dirty.insert(updater); } @@ -191,47 +219,111 @@ BlockGroupBitmapUpdater* SpaceManagerImpl::FindBitmapUpdater( return it->second.get(); } -bool SpaceManagerImpl::Shutdown() { - WriteLockGuard allocLk(allocatorsLock_); - WriteLockGuard updaterLk(updatersLock_); +void SpaceManagerImpl::Run() { + releaseT_ = std::thread(&SpaceManagerImpl::ReleaseFullBlockGroups, this); + running_ = true; +} - // sync all bitmap updater - bool ret = false; - for (auto& updater : bitmapUpdaters_) { - ret = updater.second->Sync(); - if (!ret) { - LOG(ERROR) << "Sync bitmap updater failed"; - return false; + +void SpaceManagerImpl::ReleaseFullBlockGroups() { + while (sleeper_.wait_for(std::chrono::seconds(releaseInterSec_))) { + std::vector selectBlockGroups; + + // find the blockgroup whose space usage ratio is greater than a certain + // threshold + { + ReadLockGuard lk(allocatorsLock_); + for (auto &alloc : allocators_) { + if (alloc.second->Total() == 0) { + continue; + } + + double total = alloc.second->Total(); + double available = alloc.second->AvailableSize(); + double usedPer = 1.0 - available / total; + VLOG(6) << "SpaceManagerImpl find block group=" << alloc.first + << " usedPer=" << usedPer << " total=" << (int64_t)total + << " available=" << (int64_t)available + << ", threshold_=" << threshold_; + + if (usedPer > threshold_) { + selectBlockGroups.push_back(alloc.first); + } + } } - } - // release all block group - ret = blockGroupManager_->ReleaseAllBlockGroups(); - LOG_IF(ERROR, !ret) << "Release all block groups failed"; - return ret; + if (selectBlockGroups.empty()) { + LOG(INFO) << "No block group need to release"; + continue; + } + + // release selected blockgroups + { + WriteLockGuard allocLk(allocatorsLock_); + WriteLockGuard updaterLk(updatersLock_); + for (auto &id : selectBlockGroups) { + auto iter = allocators_.find(id); + assert(iter != allocators_.end()); + auto availableSize = iter->second->AvailableSize(); + + availableBytes_.fetch_sub(availableSize, + std::memory_order_relaxed); + totalBytes_.fetch_sub(availableSize, std::memory_order_release); + + allocators_.erase(id); + bitmapUpdaters_.erase(id); + + bool ret = + blockGroupManager_->ReleaseBlockGroup(id, availableSize); + if (ret) { + VLOG(3) << "Release block group success, id: " << id; + } else { + VLOG(3) << "Release block group failed, id: " << id; + } + } + } + } } -bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) { +bool SpaceManagerImpl::Shutdown() { + bool ret = false; + { - std::unique_lock lk(mtx_); - if (allocating_) { - cond_.wait(lk); - } + WriteLockGuard allocLk(allocatorsLock_); + WriteLockGuard updaterLk(updatersLock_); - if (availableBytes_.load(std::memory_order_relaxed) >= hint) { - return true; + // sync all bitmap updater + for (auto &updater : bitmapUpdaters_) { + ret = updater.second->Sync(); + if (!ret) { + LOG(ERROR) << "Sync bitmap updater failed"; + return false; + } } + + // release all block group + ret = blockGroupManager_->ReleaseAllBlockGroups(); + LOG_IF(ERROR, !ret) << "Release all block groups failed"; } - std::lock_guard lk(mtx_); - allocating_ = true; + if (running_) { + sleeper_.interrupt(); + releaseT_.join(); + LOG(INFO) << "SpaceManagerImpl stop thread ok"; + } + return ret; +} + +bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) { + std::unique_lock lk(mtx_); + if (availableBytes_.load(std::memory_order_relaxed) >= hint) { + return true; + } std::vector out; auto ret = blockGroupManager_->AllocateBlockGroup(&out); if (!ret) { LOG(ERROR) << "Allocate block group failed"; - allocating_ = false; - cond_.notify_all(); return false; } @@ -239,7 +331,7 @@ bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) { uint64_t total = 0; WriteLockGuard allocLk(allocatorsLock_); WriteLockGuard updaterLk(updatersLock_); - for (auto& d : out) { + for (auto &d : out) { VLOG(9) << "add allocator, offset: " << d.blockGroupOffset << ", available: " << d.allocator->AvailableSize() << ", total: " << d.allocator->Total(); @@ -251,9 +343,28 @@ bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) { availableBytes_.fetch_add(available, std::memory_order_release); totalBytes_.fetch_add(total, std::memory_order_release); + return true; +} + +bool SpaceManagerImpl::AcquireBlockGroup(uint64_t blockGroupOffset) { + std::unique_lock lk(mtx_); + if (bitmapUpdaters_.find(blockGroupOffset) != bitmapUpdaters_.end()) { + return true; + } + + AllocatorAndBitmapUpdater out; + bool ret = blockGroupManager_->AcquireBlockGroup(blockGroupOffset, &out); + if (!ret) { + LOG(WARNING) << "Acquire block group failed, offset: " + << blockGroupOffset; + return false; + } + + WriteLockGuard allocLk(allocatorsLock_); + WriteLockGuard updaterLk(updatersLock_); - allocating_ = false; - cond_.notify_all(); + allocators_.emplace(blockGroupOffset, std::move(out.allocator)); + bitmapUpdaters_.emplace(blockGroupOffset, std::move(out.bitmapUpdater)); return true; } diff --git a/curvefs/src/volume/space_manager.h b/curvefs/src/volume/space_manager.h index cd24c70a58..09b04af033 100644 --- a/curvefs/src/volume/space_manager.h +++ b/curvefs/src/volume/space_manager.h @@ -27,6 +27,7 @@ #include #include +#include "src/common/interruptible_sleeper.h" #include "curvefs/proto/common.pb.h" #include "curvefs/src/client/rpcclient/mds_client.h" #include "curvefs/src/volume/allocator.h" @@ -39,6 +40,7 @@ namespace curvefs { namespace volume { using ::curvefs::client::rpcclient::MdsClient; +using ::curve::common::InterruptibleSleeper; class SpaceManager { public: @@ -48,9 +50,14 @@ class SpaceManager { const AllocateHint& hint, std::vector* extents) = 0; - virtual bool DeAlloc(const std::vector& extents) = 0; + virtual bool DeAlloc(uint64_t blockGroupOffset, + const std::vector &extents) = 0; + + virtual void Run() = 0; virtual bool Shutdown() = 0; + + virtual uint64_t GetBlockGroupSize() = 0; }; class SpaceManagerImpl final : public SpaceManager { @@ -67,13 +74,18 @@ class SpaceManagerImpl final : public SpaceManager { const AllocateHint& hint, std::vector* extents) override; - bool DeAlloc(const std::vector& extents) override; + bool DeAlloc(uint64_t blockGroupOffset, + const std::vector &extents) override; + + void Run() override; /** * @brief Shutdown space manager, release all blockgroups' rights */ bool Shutdown() override; + uint64_t GetBlockGroupSize() override { return blockGroupSize_; } + private: int64_t AllocInternal(int64_t size, const AllocateHint& hint, @@ -87,7 +99,10 @@ class SpaceManagerImpl final : public SpaceManager { */ BlockGroupBitmapUpdater* FindBitmapUpdater(const Extent& ext); - bool UpdateBitmap(const std::vector& exts); + bool UpdateBitmap(const std::vector &exts, + BlockGroupBitmapUpdater::Op op); + + void ReleaseFullBlockGroups(); private: bool AllocateBlockGroup(uint64_t hint); @@ -115,16 +130,25 @@ class SpaceManagerImpl final : public SpaceManager { std::mutex mtx_; std::condition_variable cond_; + // releaseT_ is periodically release the allocated blockgroup + double threshold_; + uint64_t releaseInterSec_; + InterruptibleSleeper sleeper_; + bool running_{false}; + std::thread releaseT_; + + private: struct Metric { bvar::LatencyRecorder allocLatency; + bvar::LatencyRecorder deallocLatency; bvar::LatencyRecorder allocSize; bvar::Adder errorCount; Metric() : allocLatency("space_alloc_latency"), - allocSize("space_alloc_size"), - errorCount("space_alloc_error") {} + deallocLatency("space_dealloc_latency"), + allocSize("space_alloc_size"), errorCount("space_alloc_error") {} }; Metric metric_; diff --git a/curvefs/test/client/mock_metaserver_client.h b/curvefs/test/client/mock_metaserver_client.h index 5b671463f1..b0fac7b1fe 100644 --- a/curvefs/test/client/mock_metaserver_client.h +++ b/curvefs/test/client/mock_metaserver_client.h @@ -152,11 +152,15 @@ class MockMetaServerClient : public MetaServerClient { MOCK_METHOD4(AsyncUpdateVolumeExtent, void(uint32_t, uint64_t, - const VolumeExtentList &, + const VolumeExtentSliceList &, MetaServerClientDone *)); - MOCK_METHOD4(GetVolumeExtent, - MetaStatusCode(uint32_t, uint64_t, bool, VolumeExtentList *)); + MOCK_METHOD4(GetVolumeExtent, MetaStatusCode(uint32_t, uint64_t, bool, + VolumeExtentSliceList *)); + + MOCK_METHOD3(UpdateDeallocatableBlockGroup, + MetaStatusCode(uint32_t, uint64_t, + DeallocatableBlockGroupMap *)); }; } // namespace rpcclient diff --git a/curvefs/test/client/rpcclient/metaserver_client_test.cpp b/curvefs/test/client/rpcclient/metaserver_client_test.cpp index 6076114953..4fdae51576 100644 --- a/curvefs/test/client/rpcclient/metaserver_client_test.cpp +++ b/curvefs/test/client/rpcclient/metaserver_client_test.cpp @@ -61,6 +61,8 @@ using ::curvefs::metaserver::BatchGetInodeAttrRequest; using ::curvefs::metaserver::BatchGetInodeAttrResponse; using ::curvefs::metaserver::BatchGetXAttrRequest; using ::curvefs::metaserver::BatchGetXAttrResponse; +using ::curvefs::metaserver::UpdateDeallocatableBlockGroupRequest; +using ::curvefs::metaserver::UpdateDeallocatableBlockGroupResponse; using ::curvefs::common::StreamServer; using ::curvefs::common::StreamOptions; using ::curvefs::common::StreamConnection; @@ -1087,7 +1089,7 @@ TEST_F(MetaServerClientImplTest, TestGetVolumeExtent) { const uint32_t partitionID = 200; for (auto streaming : {true, false}) { - metaserver::VolumeExtentList out; + metaserver::VolumeExtentSliceList out; EXPECT_CALL(*mockMetacache_, GetTarget(_, _, _, _)) .WillRepeatedly( @@ -1120,6 +1122,58 @@ TEST_F(MetaServerClientImplTest, TestGetVolumeExtent) { } } +TEST_F(MetaServerClientImplTest, TestUpdateDeallocatableBlockGroup) { + const uint32_t fsid = 1; + const uint64_t ino = 1; + const uint64_t blockgroupoffset = 0; + const uint64_t applyIndex = 10; + DeallocatableBlockGroupMap statistic; + auto &item = statistic[0]; + item.set_blockgroupoffset(blockgroupoffset); + auto increase = item.mutable_increase(); + increase->set_increasedeallocatablesize(1024 * 1024); + increase->add_inodeidlistadd(ino); + + // set response + UpdateDeallocatableBlockGroupResponse response; + response.set_statuscode(MetaStatusCode::OK); + response.set_appliedindex(applyIndex); + + { + // test rpc ok + EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(target_), Return(true))); + EXPECT_CALL(mockMetaServerService_, + UpdateDeallocatableBlockGroup(_, _, _, _)) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(SetRpcService))); + + ASSERT_EQ(MetaStatusCode::OK, + metaserverCli_.UpdateDeallocatableBlockGroup(fsid, ino, + &statistic)); + } + + { + // test rpc error + EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _)) + .WillRepeatedly(DoAll(SetArgPointee<2>(target_), Return(true))); + EXPECT_CALL(mockMetaServerService_, + UpdateDeallocatableBlockGroup(_, _, _, _)) + .WillRepeatedly( + Invoke(SetRpcService)); + EXPECT_CALL(*mockMetacache_.get(), GetTargetLeader(_, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _)) + .WillRepeatedly(DoAll(SetArgPointee<2>(target_), Return(true))); + ASSERT_EQ(MetaStatusCode::RPC_ERROR, + metaserverCli_.UpdateDeallocatableBlockGroup(fsid, ino, + &statistic)); + } +} + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/test/client/rpcclient/mock_metaserver_service.h b/curvefs/test/client/rpcclient/mock_metaserver_service.h index 02bf64addc..f40dd6b15f 100644 --- a/curvefs/test/client/rpcclient/mock_metaserver_service.h +++ b/curvefs/test/client/rpcclient/mock_metaserver_service.h @@ -119,6 +119,15 @@ class MockMetaServerService : public curvefs::metaserver::MetaServerService { const ::curvefs::metaserver::UpdateVolumeExtentRequest *request, ::curvefs::metaserver::UpdateVolumeExtentResponse *response, ::google::protobuf::Closure *done)); + + MOCK_METHOD4( + UpdateDeallocatableBlockGroup, + void(::google::protobuf::RpcController *controller, + const ::curvefs::metaserver::UpdateDeallocatableBlockGroupRequest + *request, + ::curvefs::metaserver::UpdateDeallocatableBlockGroupResponse + *response, + ::google::protobuf::Closure *done)); }; } // namespace rpcclient } // namespace client diff --git a/curvefs/test/client/test_inodeWrapper.cpp b/curvefs/test/client/test_inodeWrapper.cpp index b99faf50a6..868e1daeca 100644 --- a/curvefs/test/client/test_inodeWrapper.cpp +++ b/curvefs/test/client/test_inodeWrapper.cpp @@ -198,7 +198,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent) { EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _)) .Times(0); EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _)) - .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentList&, + .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList&, MetaServerClientDone* done) { done->SetMetaStatusCode(MetaStatusCode::OK); done->Run(); diff --git a/curvefs/test/client/volume/default_volume_storage_test.cpp b/curvefs/test/client/volume/default_volume_storage_test.cpp index 0eb3b4a5bf..3f8c0c5631 100644 --- a/curvefs/test/client/volume/default_volume_storage_test.cpp +++ b/curvefs/test/client/volume/default_volume_storage_test.cpp @@ -36,6 +36,10 @@ namespace curvefs { namespace client { +namespace common { +DECLARE_bool(enableCto); +} // namespace common + using ::curvefs::client::filesystem::FileOut; using ::curvefs::client::rpcclient::MockMetaServerClient; using ::curvefs::volume::AllocateHint; @@ -88,10 +92,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) { Inode inode; inode.set_type(FsFileType::TYPE_FILE); - VolumeExtentList exts; - auto* slice = exts.add_slices(); + VolumeExtentSliceList exts; + auto *slice = exts.add_slices(); slice->set_offset(0); - auto* ext = slice->add_extents(); + auto *ext = slice->add_extents(); ext->set_fsoffset(0); ext->set_length(4096); ext->set_volumeoffset(8192); @@ -100,8 +104,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { *extents = exts; return MetaStatusCode::OK; })); @@ -109,13 +113,12 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) { ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent()); EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); - EXPECT_CALL(blockDev_, Readv(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(blockDev_, Readv(_)).WillOnce(Return(-1)); uint64_t ino = 1; off_t offset = 0; @@ -130,10 +133,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) { Inode inode; inode.set_type(FsFileType::TYPE_FILE); - VolumeExtentList exts; - auto* slice = exts.add_slices(); + VolumeExtentSliceList exts; + auto *slice = exts.add_slices(); slice->set_offset(0); - auto* ext = slice->add_extents(); + auto *ext = slice->add_extents(); ext->set_fsoffset(0); ext->set_length(4096); ext->set_volumeoffset(8192); @@ -142,8 +145,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { *extents = exts; return MetaStatusCode::OK; })); @@ -151,7 +154,7 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) { ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent()); EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); @@ -161,11 +164,9 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) { size_t len = 4096; std::unique_ptr data(new char[len]); - EXPECT_CALL(blockDev_, Readv(_)) - .WillOnce(Return(len)); + EXPECT_CALL(blockDev_, Readv(_)).WillOnce(Return(len)); - EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) - .Times(1); + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1); ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Read(ino, offset, len, data.get())); } @@ -174,10 +175,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) { Inode inode; inode.set_type(FsFileType::TYPE_FILE); - VolumeExtentList exts; - auto* slice = exts.add_slices(); + VolumeExtentSliceList exts; + auto *slice = exts.add_slices(); slice->set_offset(0); - auto* ext = slice->add_extents(); + auto *ext = slice->add_extents(); ext->set_fsoffset(0); ext->set_length(4096); ext->set_volumeoffset(8192); @@ -186,8 +187,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { *extents = exts; return MetaStatusCode::OK; })); @@ -195,7 +196,7 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) { ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent()); EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); @@ -207,11 +208,9 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) { memset(data.get(), 'x', len); - EXPECT_CALL(blockDev_, Readv(_)) - .Times(0); + EXPECT_CALL(blockDev_, Readv(_)).Times(0); - EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) - .Times(1); + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1); ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Read(ino, offset, len, data.get())); @@ -227,8 +226,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_PrepareError) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { extents->clear_slices(); return MetaStatusCode::OK; })); @@ -236,13 +235,12 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_PrepareError) { ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent()); EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); - EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) - .WillOnce(Return(false)); + EXPECT_CALL(spaceMgr_, Alloc(_, _, _)).WillOnce(Return(false)); uint64_t ino = 1; off_t offset = 0; @@ -261,8 +259,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { extents->clear_slices(); return MetaStatusCode::OK; })); @@ -270,7 +268,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) { ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent()); EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); @@ -281,7 +279,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) { EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) .WillOnce(Invoke( - [](uint32_t size, const AllocateHint&, std::vector* exts) { + [](uint32_t size, const AllocateHint &, std::vector *exts) { exts->emplace_back(size, size); return true; })); @@ -292,8 +290,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) { std::unique_ptr data(new char[len]); FileOut fileOut; - EXPECT_CALL(blockDev_, Writev(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(blockDev_, Writev(_)).WillOnce(Return(-1)); ASSERT_EQ(CURVEFS_ERROR::IO_ERROR, storage_.Write(ino, offset, len, data.get(), &fileOut)); @@ -306,8 +303,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { auto inodeWrapper = std::make_shared(inode, metaServerCli_); EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _)) - .WillOnce( - Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) { + .WillOnce(Invoke( + [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) { extents->clear_slices(); return MetaStatusCode::OK; })); @@ -316,7 +313,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) - .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { out = inodeWrapper; return CURVEFS_ERROR::OK; })); @@ -327,7 +324,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) .WillOnce(Invoke( - [](uint32_t size, const AllocateHint&, std::vector* exts) { + [](uint32_t size, const AllocateHint &, std::vector *exts) { exts->emplace_back(size, size); return true; })); @@ -338,11 +335,9 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { std::unique_ptr data(new char[len]); FileOut fileOut; - EXPECT_CALL(blockDev_, Writev(_)) - .WillOnce(Return(len)); + EXPECT_CALL(blockDev_, Writev(_)).WillOnce(Return(len)); - EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) - .Times(1); + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1); ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Write(ino, offset, len, data.get(), &fileOut)); @@ -350,5 +345,36 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { ASSERT_EQ(offset + len, inodeWrapper->GetInode().length()); } +TEST_F(DefaultVolumeStorageTest, FlushTest) { + { + common::FLAGS_enableCto = false; + ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Flush(1)); + } + + { + common::FLAGS_enableCto = true; + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Return(CURVEFS_ERROR::UNKNOWN)); + + ASSERT_EQ(CURVEFS_ERROR::UNKNOWN, storage_.Flush(1)); + } + + { + Inode inode; + inode.set_type(FsFileType::TYPE_FILE); + auto inodeWrapper = + std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr &out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Flush(1)); + } +} + } // namespace client } // namespace curvefs diff --git a/curvefs/test/mds/fs_manager_test.cpp b/curvefs/test/mds/fs_manager_test.cpp index 28fb29550c..f23955f10e 100644 --- a/curvefs/test/mds/fs_manager_test.cpp +++ b/curvefs/test/mds/fs_manager_test.cpp @@ -32,6 +32,7 @@ #include "curvefs/test/mds/mock/mock_topology.h" #include "test/common/mock_s3_adapter.h" #include "curvefs/test/mds/mock/mock_space_manager.h" +#include "curvefs/test/mds/mock/mock_volume_space.h" #include "curvefs/test/mds/utils.h" using ::testing::AtLeast; @@ -44,6 +45,7 @@ using ::testing::SetArgPointee; using ::testing::SaveArg; using ::testing::Mock; using ::testing::Invoke; +using ::testing::Matcher; using ::curvefs::metaserver::MockMetaserverService; using ::curvefs::metaserver::CreateRootInodeRequest; using ::curvefs::metaserver::CreateRootInodeResponse; @@ -82,6 +84,7 @@ using ::curvefs::metaserver::copyset::MockCliService2; using ::curvefs::metaserver::copyset::GetLeaderResponse2; using ::curve::common::MockS3Adapter; using ::curvefs::mds::space::MockSpaceManager; +using ::curvefs::mds::space::MockVolumeSpace; namespace curvefs { namespace mds { @@ -541,9 +544,13 @@ TEST_F(FSManagerTest, test1) { ASSERT_TRUE(fsManager_->GetClientAliveTime(mountpath, &tpair)); ASSERT_EQ(fsName1, tpair.first); // test client timeout and restore session later + auto volumeSpace = new MockVolumeSpace(); { fsManager_->Run(); - EXPECT_CALL(*spaceManager_, RemoveVolume(_)) + EXPECT_CALL(*spaceManager_, GetVolumeSpace(_)) + .WillOnce(Return(volumeSpace)); + EXPECT_CALL(*volumeSpace, + ReleaseBlockGroups(Matcher(_))) .WillOnce(Return(space::SpaceOk)); // clientTimeoutSec in option sleep(4); @@ -581,18 +588,21 @@ TEST_F(FSManagerTest, test1) { // TEST UmountFs // umount UnInitSpace fail - EXPECT_CALL(*spaceManager_, RemoveVolume(_)) - .WillOnce(Return(space::SpaceErrNotFound)); + EXPECT_CALL(*spaceManager_, GetVolumeSpace(_)).WillOnce(Return(nullptr)); ret = fsManager_->UmountFs(fsName1, mountPoint); ASSERT_EQ(ret, FSStatusCode::UNINIT_SPACE_ERROR); // for persistence consider // umount UnInitSpace success - EXPECT_CALL(*spaceManager_, RemoveVolume(_)) + EXPECT_CALL(*spaceManager_, GetVolumeSpace(_)) + .WillOnce(Return(volumeSpace)); + EXPECT_CALL(*volumeSpace, + ReleaseBlockGroups(Matcher(_))) .WillOnce(Return(space::SpaceOk)); ret = fsManager_->UmountFs(fsName1, mountPoint); ASSERT_EQ(ret, FSStatusCode::OK); ASSERT_FALSE(fsManager_->GetClientAliveTime(mountpath, &tpair)); + delete volumeSpace; // umount not exist mountpoint ret = fsManager_->UmountFs(fsName1, mountPoint); diff --git a/curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp b/curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp index d1ffc35fa5..299ce4a58d 100644 --- a/curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp +++ b/curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp @@ -29,14 +29,19 @@ #include "curvefs/src/mds/heartbeat/metaserver_healthy_checker.h" #include "curvefs/test/mds/mock/mock_topology.h" #include "curvefs/test/mds/mock/mock_coordinator.h" +#include "curvefs/test/mds/mock/mock_space_manager.h" +#include "curvefs/test/mds/mock/mock_volume_space.h" #include "src/common/timeutility.h" using ::curvefs::mds::topology::MockIdGenerator; using ::curvefs::mds::topology::MockStorage; using ::curvefs::mds::topology::MockTokenGenerator; using ::curvefs::mds::topology::MockTopology; +using ::curvefs::mds::space::MockSpaceManager; +using ::curvefs::mds::space::MockVolumeSpace; using ::curvefs::mds::topology::TopoStatusCode; using ::curve::mds::heartbeat::ConfigChangeType; + using ::testing::_; using ::testing::DoAll; using ::testing::Return; @@ -58,8 +63,9 @@ class TestHeartbeatManager : public ::testing::Test { topology_ = std::make_shared(idGenerator_, tokenGenerator_, storage_); coordinator_ = std::make_shared(); - heartbeatManager_ = - std::make_shared(option, topology_, coordinator_); + spaceManager_ = std::make_shared(); + heartbeatManager_ = std::make_shared( + option, topology_, coordinator_, spaceManager_); } void TearDown() override {} @@ -69,6 +75,7 @@ class TestHeartbeatManager : public ::testing::Test { std::shared_ptr tokenGenerator_; std::shared_ptr storage_; std::shared_ptr topology_; + std::shared_ptr spaceManager_; std::shared_ptr heartbeatManager_; std::shared_ptr coordinator_; }; @@ -219,7 +226,6 @@ TEST_F(TestHeartbeatManager, test_updatespace_get_metaserver_fail2) { ASSERT_EQ(HeartbeatStatusCode::hbOK, response.statuscode()); } - TEST_F(TestHeartbeatManager, test_updatespace_get_server_fail) { ::curvefs::mds::topology::MetaServer metaServer( 1, "hostname", "hello", 1, "192.168.10.1", 9000, "", 9000); @@ -918,6 +924,58 @@ TEST_F(TestHeartbeatManager, test_update_partition) { heartbeatManager_->MetaServerHeartbeat(request, &response); ASSERT_EQ(HeartbeatStatusCode::hbOK, response.statuscode()); } + +TEST_F(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup) { + auto request = GetMetaServerHeartbeatRequestForTest(); + MetaServerHeartbeatResponse response; + + // prepare request + { + auto blockGroupStatInfo = request.add_blockgroupstatinfos(); + blockGroupStatInfo->set_fsid(1); + auto deallocatbleBlockGroup = + blockGroupStatInfo->add_deallocatableblockgroups(); + deallocatbleBlockGroup->set_blockgroupoffset(0); + deallocatbleBlockGroup->set_deallocatablesize(128 * 1024); + } + + // 1. get volumespace fail + { + EXPECT_CALL(*spaceManager_, GetVolumeSpace(1)) + .WillOnce(Return(nullptr)); + + heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response); + ASSERT_EQ(HeartbeatStatusCode::hbMetaServerFSUnkown, + response.statuscode()); + } + + // 2. no issued block group + { + MockVolumeSpace volumeSpace; + EXPECT_CALL(*spaceManager_, GetVolumeSpace(1)) + .WillOnce(Return(&volumeSpace)); + EXPECT_CALL(volumeSpace, UpdateDeallocatableBlockGroup(_, _, _, _)) + .WillOnce(Return(false)); + + heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response); + ASSERT_EQ(response.issuedblockgroups_size(), 0); + } + + // 3. has issued block group + { + MockVolumeSpace volumeSpace; + EXPECT_CALL(*spaceManager_, GetVolumeSpace(1)) + .WillOnce(Return(&volumeSpace)); + EXPECT_CALL(volumeSpace, UpdateDeallocatableBlockGroup(_, _, _, _)) + .WillOnce(DoAll(SetArgPointee<3>(1024 * 1024), Return(true))); + + heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response); + ASSERT_EQ(response.issuedblockgroups_size(), 1); + ASSERT_EQ(response.issuedblockgroups().at(1), 1024 * 1024); + } +} + } // namespace heartbeat } // namespace mds } // namespace curvefs + diff --git a/curvefs/test/mds/mds_service_test.cpp b/curvefs/test/mds/mds_service_test.cpp index ae46aa2aea..bbbfdea5b5 100644 --- a/curvefs/test/mds/mds_service_test.cpp +++ b/curvefs/test/mds/mds_service_test.cpp @@ -36,6 +36,7 @@ #include "curvefs/test/mds/mock/mock_cli2.h" #include "test/common/mock_s3_adapter.h" #include "curvefs/test/mds/mock/mock_space_manager.h" +#include "curvefs/test/mds/mock/mock_volume_space.h" #include "proto/nameserver2.pb.h" #include "curvefs/test/mds/utils.h" @@ -73,6 +74,7 @@ using ::curvefs::metaserver::FakeMetaserverImpl; using ::curvefs::metaserver::copyset::GetLeaderResponse2; using ::curvefs::metaserver::copyset::MockCliService2; + using ::testing::_; using ::testing::AtLeast; using ::testing::DoAll; @@ -87,6 +89,7 @@ using ::testing::StrEq; using ::curve::common::MockS3Adapter; using ::curvefs::mds::space::MockSpaceManager; +using ::curvefs::mds::space::MockVolumeSpace; using ::google::protobuf::util::MessageDifferencer; namespace curvefs { @@ -581,6 +584,12 @@ TEST_F(MdsServiceTest, test1) { } // TEST unmount + auto volumeSpace = new MockVolumeSpace(); + EXPECT_CALL(*spaceManager_, GetVolumeSpace(_)) + .WillRepeatedly(Return(volumeSpace)); + EXPECT_CALL(*volumeSpace, + ReleaseBlockGroups(Matcher(_))) + .WillRepeatedly(Return(space::SpaceOk)); cntl.Reset(); UmountFsRequest umountRequest; UmountFsResponse umountResponse; @@ -740,6 +749,7 @@ TEST_F(MdsServiceTest, test1) { // stop rpc server server.Stop(10); server.Join(); + delete volumeSpace; } } // namespace mds } // namespace curvefs diff --git a/curvefs/test/mds/mock/mock_volume_space.h b/curvefs/test/mds/mock/mock_volume_space.h index cd40d3b671..2f07ce2e07 100644 --- a/curvefs/test/mds/mock/mock_volume_space.h +++ b/curvefs/test/mds/mock/mock_volume_space.h @@ -48,6 +48,14 @@ class MockVolumeSpace : public AbstractVolumeSpace { MOCK_METHOD1(ReleaseBlockGroups, SpaceErrCode(const std::vector& blockGroups)); + + MOCK_METHOD1(ReleaseBlockGroups, SpaceErrCode(const std::string &)); + + MOCK_METHOD4(UpdateDeallocatableBlockGroup, + bool(uint32_t metaserverId, + const DeallocatableBlockGroupVec &groups, + const BlockGroupDeallcateStatusMap &stats, + uint64_t *issue)); }; } // namespace space diff --git a/curvefs/test/mds/space/space_manager_test.cpp b/curvefs/test/mds/space/space_manager_test.cpp index 1b4c0877bb..9e6de46e90 100644 --- a/curvefs/test/mds/space/space_manager_test.cpp +++ b/curvefs/test/mds/space/space_manager_test.cpp @@ -71,7 +71,7 @@ class SpaceManagerTest : public ::testing::Test { etcdclient_ = std::make_shared(); fsStorage_ = std::make_shared(); spaceManager_ = - absl::make_unique(etcdclient_, fsStorage_); + absl::make_unique(etcdclient_, fsStorage_, 1); } void TearDown() override {} @@ -103,7 +103,8 @@ TEST_F(SpaceManagerTest, TestGetVolumeSpace) { // add one volume space { AddOneNewVolume(); - ASSERT_NE(nullptr, spaceManager_->GetVolumeSpace(kFsId)); + auto space = spaceManager_->GetVolumeSpace(kFsId); + ASSERT_NE(nullptr, space); } } @@ -116,7 +117,7 @@ TEST_F(SpaceManagerTest, TestAddVolume_AlreadyExists) { using type = std::vector*; EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))).Times(0); - ASSERT_EQ(SpaceErrExist, spaceManager_->AddVolume(fsInfo)); + ASSERT_EQ(SpaceOk, spaceManager_->AddVolume(fsInfo)); } TEST_F(SpaceManagerTest, TestAddVolume_LoadBlockGroupsError) { diff --git a/curvefs/test/mds/space/space_service_test.cpp b/curvefs/test/mds/space/space_service_test.cpp index f058d6ccb8..6b90600a75 100644 --- a/curvefs/test/mds/space/space_service_test.cpp +++ b/curvefs/test/mds/space/space_service_test.cpp @@ -41,6 +41,7 @@ using ::curvefs::test::GenerateAnDefaultInitializedMessage; using ::testing::_; using ::testing::Invoke; using ::testing::Return; +using ::testing::Matcher; static constexpr uint32_t kRetryTimes = 10; static unsigned int seed = time(nullptr); @@ -189,7 +190,9 @@ TEST_F(SpaceServiceTest, ReleaseBlockGroupTest) { EXPECT_CALL(spaceMgr_, GetVolumeSpace(_)) .WillOnce(Return(&volumeSpace)); - EXPECT_CALL(volumeSpace, ReleaseBlockGroups(_)) + EXPECT_CALL( + volumeSpace, + ReleaseBlockGroups(Matcher &>(_))) .WillOnce(Return(err)); auto request = GenerateAnDefaultInitializedMessage( diff --git a/curvefs/test/mds/space/volume_space_test.cpp b/curvefs/test/mds/space/volume_space_test.cpp index 6d7df74369..38c2004f8d 100644 --- a/curvefs/test/mds/space/volume_space_test.cpp +++ b/curvefs/test/mds/space/volume_space_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "absl/memory/memory.h" #include "curvefs/proto/common.pb.h" @@ -42,8 +43,8 @@ namespace space { using ::testing::_; using ::testing::Invoke; -using ::testing::Return; using ::testing::Matcher; +using ::testing::Return; static constexpr uint32_t kFsId = 1; static constexpr uint64_t kVolumeSize = 10ULL * 1024 * 1024 * 1024; @@ -53,7 +54,7 @@ static constexpr curvefs::common::BitmapLocation kBitmapLocation = curvefs::common::BitmapLocation::AtStart; static constexpr uint32_t kAllocateOnce = 4; static constexpr double kExtendFactor = 1.5; -static const char* kOwner = "test"; +static const char *kOwner = "test"; static unsigned int seed = time(nullptr); @@ -72,19 +73,21 @@ class VolumeSpaceTest : public ::testing::Test { std::unique_ptr CreateOneEmptyVolumeSpace() { EXPECT_CALL(*storage_, ListBlockGroups(_, _)) - .WillOnce(Invoke([](uint32_t, std::vector* groups) { + .WillOnce(Invoke([](uint32_t, std::vector *groups) { groups->clear(); return SpaceOk; })); + // disable background threads during testing. auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(), - fsStorage_.get()); + fsStorage_.get(), 60000); EXPECT_NE(nullptr, space); return space; } - std::unique_ptr CreateVolumeSpaceWithTwoGroups( - bool allocated = true, std::vector* out = nullptr) { + std::unique_ptr + CreateVolumeSpaceWithTwoGroups(bool allocated = true, + std::vector *out = nullptr) { BlockGroup group1; group1.set_offset(0); group1.set_size(kBlockGroupSize); @@ -99,7 +102,6 @@ class VolumeSpaceTest : public ::testing::Test { group2.set_size(kBlockGroupSize); group2.set_available(kBlockGroupSize / 2); group2.set_bitmaplocation(kBitmapLocation); - group2.set_owner(kOwner); if (allocated) { group2.set_owner(kOwner); } @@ -112,17 +114,75 @@ class VolumeSpaceTest : public ::testing::Test { EXPECT_CALL(*storage_, ListBlockGroups(_, _)) .WillOnce( - Invoke([&exist](uint32_t, std::vector* groups) { + Invoke([&exist](uint32_t, std::vector *groups) { *groups = exist; return SpaceOk; })); auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(), - fsStorage_.get()); + fsStorage_.get(), 1); EXPECT_NE(nullptr, space); return space; } + void + UpdateBlockGroupSpaceInfo(uint32_t metaserverId, + std::unique_ptr &space, // NOLINT + bool needIssued = false, + uint64_t expectIssued = 0) { + BlockGroupDeallcateStatusMap stats; + DeallocatableBlockGroupVec groups; + { + DeallocatableBlockGroup group1; + group1.set_blockgroupoffset(0); + group1.set_deallocatablesize(kBlockGroupSize / 2); + groups.Add()->CopyFrom(group1); + + + DeallocatableBlockGroup group2; + group2.set_blockgroupoffset(kBlockGroupSize); + group2.set_deallocatablesize(kBlockGroupSize / 4); + groups.Add()->CopyFrom(group2); + } + + uint64_t issued = 0; + if (needIssued) { + ASSERT_TRUE(space->UpdateDeallocatableBlockGroup( + metaserverId, groups, stats, &issued)); + ASSERT_EQ(issued, expectIssued); + + } else { + ASSERT_FALSE(space->UpdateDeallocatableBlockGroup( + metaserverId, groups, stats, &issued)); + } + } + + void + UpdateBlockGroupIssueStat(uint32_t metaserverId, + std::unique_ptr &space, // NOLINT + uint64_t blockoffset, + BlockGroupDeallcateStatusCode status, + bool needIssue = true) { + DeallocatableBlockGroupVec groups; + BlockGroupDeallcateStatusMap stats; + stats[blockoffset] = status; + if (status == BlockGroupDeallcateStatusCode::BGDP_DONE) { + DeallocatableBlockGroup group; + group.set_blockgroupoffset(blockoffset); + group.set_deallocatablesize(0); + groups.Add()->CopyFrom(group); + } + + uint64_t issued = 0; + if (needIssue) { + ASSERT_TRUE(space->UpdateDeallocatableBlockGroup( + metaserverId, groups, stats, &issued)); + } else { + ASSERT_FALSE(space->UpdateDeallocatableBlockGroup( + metaserverId, groups, stats, &issued)); + } + } + protected: std::unique_ptr storage_; std::unique_ptr fsStorage_; @@ -133,20 +193,20 @@ TEST_F(VolumeSpaceTest, TestCreate_ListError) { EXPECT_CALL(*storage_, ListBlockGroups(_, _)) .WillOnce(Return(SpaceErrStorage)); - auto space = - VolumeSpace::Create(kFsId, volume_, storage_.get(), fsStorage_.get()); + auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(), + fsStorage_.get(), 1); EXPECT_EQ(nullptr, space); } TEST_F(VolumeSpaceTest, TestCreate_Success) { EXPECT_CALL(*storage_, ListBlockGroups(_, _)) - .WillOnce(Invoke([](uint32_t, std::vector* groups) { + .WillOnce(Invoke([](uint32_t, std::vector *groups) { groups->clear(); return SpaceOk; })); - auto space = - VolumeSpace::Create(kFsId, volume_, storage_.get(), fsStorage_.get()); + auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(), + fsStorage_.get(), 1); EXPECT_NE(nullptr, space); } @@ -186,7 +246,7 @@ TEST_F(VolumeSpaceTest, TestAllocateBlockGroups) { } std::set offsets; - for (auto& group : groups) { + for (auto &group : groups) { ASSERT_EQ(kOwner, group.owner()); offsets.insert(group.offset()); } @@ -194,6 +254,7 @@ TEST_F(VolumeSpaceTest, TestAllocateBlockGroups) { ASSERT_EQ(0, *offsets.begin()); ASSERT_EQ(kVolumeSize - kBlockGroupSize, *offsets.rbegin()); + groups.clear(); ASSERT_EQ(SpaceErrNoSpace, space->AllocateBlockGroups(kAllocateOnce, kOwner, &groups)); } @@ -225,10 +286,10 @@ TEST_F(VolumeSpaceTest, TestAutoExtendVolume_ExtendError) { namespace { class FakeCurveFSService : public curve::mds::CurveFSService { public: - void ExtendFile(::google::protobuf::RpcController* controller, - const ::curve::mds::ExtendFileRequest* request, - ::curve::mds::ExtendFileResponse* response, - ::google::protobuf::Closure* done) override { + void ExtendFile(::google::protobuf::RpcController *controller, + const ::curve::mds::ExtendFileRequest *request, + ::curve::mds::ExtendFileResponse *response, + ::google::protobuf::Closure *done) override { brpc::ClosureGuard guard(done); if (request->newsize() % kBlockGroupSize != 0) { response->set_statuscode(curve::mds::kParaError); @@ -304,16 +365,14 @@ TEST_F(VolumeSpaceTest, TestAutoExtendVolumeSuccess) { EXPECT_CALL(*fsStorage_, Get(Matcher(_), _)) .WillOnce(Return(FSStatusCode::OK)); - EXPECT_CALL(*fsStorage_, Update(_)) - .WillOnce(Return(FSStatusCode::OK)); + EXPECT_CALL(*fsStorage_, Update(_)).WillOnce(Return(FSStatusCode::OK)); EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) .Times(1) .WillRepeatedly(Return(SpaceOk)); std::vector newGroups; - ASSERT_EQ(SpaceOk, - space->AllocateBlockGroups(1, kOwner, &newGroups)); + ASSERT_EQ(SpaceOk, space->AllocateBlockGroups(1, kOwner, &newGroups)); ASSERT_EQ(totalGroups + 1, space->allocatedGroups_.size()); ASSERT_TRUE(space->availableGroups_.empty()); @@ -439,9 +498,7 @@ TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsNotUsed_ButClearError) { ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists)); } -MATCHER(NoOwner, "") { - return !arg.has_owner(); -} +MATCHER(NoOwner, "") { return !arg.has_owner(); } TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed) { std::vector exists; @@ -470,6 +527,121 @@ TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed_PutError) { ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists)); } +TEST_F(VolumeSpaceTest, Test_CalBlockGroupAvailableForDeAllocate) { + // 1. test cal none + { + auto space = CreateOneEmptyVolumeSpace(); + sleep(1); + ASSERT_EQ(0, space->waitDeallocateGroups_.size()); + + space = CreateVolumeSpaceWithTwoGroups(false); + sleep(1); + ASSERT_EQ(0, space->waitDeallocateGroups_.size()); + ASSERT_EQ(2, space->availableGroups_.size()); + } + + // 2. test cal one and issue + { + // cal one + uint32_t metaserverId1 = 1; + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(false, &exists); + UpdateBlockGroupSpaceInfo(metaserverId1, space); + space->CalBlockGroupAvailableForDeAllocate(); + ASSERT_EQ(1, space->waitDeallocateGroups_.size()); + ASSERT_EQ(exists[0].offset(), space->waitDeallocateGroups_[0].offset()); + ASSERT_EQ(1, space->availableGroups_.size()); + ASSERT_EQ(exists[0].offset(), space->availableGroups_[0].offset()); + ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[0].offset()]); + ASSERT_EQ(kBlockGroupSize / 4, space->summary_[exists[1].offset()]); + + // issue to metaserver1 + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + UpdateBlockGroupSpaceInfo(metaserverId1, space, true, + exists[0].offset()); + ASSERT_TRUE(space->waitDeallocateGroups_.empty()); + ASSERT_EQ(1, space->deallocatingGroups_.size()); + ASSERT_EQ( + metaserverId1, + space->deallocatingGroups_[exists[0].offset()].deallocating()[0]); + + // issue to metaserver2 + uint32_t metaserverId2 = 2; + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + UpdateBlockGroupSpaceInfo(metaserverId2, space, true, + exists[0].offset()); + ASSERT_EQ( + metaserverId1, + space->deallocatingGroups_[exists[0].offset()].deallocating()[0]); + + ASSERT_EQ( + metaserverId2, + space->deallocatingGroups_[exists[0].offset()].deallocating()[1]); + ASSERT_EQ(kBlockGroupSize, space->summary_[exists[0].offset()]); + ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]); + + // metaserver2 report, update summary and be issued again + UpdateBlockGroupSpaceInfo(metaserverId2, space, true, + exists[0].offset()); + ASSERT_EQ(kBlockGroupSize, space->summary_[exists[0].offset()]); + ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]); + + // metaserver1 report issued stat: BGDP_PROCESSING + UpdateBlockGroupIssueStat( + metaserverId1, space, exists[0].offset(), + BlockGroupDeallcateStatusCode::BGDP_PROCESSING); + ASSERT_EQ(2, space->deallocatingGroups_[exists[0].offset()] + .deallocating() + .size()); + ASSERT_EQ(2, space->summary_.size()); + + // metaserver1 report issued stat: BGDP_DONE + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + UpdateBlockGroupIssueStat(metaserverId1, space, exists[0].offset(), + BlockGroupDeallcateStatusCode::BGDP_DONE, + false); + ASSERT_EQ( + metaserverId2, + space->deallocatingGroups_[exists[0].offset()].deallocating()[0]); + ASSERT_EQ( + metaserverId1, + space->deallocatingGroups_[exists[0].offset()].deallocated()[0]); + ASSERT_EQ(2, space->summary_.size()); + + // metaserver2 report issued stat: BGDP_DONE + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + UpdateBlockGroupIssueStat(metaserverId2, space, exists[0].offset(), + BlockGroupDeallcateStatusCode::BGDP_DONE, + false); + ASSERT_EQ( + metaserverId1, + space->deallocatingGroups_[exists[0].offset()].deallocated()[0]); + ASSERT_EQ( + metaserverId2, + space->deallocatingGroups_[exists[0].offset()].deallocated()[1]); + ASSERT_EQ(1, space->summary_.size()); + ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]); + + // metaserver1 and metaserver2 process done + EXPECT_CALL(*storage_, PutBlockGroup(_, exists[0].offset(), _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + space->CalBlockGroupAvailableForDeAllocate(); + ASSERT_EQ(1, space->waitDeallocateGroups_.count(exists[1].offset())); + ASSERT_EQ(1, space->waitDeallocateGroups_.size()); + ASSERT_EQ(1, space->availableGroups_.count(exists[0].offset())); + ASSERT_EQ(1, space->availableGroups_.size()); + } +} + } // namespace space } // namespace mds } // namespace curvefs diff --git a/curvefs/test/metaserver/BUILD b/curvefs/test/metaserver/BUILD index e930cb5ae4..66e3bc70e8 100644 --- a/curvefs/test/metaserver/BUILD +++ b/curvefs/test/metaserver/BUILD @@ -85,10 +85,7 @@ cc_test( "//curvefs/src/metaserver:curvefs_metaserver", "@com_google_googletest//:gtest_main", "@com_google_googletest//:gtest", - # # "//external:brpc", - # "//curvefs/proto:curvefs_topology_cc_proto", "//curvefs/test/metaserver/copyset/mock:metaserver_copyset_test_mock", - # "//curvefs/src/metaserver:metaserver_s3_lib", "//curvefs/test/client/rpcclient:rpcclient_test_mock", "//curvefs/test/client:mock", "//curvefs/test/metaserver/storage:metaserver_storage_test_utils", @@ -143,5 +140,6 @@ cc_test( "@com_google_googletest//:gtest", "//external:brpc", "//curvefs/test/metaserver/storage:metaserver_storage_test_utils", + "//curvefs/test/client/rpcclient:rpcclient_test_mock", ], ) diff --git a/curvefs/test/metaserver/copyset/copyset_node_block_group_test.cpp b/curvefs/test/metaserver/copyset/copyset_node_block_group_test.cpp new file mode 100644 index 0000000000..4dec486122 --- /dev/null +++ b/curvefs/test/metaserver/copyset/copyset_node_block_group_test.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Mon Apr 24 14:30:20 CST 2023 + * Author: lixiaocui + */ + +#include + +#include "curvefs/src/metaserver/copyset/copyset_node.h" +#include "curvefs/src/metaserver/copyset/copyset_node_manager.h" +#include "curvefs/test/metaserver/mock/mock_partition.h" +#include "curvefs/test/metaserver/mock/mock_metastore.h" +#include "src/common/uuid.h" +#include "test/fs/mock_local_filesystem.h" + +namespace curvefs { +namespace metaserver { +namespace copyset { + +using ::curve::common::UUIDGenerator; +using ::curve::fs::MockLocalFileSystem; +using ::curvefs::metaserver::mock::MockPartition; +using ::testing::_; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::DoAll; + +class CopysetNodeBlockGroupTest : public testing::Test { + protected: + void SetUp() override { + dataPath_ = "./runlog/" + UUIDGenerator{}.GenerateUUID(); + trashPath_ = dataPath_ + "_trash"; + mockfs_ = absl::make_unique(); + nodeManager_ = &CopysetNodeManager::GetInstance(); + mockMetaStore_ = absl::make_unique(); + + poolId_ = time(nullptr) % 12345; + copysetId_ = time(nullptr) % reinterpret_cast(this); + + options_.ip = "127.0.0.1"; + options_.port = 6666; + + options_.dataUri = "local://" + dataPath_; + options_.raftNodeOptions.log_uri = "local://" + dataPath_; + options_.raftNodeOptions.raft_meta_uri = "local://" + dataPath_; + options_.raftNodeOptions.snapshot_uri = "local://" + dataPath_; + + // disable raft snapshot + options_.raftNodeOptions.snapshot_interval_s = -1; + + options_.localFileSystem = mockfs_.get(); + + options_.trashOptions.trashUri = "local://" + trashPath_; + options_.storageOptions.type = "memory"; + + butil::ip_t ip; + ASSERT_EQ(0, butil::str2ip(options_.ip.c_str(), &ip)); + butil::EndPoint listenAddr(ip, 6666); + + ASSERT_TRUE(nodeManager_->Init(options_)); + } + + void TearDown() { + system(std::string("rm -rf " + dataPath_).c_str()); + } + + protected: + PoolId poolId_; + CopysetId copysetId_; + std::string dataPath_; + std::string trashPath_; + + CopysetNodeOptions options_; + CopysetNodeManager *nodeManager_; + + std::unique_ptr mockfs_; + std::unique_ptr mockMetaStore_; +}; + +TEST_F(CopysetNodeBlockGroupTest, Test_AggregateBlockStatInfo) { + auto partition = std::make_shared(); + std::map blockStatInfoMap; + uint32_t blockGroupNum = 0; + uint32_t fsId = 1; + uint32_t partitionId = 1; + + auto copyset = nodeManager_->GetCopysetNode(poolId_, copysetId_); + + // get fail + { + EXPECT_CALL(*partition, GetFsId()).WillOnce(Return(fsId)); + EXPECT_CALL(*partition, GetAllBlockGroup(_)) + .WillOnce(Return(MetaStatusCode::PARAM_ERROR)); + EXPECT_CALL(*partition, GetPartitionId()) + .Times(2) + .WillRepeatedly(Return(partitionId)); + ASSERT_FALSE(copyset->AggregateBlockStatInfo( + partition, &blockStatInfoMap, &blockGroupNum)); + ASSERT_EQ(0, blockGroupNum); + } + + // get success + { + std::vector in; + DeallocatableBlockGroup bg; + bg.set_blockgroupoffset(0); + bg.set_deallocatablesize(1024); + bg.add_inodeidlist(1); + in.emplace_back(bg); + + EXPECT_CALL(*partition, GetFsId()).WillOnce(Return(fsId)); + EXPECT_CALL(*partition, GetAllBlockGroup(_)) + .WillOnce(DoAll(SetArgPointee<0>(in), Return(MetaStatusCode::OK))); + ASSERT_TRUE(copyset->AggregateBlockStatInfo( + partition, &blockStatInfoMap, &blockGroupNum)); + ASSERT_EQ(1, blockGroupNum); + ASSERT_EQ(1, blockStatInfoMap.size()); + auto &out = blockStatInfoMap[fsId]; + ASSERT_EQ(fsId, out.fsid()); + ASSERT_EQ(1, out.deallocatableblockgroups_size()); + } +} + +} // namespace copyset +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp b/curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp index 8eb1b61925..a4e4f83026 100644 --- a/curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp +++ b/curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp @@ -402,7 +402,15 @@ void RunGetPartitionInfoList(CopysetNode* node, uint32_t sleepSec, ASSERT_EQ(node->GetPartitionInfoList(&partitionInfoList), expectedValue); } -// get partition list while copyset is loading +void RunGetBlockStatInfo(CopysetNode *node, uint32_t sleepSec, + bool expectedValue) { + sleep(sleepSec); + std::map blockStatInfoMap; + + ASSERT_EQ(node->GetBlockStatInfo(&blockStatInfoMap), expectedValue); +} + +// get partition list / get block statinfo while copyset is loading TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess1) { ASSERT_TRUE(CreateOneCopyset()); @@ -431,8 +439,10 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess1) { std::thread thread1(RunOnSnapshotLoad, node, &reader, 0); std::thread thread2(RunGetPartitionInfoList, node, 3, false); + std::thread thread3(RunGetBlockStatInfo, node, 3, false); thread1.join(); thread2.join(); + thread3.join(); node->SetMetaStore(nullptr); } @@ -465,11 +475,18 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess2) { .Times(1); EXPECT_CALL(*mockMetaStore, GetPartitionInfoList(_)) .WillOnce(Return(true)); + EXPECT_CALL(*mockMetaStore, GetPartitionSnap(_)) + .Times(2) + .WillOnce(Return(false)) + .WillOnce(Return(true)); std::thread thread1(RunOnSnapshotLoad, node, &reader, 0); std::thread thread2(RunGetPartitionInfoList, node, 3, true); + std::thread thread3(RunGetBlockStatInfo, node, 5, true); + thread1.join(); thread2.join(); + thread3.join(); node->SetMetaStore(nullptr); } @@ -510,6 +527,7 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess3) { node->SetMetaStore(nullptr); } + } // namespace copyset } // namespace metaserver } // namespace curvefs diff --git a/curvefs/test/metaserver/copyset/meta_operator_test.cpp b/curvefs/test/metaserver/copyset/meta_operator_test.cpp index 935ba64c6b..010e69ef38 100644 --- a/curvefs/test/metaserver/copyset/meta_operator_test.cpp +++ b/curvefs/test/metaserver/copyset/meta_operator_test.cpp @@ -147,6 +147,8 @@ TEST_F(MetaOperatorTest, OperatorTypeTest) { TEST_OPERATOR_TYPE(CreatePartition); TEST_OPERATOR_TYPE(DeletePartition); TEST_OPERATOR_TYPE(PrepareRenameTx); + TEST_OPERATOR_TYPE(UpdateDeallocatableBlockGroup); + #undef TEST_OPERATOR_TYPE } diff --git a/curvefs/test/metaserver/copyset/mock/mock_copyset_node.h b/curvefs/test/metaserver/copyset/mock/mock_copyset_node.h index aca102da41..f5d49379c3 100644 --- a/curvefs/test/metaserver/copyset/mock/mock_copyset_node.h +++ b/curvefs/test/metaserver/copyset/mock/mock_copyset_node.h @@ -26,6 +26,8 @@ #include #include +#include +#include #include "curvefs/src/metaserver/copyset/copyset_node.h" @@ -46,6 +48,17 @@ class MockCopysetNode : public CopysetNode { MOCK_CONST_METHOD0(IsLeaderTerm, bool()); MOCK_METHOD1(Propose, void(const braft::Task& task)); MOCK_METHOD(void, GetLeaderLeaseStatus, (braft::LeaderLeaseStatus*), (override)); // NOLINT + MOCK_METHOD2(GetConfChange, void(ConfigChangeType *type, Peer *alterPeer)); + + MOCK_CONST_METHOD0(GetPoolId, PoolId()); + MOCK_CONST_METHOD0(GetPeerId, const braft::PeerId &()); + MOCK_CONST_METHOD0(GetLeaderId, PeerId()); + MOCK_CONST_METHOD0(GetCopysetId, CopysetId()); + + MOCK_METHOD1(GetPartitionInfoList, bool(std::list *)); + MOCK_CONST_METHOD0(IsLoading, bool()); + MOCK_METHOD1(GetBlockStatInfo, + bool(std::map *)); }; } // namespace copyset diff --git a/curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h b/curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h index 08aa916623..e56a68892b 100644 --- a/curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h +++ b/curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h @@ -24,7 +24,7 @@ #define CURVEFS_TEST_METASERVER_COPYSET_MOCK_MOCK_COPYSET_NODE_MANAGER_H_ #include - +#include #include "curvefs/src/metaserver/copyset/copyset_node_manager.h" namespace curvefs { @@ -35,6 +35,7 @@ class MockCopysetNodeManager : public CopysetNodeManager { public: MOCK_METHOD2(GetCopysetNode, CopysetNode*(PoolId, CopysetId)); MOCK_METHOD2(PurgeCopysetNode, bool(PoolId, CopysetId)); + MOCK_CONST_METHOD1(GetAllCopysets, void(std::vector *)); MOCK_CONST_METHOD0(IsLoadFinished, bool()); }; diff --git a/curvefs/test/metaserver/fsinfo_manager_test.cpp b/curvefs/test/metaserver/fsinfo_manager_test.cpp new file mode 100644 index 0000000000..1c3ddb422f --- /dev/null +++ b/curvefs/test/metaserver/fsinfo_manager_test.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Fri May 05 11:42:53 CST 2023 + * Author: lixiaocui + */ + +#include + +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" +#include "curvefs/test/client/rpcclient/mock_mds_client.h" + +using ::curvefs::client::rpcclient::MockMdsClient; + +using ::testing::_; + +namespace curvefs { +namespace metaserver { + +TEST(FsInfoManagerTest, TestGetFsInfo) { + auto &manager = FsInfoManager::GetInstance(); + + auto mdsCli = std::make_shared(); + manager.SetMdsClient(mdsCli); + uint32_t fsId = 1; + + EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _)) + .WillOnce(Return(FSStatusCode::NOT_FOUND)); + FsInfo fsInfo; + ASSERT_FALSE(manager.GetFsInfo(fsId, &fsInfo)); + + EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _)) + .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR)); + ASSERT_FALSE(manager.GetFsInfo(fsId, &fsInfo)); + + EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _)).WillOnce(Return(FSStatusCode::OK)); + ASSERT_TRUE(manager.GetFsInfo(fsId, &fsInfo)); +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/test/metaserver/heartbeat_test.cpp b/curvefs/test/metaserver/heartbeat_test.cpp index 6e67841521..e26ddd8431 100644 --- a/curvefs/test/metaserver/heartbeat_test.cpp +++ b/curvefs/test/metaserver/heartbeat_test.cpp @@ -22,9 +22,12 @@ #include #include #include "curvefs/src/metaserver/heartbeat.h" +#include "curvefs/proto/heartbeat.pb.h" #include "curvefs/test/metaserver/mock_heartbeat_service.h" #include "curvefs/src/metaserver/storage/storage.h" #include "curvefs/src/metaserver/resource_statistic.h" +#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h" +#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" using ::testing::AtLeast; using ::testing::StrEq; @@ -42,6 +45,9 @@ using ::curvefs::mds::heartbeat::HeartbeatStatusCode; using ::curve::fs::FileSystemType; using ::curve::fs::LocalFsFactory; using ::curvefs::metaserver::storage::StorageOptions; +using ::curvefs::metaserver::copyset::MockCopysetNodeManager; +using ::curvefs::metaserver::copyset::MockCopysetNode; +using ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode; namespace curvefs { namespace metaserver { @@ -56,6 +62,8 @@ class HeartbeatTest : public ::testing::Test { resourceCollector_ = absl::make_unique( options_.maxDiskQuotaBytes, options_.maxMemoryQuotaBytes, options_.dataDir); + + GetHeartbeatOption(&hbopts_); } bool GetMetaserverSpaceStatus(MetaServerSpaceStatus* status, @@ -67,9 +75,25 @@ class HeartbeatTest : public ::testing::Test { return heartbeat.GetMetaserverSpaceStatus(status, ncopysets); } + void GetHeartbeatOption(HeartbeatOptions *opt) { + opt->metaserverId = 1; + opt->metaserverToken = "token"; + opt->intervalSec = 2; + opt->timeout = 1000; + opt->ip = "127.0.0.1"; + opt->port = 6000; + opt->mdsListenAddr = "127.0.0.1:6710"; + opt->copysetNodeManager = &mockCopysetManager_; + opt->storeUri = "local://./metaserver_data/copysets"; + opt->fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); + opt->resourceCollector = resourceCollector_.get(); + } + protected: StorageOptions options_; std::unique_ptr resourceCollector_; + HeartbeatOptions hbopts_; + MockCopysetNodeManager mockCopysetManager_; }; template ( - 0, 0, options_.dataDir).get(); - // mds service not start - ASSERT_EQ(heartbeat.Init(options), 0); + ASSERT_EQ(heartbeat.Init(hbopts_), 0); ASSERT_EQ(heartbeat.Run(), 0); sleep(2); ASSERT_EQ(heartbeat.Fini(), 0); } TEST_F(HeartbeatTest, test_ok) { - HeartbeatOptions options; Heartbeat heartbeat; - options.metaserverId = 1; - options.metaserverToken = "token"; - options.intervalSec = 2; - options.timeout = 1000; - options.ip = "127.0.0.1"; - options.port = 6000; - options.mdsListenAddr = "127.0.0.1:6710"; - options.copysetNodeManager = &CopysetNodeManager::GetInstance(); - options.storeUri = "local://./metaserver_data/copysets"; - options.fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); - options.resourceCollector = resourceCollector_.get(); - // send heartbeat ok brpc::Server server; MockHeartbeatService mockHeartbeatService; @@ -161,7 +158,7 @@ TEST_F(HeartbeatTest, test_ok) { Invoke(RpcService))); - ASSERT_EQ(heartbeat.Init(options), 0); + ASSERT_EQ(heartbeat.Init(hbopts_), 0); ASSERT_EQ(heartbeat.Run(), 0); sleep(5); ASSERT_EQ(heartbeat.Fini(), 0); @@ -171,20 +168,8 @@ TEST_F(HeartbeatTest, test_ok) { } TEST_F(HeartbeatTest, test_fail) { - HeartbeatOptions options; Heartbeat heartbeat; - options.metaserverId = 1; - options.metaserverToken = "token"; - options.intervalSec = 2; - options.timeout = 1000; - options.ip = "127.0.0.1"; - options.port = 6000; - options.mdsListenAddr = "127.0.0.1:6710"; - options.copysetNodeManager = &CopysetNodeManager::GetInstance(); - options.storeUri = "local://./metaserver_data/copysets"; - options.fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); - // send heartbeat ok brpc::Server server; MockHeartbeatService mockHeartbeatService; @@ -200,7 +185,7 @@ TEST_F(HeartbeatTest, test_fail) { Invoke(RpcService))); - ASSERT_EQ(heartbeat.Init(options), 0); + ASSERT_EQ(heartbeat.Init(hbopts_), 0); ASSERT_EQ(heartbeat.Run(), 0); sleep(5); ASSERT_EQ(heartbeat.Fini(), 0); @@ -231,5 +216,67 @@ TEST_F(HeartbeatTest, GetMetaServerSpaceStatusTest) { status.memoryusedbyte()); } +TEST_F(HeartbeatTest, Test_BuildRequest) { + Heartbeat heartbeat; + ASSERT_EQ(heartbeat.Init(hbopts_), 0); + + MockCopysetNode mockCopysetNode; + std::vector copysetVec; + copysetVec.emplace_back(&mockCopysetNode); + + BlockGroupStatInfoMap blockGroupStatInfoMap; + uint32_t fsId = 1; + uint64_t offset = 0; + auto &statInfo = blockGroupStatInfoMap[fsId]; + statInfo.set_fsid(1); + auto de = statInfo.add_deallocatableblockgroups(); + de->set_blockgroupoffset(offset); + de->set_deallocatablesize(1024); + + EXPECT_CALL(mockCopysetManager_, GetAllCopysets(_)) + .WillOnce(SetArgPointee<0>(copysetVec)); + EXPECT_CALL(mockCopysetNode, GetPoolId()).WillOnce(Return(1)); + EXPECT_CALL(mockCopysetNode, GetCopysetId()).WillOnce(Return(1)); + EXPECT_CALL(mockCopysetNode, ListPeers(_)).Times(1); + EXPECT_CALL(mockCopysetNode, GetLeaderId()).Times(1); + EXPECT_CALL(mockCopysetNode, IsLoading()).WillRepeatedly(Return(false)); + EXPECT_CALL(mockCopysetNode, GetPartitionInfoList(_)) + .WillOnce(Return(true)); + EXPECT_CALL(mockCopysetNode, GetConfChange(_, _)).Times(1); + EXPECT_CALL(mockCopysetNode, GetConfEpoch()).WillOnce(Return(1)); + EXPECT_CALL(mockCopysetNode, IsLeaderTerm()).WillOnce(Return(true)); + EXPECT_CALL(mockCopysetNode, GetBlockStatInfo(_)) + .WillOnce(DoAll(SetArgPointee<0>(blockGroupStatInfoMap), Return(true))); + + HeartbeatRequest req; + heartbeat.taskExecutor_->SetDeallocTask(fsId, offset); + heartbeat.BuildRequest(&req); + + // assert block group stat info + { + // assert deallocatableBlockGroups + auto outStatInfos = req.blockgroupstatinfos(); + ASSERT_EQ(outStatInfos.size(), 1); + + auto outOne = outStatInfos[0]; + ASSERT_EQ(outOne.fsid(), fsId); + + auto outDes = outOne.deallocatableblockgroups(); + ASSERT_EQ(outDes.size(), 1); + + auto outDesOne = outDes[0]; + ASSERT_EQ(outDesOne.blockgroupoffset(), de->blockgroupoffset()); + ASSERT_EQ(outDesOne.deallocatablesize(), de->deallocatablesize()); + ASSERT_TRUE(outDesOne.inodeidlist().empty()); + ASSERT_TRUE(outDesOne.inodeidunderdeallocate().empty()); + + // assert blockGroupDeallocateStatus + auto outStatus = outOne.blockgroupdeallocatestatus(); + ASSERT_EQ(outStatus.size(), 1); + + auto outStatusOne = outStatus[offset]; + ASSERT_EQ(outStatusOne, BlockGroupDeallcateStatusCode::BGDP_DONE); + } +} } // namespace metaserver } // namespace curvefs diff --git a/curvefs/test/metaserver/inode_storage_test.cpp b/curvefs/test/metaserver/inode_storage_test.cpp index e8a22ee436..bae78b1fef 100644 --- a/curvefs/test/metaserver/inode_storage_test.cpp +++ b/curvefs/test/metaserver/inode_storage_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include "curvefs/proto/metaserver.pb.h" +#include "curvefs/proto/common.pb.h" #include "curvefs/src/metaserver/inode_storage.h" #include "curvefs/src/common/define.h" @@ -42,20 +44,21 @@ #include "curvefs/test/metaserver/mock/mock_kv_storage.h" #include "src/fs/ext4_filesystem_impl.h" -using ::testing::AtLeast; -using ::testing::StrEq; using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; using ::testing::Return; using ::testing::ReturnArg; -using ::testing::DoAll; -using ::testing::SetArgPointee; using ::testing::SaveArg; +using ::testing::SetArgPointee; +using ::testing::StrEq; -using ::curvefs::metaserver::storage::KVStorage; -using ::curvefs::metaserver::storage::StorageOptions; -using ::curvefs::metaserver::storage::RocksDBStorage; +using ::curvefs::metaserver::storage::Key4DeallocatableBlockGroup; using ::curvefs::metaserver::storage::Key4S3ChunkInfoList; +using ::curvefs::metaserver::storage::KVStorage; using ::curvefs::metaserver::storage::RandomStoragePath; +using ::curvefs::metaserver::storage::RocksDBStorage; +using ::curvefs::metaserver::storage::StorageOptions; namespace curvefs { namespace metaserver { @@ -83,7 +86,7 @@ class InodeStorageTest : public ::testing::Test { ASSERT_EQ(output.size(), 0); } - std::string execShell(const std::string& cmd) { + std::string execShell(const std::string &cmd) { std::array buffer; std::string result; std::unique_ptr pipe(popen(cmd.c_str(), "r"), @@ -126,7 +129,7 @@ class InodeStorageTest : public ::testing::Test { uint64_t lastChunkId) { S3ChunkInfoList list; for (uint64_t id = firstChunkId; id <= lastChunkId; id++) { - S3ChunkInfo* info = list.add_s3chunks(); + S3ChunkInfo *info = list.add_s3chunks(); info->set_chunkid(id); info->set_compaction(0); info->set_offset(0); @@ -137,17 +140,15 @@ class InodeStorageTest : public ::testing::Test { return list; } - bool EqualS3ChunkInfo(const S3ChunkInfo& lhs, const S3ChunkInfo& rhs) { + bool EqualS3ChunkInfo(const S3ChunkInfo &lhs, const S3ChunkInfo &rhs) { return lhs.chunkid() == rhs.chunkid() && - lhs.compaction() == rhs.compaction() && - lhs.offset() == rhs.offset() && - lhs.len() == rhs.len() && - lhs.size() == rhs.size() && - lhs.zero() == rhs.zero(); + lhs.compaction() == rhs.compaction() && + lhs.offset() == rhs.offset() && lhs.len() == rhs.len() && + lhs.size() == rhs.size() && lhs.zero() == rhs.zero(); } - bool EqualS3ChunkInfoList(const S3ChunkInfoList& lhs, - const S3ChunkInfoList& rhs) { + bool EqualS3ChunkInfoList(const S3ChunkInfoList &lhs, + const S3ChunkInfoList &rhs) { size_t size = lhs.s3chunks_size(); if (size != rhs.s3chunks_size()) { return false; @@ -161,8 +162,8 @@ class InodeStorageTest : public ::testing::Test { return true; } - void CHECK_INODE_S3CHUNKINFOLIST(InodeStorage* storage, - uint32_t fsId, uint64_t inodeId, + void CHECK_INODE_S3CHUNKINFOLIST(InodeStorage *storage, uint32_t fsId, + uint64_t inodeId, const std::vector chunkIndexs, const std::vector lists) { ASSERT_EQ(chunkIndexs.size(), lists.size()); @@ -217,8 +218,7 @@ TEST_F(InodeStorageTest, test1) { // delete ASSERT_EQ(storage.Delete(Key4Inode(inode1)), MetaStatusCode::OK); ASSERT_EQ(storage.Size(), 2); - ASSERT_EQ(storage.Get(Key4Inode(inode1), &temp), - MetaStatusCode::NOT_FOUND); + ASSERT_EQ(storage.Get(Key4Inode(inode1), &temp), MetaStatusCode::NOT_FOUND); ASSERT_EQ(storage.Delete(Key4Inode(inode1)), MetaStatusCode::OK); // update @@ -259,7 +259,7 @@ TEST_F(InodeStorageTest, testGetAttrNotFound) { ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK); InodeAttr attr; ASSERT_EQ(storage.GetAttr(Key4Inode(1, 2), &attr), - MetaStatusCode::NOT_FOUND); + MetaStatusCode::NOT_FOUND); } TEST_F(InodeStorageTest, testGetAttr) { @@ -357,8 +357,8 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { { LOG(INFO) << "CASE 2: append one s3chunkinfo"; ASSERT_EQ(storage.Clear(), MetaStatusCode::OK); - std::vector chunkIndexs{ 1 }; - std::vector lists2add{ GenS3ChunkInfoList(1, 1) }; + std::vector chunkIndexs{1}; + std::vector lists2add{GenS3ChunkInfoList(1, 1)}; for (size_t i = 0; i < chunkIndexs.size(); i++) { MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList( @@ -366,15 +366,15 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { ASSERT_EQ(rc, MetaStatusCode::OK); } - CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - chunkIndexs, lists2add); + CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, chunkIndexs, + lists2add); } // CASE 3: append multi s3chunkinfos { LOG(INFO) << "CASE 3: append multi s3chunkinfos"; ASSERT_EQ(storage.Clear(), MetaStatusCode::OK); - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(1, 1), GenS3ChunkInfoList(2, 2), @@ -387,15 +387,15 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { ASSERT_EQ(rc, MetaStatusCode::OK); } - CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - chunkIndexs, lists2add); + CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, chunkIndexs, + lists2add); } // CASE 4: check order for s3chunkinfo's chunk index { LOG(INFO) << "CASE 4: check order for s3chunkinfo's chunk index"; ASSERT_EQ(storage.Clear(), MetaStatusCode::OK); - std::vector chunkIndexs{ 2, 1, 3 }; + std::vector chunkIndexs{2, 1, 3}; std::vector lists2add{ GenS3ChunkInfoList(2, 2), GenS3ChunkInfoList(1, 1), @@ -409,24 +409,22 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { } CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - std::vector{ 1, 2, 3 }, - std::vector{ - GenS3ChunkInfoList(1, 1), - GenS3ChunkInfoList(2, 2), - GenS3ChunkInfoList(3, 3), - }); + std::vector{1, 2, 3}, + std::vector{ + GenS3ChunkInfoList(1, 1), + GenS3ChunkInfoList(2, 2), + GenS3ChunkInfoList(3, 3), + }); } // CASE 5: check order for s3chunkinfo's chunk id { LOG(INFO) << "CASE 5: check order for s3chunkinfo's chunk id"; ASSERT_EQ(storage.Clear(), MetaStatusCode::OK); - std::vector chunkIndexs{ 2, 1, 3, 1, 2 }; + std::vector chunkIndexs{2, 1, 3, 1, 2}; std::vector lists2add{ - GenS3ChunkInfoList(200, 210), - GenS3ChunkInfoList(120, 130), - GenS3ChunkInfoList(300, 310), - GenS3ChunkInfoList(100, 110), + GenS3ChunkInfoList(200, 210), GenS3ChunkInfoList(120, 130), + GenS3ChunkInfoList(300, 310), GenS3ChunkInfoList(100, 110), GenS3ChunkInfoList(220, 230), }; @@ -437,14 +435,14 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { } CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - std::vector{ 1, 1, 2, 2, 3 }, - std::vector{ - GenS3ChunkInfoList(100, 110), - GenS3ChunkInfoList(120, 130), - GenS3ChunkInfoList(200, 210), - GenS3ChunkInfoList(220, 230), - GenS3ChunkInfoList(300, 310), - }); + std::vector{1, 1, 2, 2, 3}, + std::vector{ + GenS3ChunkInfoList(100, 110), + GenS3ChunkInfoList(120, 130), + GenS3ChunkInfoList(200, 210), + GenS3ChunkInfoList(220, 230), + GenS3ChunkInfoList(300, 310), + }); } @@ -455,7 +453,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step1: add s3chunkinfo { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(100, 199), GenS3ChunkInfoList(200, 299), @@ -471,7 +469,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step2: compaction { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(199, 199), GenS3ChunkInfoList(299, 299), @@ -486,8 +484,8 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { for (size_t i = 0; i < chunkIndexs.size(); i++) { MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList( - fsId, inodeId, chunkIndexs[i], - &lists2add[i], &lists2del[i]); + fsId, inodeId, chunkIndexs[i], &lists2add[i], + &lists2del[i]); ASSERT_EQ(rc, MetaStatusCode::STORAGE_INTERNAL_ERROR); } } @@ -500,7 +498,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step1: add s3chunkinfo { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(100, 199), GenS3ChunkInfoList(200, 299), @@ -516,7 +514,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step2: compaction { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(199, 199), GenS3ChunkInfoList(299, 299), @@ -531,20 +529,20 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { for (size_t i = 0; i < chunkIndexs.size(); i++) { MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList( - fsId, inodeId, chunkIndexs[i], - &lists2add[i], &lists2del[i]); + fsId, inodeId, chunkIndexs[i], &lists2add[i], + &lists2del[i]); ASSERT_EQ(rc, MetaStatusCode::OK); } } // step3: check result CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - std::vector{ 1, 2, 3 }, - std::vector{ - GenS3ChunkInfoList(199, 199), - GenS3ChunkInfoList(299, 299), - GenS3ChunkInfoList(399, 399), - }); + std::vector{1, 2, 3}, + std::vector{ + GenS3ChunkInfoList(199, 199), + GenS3ChunkInfoList(299, 299), + GenS3ChunkInfoList(399, 399), + }); } @@ -555,7 +553,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step1: add s3chunkinfo { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(100, 199), GenS3ChunkInfoList(1000, 1999), @@ -571,7 +569,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step2: add s3chunkinfo again { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(200, 299), GenS3ChunkInfoList(2000, 2999), @@ -587,7 +585,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { // step2: compaction { - std::vector chunkIndexs{ 1, 2, 3 }; + std::vector chunkIndexs{1, 2, 3}; std::vector lists2add{ GenS3ChunkInfoList(199, 199), GenS3ChunkInfoList(2999, 2999), @@ -602,22 +600,22 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { for (size_t i = 0; i < chunkIndexs.size(); i++) { MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList( - fsId, inodeId, chunkIndexs[i], - &lists2add[i], &lists2del[i]); + fsId, inodeId, chunkIndexs[i], &lists2add[i], + &lists2del[i]); ASSERT_EQ(rc, MetaStatusCode::OK); } } // step3: check result CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, - std::vector{ 1, 1, 2, 3, 3 }, - std::vector{ - GenS3ChunkInfoList(199, 199), - GenS3ChunkInfoList(200, 299), - GenS3ChunkInfoList(2999, 2999), - GenS3ChunkInfoList(19999, 19999), - GenS3ChunkInfoList(20000, 29999), - }); + std::vector{1, 1, 2, 3, 3}, + std::vector{ + GenS3ChunkInfoList(199, 199), + GenS3ChunkInfoList(200, 299), + GenS3ChunkInfoList(2999, 2999), + GenS3ChunkInfoList(19999, 19999), + GenS3ChunkInfoList(20000, 29999), + }); } } @@ -632,12 +630,10 @@ TEST_F(InodeStorageTest, PaddingInodeS3ChunkInfo) { ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK); // step2: append s3chunkinfo - std::vector chunkIndexs{ 1, 3, 2, 1, 2 }; + std::vector chunkIndexs{1, 3, 2, 1, 2}; std::vector lists2add{ - GenS3ChunkInfoList(100, 109), - GenS3ChunkInfoList(300, 310), - GenS3ChunkInfoList(200, 209), - GenS3ChunkInfoList(110, 120), + GenS3ChunkInfoList(100, 109), GenS3ChunkInfoList(300, 310), + GenS3ChunkInfoList(200, 209), GenS3ChunkInfoList(110, 120), GenS3ChunkInfoList(210, 220), }; @@ -713,8 +709,8 @@ TEST_F(InodeStorageTest, GetAllS3ChunkInfoList) { size_t size = 0; Key4S3ChunkInfoList key; S3ChunkInfoList list4get; - std::vector fsIds{ 1, 2 }; - std::vector inodeIds{ 1, 2 }; + std::vector fsIds{1, 2}; + std::vector inodeIds{1, 2}; auto iterator = storage.GetAllS3ChunkInfoList(); for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { ASSERT_TRUE(conv_->ParseFromString(iterator->Key(), &key)); @@ -732,11 +728,10 @@ TEST_F(InodeStorageTest, TestUpdateVolumeExtentSlice) { using storage::Status; const std::vector> cases{ - { MetaStatusCode::OK, Status::OK()}, - {MetaStatusCode::STORAGE_INTERNAL_ERROR, Status::InternalError()} - }; + {MetaStatusCode::OK, Status::OK()}, + {MetaStatusCode::STORAGE_INTERNAL_ERROR, Status::InternalError()}}; - for (const auto& test : cases) { + for (const auto &test : cases) { auto kvStorage = std::make_shared(); InodeStorage storage(kvStorage, nameGenerator_, 0); @@ -756,7 +751,7 @@ TEST_F(InodeStorageTest, TestUpdateVolumeExtentSlice) { } } -static void RandomSetExtent(VolumeExtent* ext) { +static void RandomSetExtent(VolumeExtent *ext) { std::random_device rd; ext->set_fsoffset(rd()); @@ -765,10 +760,9 @@ static void RandomSetExtent(VolumeExtent* ext) { ext->set_isused(rd() & 1); } -static bool PrepareGetAllVolumeExtentTest(InodeStorage* storage, - uint32_t fsId, +static bool PrepareGetAllVolumeExtentTest(InodeStorage *storage, uint32_t fsId, uint64_t inodeId, - std::vector* out) { + std::vector *out) { VolumeExtentSlice slice1; slice1.set_offset(0); @@ -804,22 +798,22 @@ static bool PrepareGetAllVolumeExtentTest(InodeStorage* storage, return true; } -static bool operator==(const VolumeExtentList& list, - const std::vector& slices) { +static bool operator==(const VolumeExtentSliceList &list, + const std::vector &slices) { std::vector clist(list.slices().begin(), list.slices().end()); auto copy = slices; std::sort(copy.begin(), copy.end(), - [](const VolumeExtentSlice& s1, const VolumeExtentSlice& s2) { + [](const VolumeExtentSlice &s1, const VolumeExtentSlice &s2) { return s1.offset() < s2.offset(); }); return true; } -static bool operator==(const VolumeExtentSlice& s1, - const VolumeExtentSlice& s2) { +static bool operator==(const VolumeExtentSlice &s1, + const VolumeExtentSlice &s2) { return google::protobuf::util::MessageDifferencer::Equals(s1, s2); } @@ -831,7 +825,7 @@ TEST_F(InodeStorageTest, TestGetAllVolumeExtent) { std::make_shared(opts); std::shared_ptr kvStore = kvStorage_; - for (auto& store : {memStore, kvStore}) { + for (auto &store : {memStore, kvStore}) { InodeStorage storage(store, nameGenerator_, 0); const uint32_t fsId = 1; const uint64_t inodeId = 2; @@ -840,20 +834,28 @@ TEST_F(InodeStorageTest, TestGetAllVolumeExtent) { ASSERT_TRUE( PrepareGetAllVolumeExtentTest(&storage, fsId, inodeId, &slices)); - VolumeExtentList list; + VolumeExtentSliceList list; ASSERT_EQ(MetaStatusCode::OK, storage.GetAllVolumeExtent(fsId, inodeId, &list)); ASSERT_EQ(2, list.slices().size()); ASSERT_EQ(list, slices); + + auto iter = storage.GetAllVolumeExtent(fsId, inodeId); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); } } static std::string ToString(storage::STORAGE_TYPE type) { switch (type) { - case storage::STORAGE_TYPE::MEMORY_STORAGE: - return "memory"; - case storage::STORAGE_TYPE::ROCKSDB_STORAGE: - return "rocksdb"; + case storage::STORAGE_TYPE::MEMORY_STORAGE: + return "memory"; + case storage::STORAGE_TYPE::ROCKSDB_STORAGE: + return "rocksdb"; } return "unknown"; @@ -867,7 +869,7 @@ TEST_F(InodeStorageTest, TestGetVolumeExtentByOffset) { std::make_shared(opts); std::shared_ptr kvStore = kvStorage_; - for (auto& store : {kvStorage_, memStore}) { + for (auto &store : {kvStorage_, memStore}) { InodeStorage storage(store, nameGenerator_, 0); const uint32_t fsId = 1; const uint64_t inodeId = 2; @@ -894,5 +896,104 @@ TEST_F(InodeStorageTest, TestGetVolumeExtentByOffset) { } } +TEST_F(InodeStorageTest, Test_UpdateInodeWithDeallocate) { + auto inode = GenInode(1, 1); + InodeStorage storage(kvStorage_, nameGenerator_, 0); + ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK); + + ASSERT_EQ(MetaStatusCode::OK, storage.Update(inode, true)); + + auto tableName = nameGenerator_->GetDeallocatableInodeTableName(); + Key4Inode key(1, 1); + auto skey = key.SerializeToString(); + google::protobuf::Empty value; + auto s = kvStorage_->HGet(tableName, skey, &value); + ASSERT_TRUE(s.ok()); +} + +TEST_F(InodeStorageTest, Test_UpdateDeallocatableBlockGroup) { + uint64_t blockGroupOffset = 0; + uint32_t fsId = 1; + auto tableName = nameGenerator_->GetDeallocatableBlockGroupTableName(); + InodeStorage storage(kvStorage_, nameGenerator_, 0); + uint64_t increaseSize = 100 * 1024; + uint64_t decreaseSize = 4 * 1024; + Key4DeallocatableBlockGroup key(fsId, blockGroupOffset); + std::vector deallocatableBlockGroupVec; + ASSERT_EQ(MetaStatusCode::NOT_FOUND, + storage.GetAllBlockGroup(&deallocatableBlockGroupVec)); + + // test increase + { + DeallocatableBlockGroupVec inputVec; + + DeallocatableBlockGroup blockGroup; + auto increase = blockGroup.mutable_increase(); + increase->set_increasedeallocatablesize(increaseSize); + increase->add_inodeidlistadd(1); + inputVec.Add()->CopyFrom(blockGroup); + + // increase first time + ASSERT_EQ(MetaStatusCode::OK, + storage.UpdateDeallocatableBlockGroup(fsId, inputVec)); + DeallocatableBlockGroup out; + auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(increaseSize, out.deallocatablesize()); + ASSERT_EQ(1, out.inodeidlist_size()); + ASSERT_EQ(blockGroupOffset, out.blockgroupoffset()); + ASSERT_EQ(0, out.inodeidunderdeallocate_size()); + + // increase second time + ASSERT_EQ(MetaStatusCode::OK, + storage.UpdateDeallocatableBlockGroup(fsId, inputVec)); + st = kvStorage_->HGet(tableName, key.SerializeToString(), &out); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(increaseSize * 2, out.deallocatablesize()); + ASSERT_EQ(1, out.inodeidlist_size()); + ASSERT_EQ(blockGroupOffset, out.blockgroupoffset()); + ASSERT_EQ(0, out.inodeidunderdeallocate_size()); + } + + // test mark + { + DeallocatableBlockGroupVec inputVec; + + DeallocatableBlockGroup blockGroup; + auto mark = blockGroup.mutable_mark(); + mark->add_inodeidunderdeallocate(1); + inputVec.Add()->CopyFrom(blockGroup); + + ASSERT_EQ(MetaStatusCode::OK, + storage.UpdateDeallocatableBlockGroup(fsId, inputVec)); + DeallocatableBlockGroup out; + auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(increaseSize * 2, out.deallocatablesize()); + ASSERT_EQ(0, out.inodeidlist_size()); + ASSERT_EQ(1, out.inodeidunderdeallocate_size()); + } + + // test decrease + { + DeallocatableBlockGroupVec inputVec; + + DeallocatableBlockGroup blockGroup; + auto decrease = blockGroup.mutable_decrease(); + decrease->set_decreasedeallocatablesize(decreaseSize); + inputVec.Add()->CopyFrom(blockGroup); + ASSERT_EQ(MetaStatusCode::OK, + storage.UpdateDeallocatableBlockGroup(fsId, inputVec)); + DeallocatableBlockGroup out; + auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(increaseSize * 2 - decreaseSize, out.deallocatablesize()); + } + + ASSERT_EQ(MetaStatusCode::OK, + storage.GetAllBlockGroup(&deallocatableBlockGroupVec)); + ASSERT_EQ(1, deallocatableBlockGroupVec.size()); +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/test/metaserver/metacli_manager_test.cpp b/curvefs/test/metaserver/metacli_manager_test.cpp new file mode 100644 index 0000000000..448d37bc16 --- /dev/null +++ b/curvefs/test/metaserver/metacli_manager_test.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Jul 12 10:08:00 CST 2023 + * Author: lixiaocui + */ + +#include +#include "curvefs/src/metaserver/metacli_manager.h" + +namespace curvefs { +namespace metaserver { + +TEST(MetaCliManagerTest, TestGetMetaCli) { + MetaCliManagerOpt opt; + MetaCliManager::GetInstance().Init(std::move(opt)); + + ASSERT_TRUE(MetaCliManager::GetInstance().GetMetaCli(1) != nullptr); +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/test/metaserver/metaserver_service_test2.cpp b/curvefs/test/metaserver/metaserver_service_test2.cpp index d30c733bec..e7376b2e4f 100644 --- a/curvefs/test/metaserver/metaserver_service_test2.cpp +++ b/curvefs/test/metaserver/metaserver_service_test2.cpp @@ -92,6 +92,10 @@ TEST_F(MetaServerServiceTest2, ServiceOverload) { TEST_SERVICE_OVERLOAD(CreatePartition); TEST_SERVICE_OVERLOAD(DeletePartition); TEST_SERVICE_OVERLOAD(PrepareRenameTx); + TEST_SERVICE_OVERLOAD(GetVolumeExtent); + TEST_SERVICE_OVERLOAD(UpdateVolumeExtent); + TEST_SERVICE_OVERLOAD(UpdateDeallocatableBlockGroup); + #undef TEST_SERVICE_OVERLOAD } @@ -126,6 +130,9 @@ TEST_F(MetaServerServiceTest2, CopysetNodeNotFound) { TEST_COPYSETNODE_NOTFOUND(CreatePartition); TEST_COPYSETNODE_NOTFOUND(DeletePartition); TEST_COPYSETNODE_NOTFOUND(PrepareRenameTx); + TEST_COPYSETNODE_NOTFOUND(GetVolumeExtent); + TEST_COPYSETNODE_NOTFOUND(UpdateVolumeExtent); + TEST_COPYSETNODE_NOTFOUND(UpdateDeallocatableBlockGroup); #undef TEST_COPYSETNODE_NOTFOUND } diff --git a/curvefs/test/metaserver/metastore_test.cpp b/curvefs/test/metaserver/metastore_test.cpp index fbb41016b6..78e357d5c7 100644 --- a/curvefs/test/metaserver/metastore_test.cpp +++ b/curvefs/test/metaserver/metastore_test.cpp @@ -33,9 +33,11 @@ #include "curvefs/src/metaserver/storage/rocksdb_storage.h" #include "curvefs/src/metaserver/storage/converter.h" #include "curvefs/src/metaserver/copyset/copyset_node.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "curvefs/test/metaserver/storage/utils.h" #include "src/common/uuid.h" #include "src/fs/ext4_filesystem_impl.h" +#include "curvefs/test/client/rpcclient/mock_mds_client.h" using ::testing::_; using ::testing::AtLeast; @@ -55,6 +57,7 @@ using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::RandomStoragePath; using ::curvefs::metaserver::copyset::CopysetNode; +using ::curvefs::client::rpcclient::MockMdsClient; namespace { @@ -95,6 +98,10 @@ class MetastoreTest : public ::testing::Test { conv_ = std::make_shared(); braft::Configuration conf; copyset_ = std::make_shared(1, 1, conf, nullptr); + + // init fsinfo + mdsCli_ = std::make_shared(); + FsInfoManager::GetInstance().SetMdsClient(mdsCli_); } void TearDown() override { @@ -276,6 +283,7 @@ class MetastoreTest : public ::testing::Test { std::shared_ptr kvStorage_; std::shared_ptr conv_; std::shared_ptr copyset_; + std::shared_ptr mdsCli_; StorageOptions options_; }; diff --git a/curvefs/test/metaserver/mock/mock_metastore.h b/curvefs/test/metaserver/mock/mock_metastore.h index 2cba2e1d5a..da5a7b7c56 100644 --- a/curvefs/test/metaserver/mock/mock_metastore.h +++ b/curvefs/test/metaserver/mock/mock_metastore.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "curvefs/src/metaserver/metastore.h" @@ -47,6 +48,8 @@ class MockMetaStore : public curvefs::metaserver::MetaStore { MOCK_METHOD2(DeletePartition, MetaStatusCode(const DeletePartitionRequest*, DeletePartitionResponse*)); MOCK_METHOD1(GetPartitionInfoList, bool(std::list *)); + MOCK_METHOD1(GetPartitionSnap, + bool(std::map> *)); MOCK_METHOD2(CreateDentry, MetaStatusCode(const CreateDentryRequest*, CreateDentryResponse*)); @@ -98,6 +101,10 @@ class MockMetaStore : public curvefs::metaserver::MetaStore { MOCK_METHOD2(UpdateVolumeExtent, MetaStatusCode(const UpdateVolumeExtentRequest*, UpdateVolumeExtentResponse*)); + + MOCK_METHOD2(UpdateDeallocatableBlockGroup, + MetaStatusCode(const UpdateDeallocatableBlockGroupRequest *, + UpdateDeallocatableBlockGroupResponse *)); }; } // namespace mock diff --git a/curvefs/test/metaserver/mock/mock_partition.h b/curvefs/test/metaserver/mock/mock_partition.h new file mode 100644 index 0000000000..9dd48c0f12 --- /dev/null +++ b/curvefs/test/metaserver/mock/mock_partition.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Mon Apr 24 20:04:24 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_ +#define CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_ + +#include +#include +#include "curvefs/src/metaserver/partition.h" + +namespace curvefs { +namespace metaserver { +namespace mock { +class MockPartition : public curvefs::metaserver::Partition { + public: + MockPartition() : Partition() {} + MOCK_METHOD1(GetAllBlockGroup, + MetaStatusCode(std::vector *)); + MOCK_CONST_METHOD0(GetPartitionId, uint32_t()); + MOCK_CONST_METHOD0(GetFsId, uint32_t()); +}; +} // namespace mock +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_ diff --git a/curvefs/test/metaserver/partition_clean_test.cpp b/curvefs/test/metaserver/partition_clean_test.cpp index c3f026b8f8..d6dedd6234 100644 --- a/curvefs/test/metaserver/partition_clean_test.cpp +++ b/curvefs/test/metaserver/partition_clean_test.cpp @@ -30,19 +30,21 @@ #include "curvefs/src/metaserver/storage/storage.h" #include "curvefs/src/metaserver/storage/rocksdb_storage.h" #include "curvefs/test/metaserver/storage/utils.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "src/fs/ext4_filesystem_impl.h" -using ::curvefs::mds::FSStatusCode; using ::curvefs::client::rpcclient::MockMdsClient; +using ::curvefs::mds::FSStatusCode; using ::testing::_; using ::testing::Invoke; using ::testing::Matcher; using ::testing::Return; using ::curvefs::metaserver::storage::KVStorage; -using ::curvefs::metaserver::storage::StorageOptions; -using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::RandomStoragePath; +using ::curvefs::metaserver::storage::RocksDBStorage; +using ::curvefs::metaserver::storage::StorageOptions; +using ::curvefs::metaserver::copyset::MockCopysetNode; namespace curvefs { namespace metaserver { @@ -60,15 +62,22 @@ class PartitionCleanManagerTest : public testing::Test { options.localFileSystem = localfs.get(); kvStorage_ = std::make_shared(options); ASSERT_TRUE(kvStorage_->Open()); + + mdsCli_ = std::make_shared(); + s3Adaptor_ = std::make_shared(); + copyset_ = new MockCopysetNode(); + FsInfoManager::GetInstance().SetMdsClient(mdsCli_); } void TearDown() override { ASSERT_TRUE(kvStorage_->Close()); auto output = execShell("rm -rf " + dataDir_); ASSERT_EQ(output.size(), 0); + + delete copyset_; } - std::string execShell(const string& cmd) { + std::string execShell(const string &cmd) { std::array buffer; std::string result; std::unique_ptr pipe(popen(cmd.c_str(), "r"), @@ -85,21 +94,20 @@ class PartitionCleanManagerTest : public testing::Test { protected: std::string dataDir_; std::shared_ptr kvStorage_; + std::shared_ptr mdsCli_; + std::shared_ptr s3Adaptor_; + MockCopysetNode *copyset_; }; TEST_F(PartitionCleanManagerTest, test1) { ASSERT_TRUE(true); - PartitionCleanManager* manager = &PartitionCleanManager::GetInstance(); + PartitionCleanManager *manager = &PartitionCleanManager::GetInstance(); PartitionCleanOption option; option.scanPeriodSec = 1; option.inodeDeletePeriodMs = 500; - std::shared_ptr s3Adaptor = - std::make_shared(); - option.s3Adaptor = s3Adaptor; - std::shared_ptr mdsclient = - std::make_shared(); - option.mdsClient = mdsclient; + option.s3Adaptor = s3Adaptor_; + option.mdsClient = mdsCli_; manager->Init(option); manager->Run(); uint32_t partitionId = 1; @@ -110,7 +118,7 @@ TEST_F(PartitionCleanManagerTest, test1) { partitionInfo.set_start(0); partitionInfo.set_end(2000); std::shared_ptr partition = - std::make_shared(partitionInfo, kvStorage_); + std::make_shared(partitionInfo, kvStorage_); Dentry dentry; dentry.set_fsid(fsId); dentry.set_parentinodeid(1); @@ -140,76 +148,63 @@ TEST_F(PartitionCleanManagerTest, test1) { ASSERT_EQ(partition->GetDentryNum(), 1); Inode inode2; ASSERT_EQ(partition->GetInode(fsId, ROOTINODEID, &inode2), - MetaStatusCode::OK); + MetaStatusCode::OK); std::shared_ptr partitionCleaner = - std::make_shared(partition); - - copyset::MockCopysetNode copysetNode; + std::make_shared(partition); - EXPECT_CALL(copysetNode, IsLeaderTerm()) + EXPECT_CALL(*copyset_, IsLeaderTerm()) .WillOnce(Return(false)) .WillRepeatedly(Return(true)); - EXPECT_CALL(copysetNode, Propose(_)) - .WillOnce(Invoke([partition, fsId](const braft::Task& task) { + EXPECT_CALL(*copyset_, Propose(_)) + .WillOnce(Invoke([partition, fsId](const braft::Task &task) { ASSERT_EQ(partition->DeleteInode(fsId, ROOTINODEID), MetaStatusCode::OK); LOG(INFO) << "Partition DeleteInode, fsId = " << fsId << ", inodeId = " << ROOTINODEID; task.done->Run(); })) - .WillOnce(Invoke([partition, fsId, inode1](const braft::Task& task) { + .WillOnce(Invoke([partition, fsId, inode1](const braft::Task &task) { ASSERT_EQ(partition->DeleteInode(fsId, inode1.inodeid()), MetaStatusCode::OK); LOG(INFO) << "Partition DeleteInode, fsId = " << fsId << ", inodeId = " << inode1.inodeid(); task.done->Run(); })) - .WillOnce(Invoke([partition](const braft::Task& task) { + .WillOnce(Invoke([partition](const braft::Task &task) { LOG(INFO) << "Partition deletePartition"; task.done->Run(); })); - EXPECT_CALL(*s3Adaptor, Delete(_)) - .WillOnce(Return(0)); - EXPECT_CALL(*mdsclient, GetFsInfo(Matcher(_), _)) - .WillOnce(Return(FSStatusCode::OK)); - EXPECT_CALL(*s3Adaptor, GetS3ClientAdaptorOption(_)); - EXPECT_CALL(*s3Adaptor, Reinit(_, _, _, _, _)); + EXPECT_CALL(*s3Adaptor_, Delete(_)).WillOnce(Return(0)); + EXPECT_CALL(*s3Adaptor_, GetS3ClientAdaptorOption(_)); + EXPECT_CALL(*s3Adaptor_, Reinit(_, _, _, _, _)); - manager->Add(partitionId, partitionCleaner, ©setNode); + manager->Add(partitionId, partitionCleaner, copyset_); sleep(4); manager->Fini(); ASSERT_EQ(manager->GetCleanerCount(), 0); } -TEST_F(PartitionCleanManagerTest, fsinfo_not_found) { +TEST_F(PartitionCleanManagerTest, GetFsInfoFail) { PartitionInfo partition; - PartitionCleaner cleaner(std::make_shared( - partition, kvStorage_)); - std::shared_ptr mdsClient = - std::make_shared(); - cleaner.SetMdsClient(mdsClient); + PartitionCleaner cleaner( + std::make_shared(partition, kvStorage_)); + cleaner.SetMdsClient(mdsCli_); + cleaner.SetS3Aapter(s3Adaptor_); + cleaner.SetCopysetNode(copyset_); Inode inode; inode.set_type(FsFileType::TYPE_S3); - EXPECT_CALL(*mdsClient, GetFsInfo(Matcher(_), _)) + inode.set_fsid(10); + EXPECT_CALL(*mdsCli_, GetFsInfo(Matcher(_), _)) .WillOnce(Return(FSStatusCode::NOT_FOUND)); ASSERT_EQ(cleaner.CleanDataAndDeleteInode(inode), MetaStatusCode::S3_DELETE_ERR); -} -TEST_F(PartitionCleanManagerTest, fsinfo_other_error) { - PartitionInfo partition; - PartitionCleaner cleaner(std::make_shared( - partition, kvStorage_)); - std::shared_ptr mdsClient = - std::make_shared(); - cleaner.SetMdsClient(mdsClient); - Inode inode; - inode.set_type(FsFileType::TYPE_S3); - EXPECT_CALL(*mdsClient, GetFsInfo(Matcher(_), _)) + inode.set_fsid(11); + EXPECT_CALL(*mdsCli_, GetFsInfo(Matcher(_), _)) .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR)); ASSERT_EQ(cleaner.CleanDataAndDeleteInode(inode), MetaStatusCode::S3_DELETE_ERR); diff --git a/curvefs/test/metaserver/recycle_cleaner_test.cpp b/curvefs/test/metaserver/recycle_cleaner_test.cpp index ab8cdc1f3b..89321fc97d 100644 --- a/curvefs/test/metaserver/recycle_cleaner_test.cpp +++ b/curvefs/test/metaserver/recycle_cleaner_test.cpp @@ -30,6 +30,7 @@ #include "curvefs/test/client/rpcclient/mock_mds_client.h" #include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" #include "curvefs/test/metaserver/storage/utils.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "src/fs/ext4_filesystem_impl.h" using ::testing::_; @@ -70,7 +71,8 @@ class RecycleCleanerTest : public testing::Test { partitionInfo.set_fsid(fsId); partitionInfo.set_start(0); partitionInfo.set_end(2000); - partition_ = std::make_shared(partitionInfo, kvStorage_); + partition_ = std::make_shared(partitionInfo, kvStorage_, + false, false); cleaner_ = std::make_shared(partition_); mdsclient_ = std::make_shared(); metaClient_ = std::make_shared(); @@ -79,6 +81,7 @@ class RecycleCleanerTest : public testing::Test { cleaner_->SetMdsClient(mdsclient_); cleaner_->SetMetaClient(metaClient_); cleaner_->SetScanLimit(5); + FsInfoManager::GetInstance().SetMdsClient(mdsclient_); // create recycle dir and root dir InodeParam rootPram; diff --git a/curvefs/test/metaserver/recycle_manager_test.cpp b/curvefs/test/metaserver/recycle_manager_test.cpp index 89c14ed688..51b64d491d 100644 --- a/curvefs/test/metaserver/recycle_manager_test.cpp +++ b/curvefs/test/metaserver/recycle_manager_test.cpp @@ -26,6 +26,7 @@ #include "curvefs/src/metaserver/storage/rocksdb_storage.h" #include "curvefs/src/metaserver/storage/storage.h" +#include "curvefs/src/metaserver/mds/fsinfo_manager.h" #include "curvefs/test/client/mock_metaserver_client.h" #include "curvefs/test/client/rpcclient/mock_mds_client.h" #include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" @@ -62,6 +63,9 @@ class RecycleManangeTest : public testing::Test { options.localFileSystem = localfs.get(); kvStorage_ = std::make_shared(options); ASSERT_TRUE(kvStorage_->Open()); + + mdsclient_ = std::make_shared(); + FsInfoManager::GetInstance().SetMdsClient(mdsclient_); } void TearDown() override { @@ -97,14 +101,13 @@ class RecycleManangeTest : public testing::Test { protected: std::string dataDir_; std::shared_ptr kvStorage_; + std::shared_ptr mdsclient_; }; TEST_F(RecycleManangeTest, test_empty_recycle) { RecycleManager* manager = &RecycleManager::GetInstance(); RecycleManagerOption opt; - std::shared_ptr mdsclient = - std::make_shared(); - opt.mdsClient = mdsclient; + opt.mdsClient = mdsclient_; opt.metaClient = std::make_shared(); opt.scanPeriodSec = 1; opt.scanLimit = 2; @@ -142,7 +145,7 @@ TEST_F(RecycleManangeTest, test_empty_recycle) { FsInfo fsInfo; fsInfo.set_recycletimehour(10); - EXPECT_CALL(*mdsclient, GetFsInfo(fsId, _)) + EXPECT_CALL(*mdsclient_, GetFsInfo(fsId, _)) .WillRepeatedly( DoAll(SetArgPointee<1>(fsInfo), Return(FSStatusCode::OK))); diff --git a/curvefs/test/metaserver/space/BUILD b/curvefs/test/metaserver/space/BUILD new file mode 100644 index 0000000000..009aca6e26 --- /dev/null +++ b/curvefs/test/metaserver/space/BUILD @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_TEST_COPTS") + +cc_test( + name = "space_test", + srcs = glob( + [ + "*.cpp", + "*.h", + ], + ), + copts = CURVE_TEST_COPTS, + defines = ["UNIT_TEST"], + visibility = ["//visibility:public"], + deps = [ + "//curvefs/src/metaserver:curvefs_metaserver", + "//curvefs/test/metaserver/storage:metaserver_storage_test_utils", + "//curvefs/test/client:mock", + "//curvefs/test/metaserver/copyset/mock:metaserver_copyset_test_mock", + "//external:gflags", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest_prod", + ], +) \ No newline at end of file diff --git a/curvefs/test/metaserver/space/inode_volume_space_deallocate_test.cpp b/curvefs/test/metaserver/space/inode_volume_space_deallocate_test.cpp new file mode 100644 index 0000000000..0bd1513379 --- /dev/null +++ b/curvefs/test/metaserver/space/inode_volume_space_deallocate_test.cpp @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 18 11:47:54 CST 2023 + * Author: lixiaocui + */ + +#include +#include + +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" +#include "curvefs/src/metaserver/inode_storage.h" +#include "curvefs/src/metaserver/storage/storage.h" +#include "curvefs/src/metaserver/storage/rocksdb_storage.h" +#include "curvefs/src/metaserver/storage/converter.h" +#include "src/fs/ext4_filesystem_impl.h" +#include "curvefs/test/metaserver/storage/utils.h" +#include "curvefs/test/metaserver/space/mock_volume_space_manager.h" +#include "curvefs/test/client/mock_metaserver_client.h" +#include "curvefs/test/metaserver/space/utils.h" +#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" +#include "curvefs/src/client/rpcclient/task_excutor.h" + +namespace curvefs { +namespace metaserver { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +using ::curvefs::client::rpcclient::MetaServerClientDone; + +namespace { +auto localfs = curve::fs::Ext4FileSystemImpl::getInstance(); +} + +using curvefs::client::rpcclient::MockMetaServerClient; +using curvefs::metaserver::NameGenerator; +using curvefs::metaserver::copyset::MockCopysetNode; +using curvefs::metaserver::storage::Key4VolumeExtentSlice; +using curvefs::metaserver::storage::KVStorage; +using curvefs::metaserver::storage::RandomStoragePath; +using curvefs::metaserver::storage::RocksDBStorage; +using curvefs::metaserver::storage::StorageOptions; + + +class InodeVolumeSpaceDeallocateTest : public ::testing::Test { + protected: + void SetUp() override { + dataDir_ = RandomStoragePath(); + StorageOptions options; + options.dataDir = dataDir_; + options.localFileSystem = localfs.get(); + kvStorage_ = std::make_shared(options); + ASSERT_TRUE(kvStorage_->Open()); + + nameGen_ = std::make_shared(1); + inodeStorage_ = std::make_shared(kvStorage_, nameGen_, 0); + + volumeSpaceManager_ = std::make_shared(); + metaClient_ = std::make_shared(); + + VolumeDeallocateCalOption calOpt; + calOpt.kvStorage = kvStorage_; + calOpt.inodeStorage = inodeStorage_; + calOpt.nameGen = nameGen_; + + VolumeDeallocateExecuteOption executeOpt; + executeOpt.volumeSpaceManager = volumeSpaceManager_; + executeOpt.metaClient = metaClient_; + executeOpt.batchClean = 10; + + EXPECT_CALL(*volumeSpaceManager_, GetBlockGroupSize(_)) + .WillRepeatedly(Return(blockGroupSize_)); + inodeVolumeSpaceDeallocate_ = + std::make_shared(fsId_, partitionId_, + copysetNode_); + inodeVolumeSpaceDeallocate_->Init(calOpt); + inodeVolumeSpaceDeallocate_->Init(executeOpt); + inodeVolumeSpaceDeallocate_->metaCli_ = metaClient_; + } + + void TearDown() override { + ASSERT_TRUE(kvStorage_->Close()); + auto output = execShell("rm -rf " + dataDir_); + ASSERT_EQ(output.size(), 0); + } + + void PrepareOneDeallocatableInode(Inode *inode, VolumeExtentSlice *slice) { + // inode + uint64_t len = 1024 * 1024 * 1024 + 8 * 1024; + inode->set_inodeid(1); + inode->set_fsid(fsId_); + inode->set_length(len); + inode->set_ctime(1681884587); + inode->set_ctime_ns(0); + inode->set_mtime(1681884587); + inode->set_mtime_ns(0); + inode->set_atime(1681884587); + inode->set_atime_ns(0); + inode->set_uid(100); + inode->set_gid(100); + inode->set_mode(777); + inode->set_nlink(0); + inode->set_type(FsFileType::TYPE_FILE); + + // extent + uint64_t validBlockLen = blockGroupSize_ - 4 * 1024; + slice->set_offset(0); + uint16_t blockNum = len / validBlockLen; + for (int i = 0; i <= blockNum; i++) { + auto extent = slice->add_extents(); + extent->set_fsoffset(validBlockLen * i); + extent->set_volumeoffset(blockGroupSize_ * i + 4 * 1024); + if (i == blockNum) { + extent->set_length(len - validBlockLen * (i - 1)); + extent->set_isused(false); + } else { + extent->set_length(validBlockLen); + extent->set_isused(true); + } + } + + // persist extent + Converter conv; + auto sliceKey = conv.SerializeToString( + Key4VolumeExtentSlice(fsId_, 1, slice->offset())); + auto st = kvStorage_->SSet(nameGen_->GetVolumeExtentTableName(), + sliceKey, *slice); + ASSERT_TRUE(st.ok()); + + // persist inode + auto inodeKey = + conv.SerializeToString(Key4Inode(inode->fsid(), inode->inodeid())); + st = kvStorage_->HSet(nameGen_->GetInodeTableName(), inodeKey, *inode); + ASSERT_TRUE(st.ok()); + + // persist deallocatable inode + st = kvStorage_->HSet(nameGen_->GetDeallocatableInodeTableName(), + inodeKey, google::protobuf::Empty()); + ASSERT_TRUE(st.ok()); + } + + void PrepareDeallocatableGroup(Inode *inode, VolumeExtentSlice *slice, + DeallocatableBlockGroupMap *demap) { + PrepareOneDeallocatableInode(inode, slice); + + for (auto extent : slice->extents()) { + DeallocatableBlockGroup tmp; + uint64_t blockgroupOffset = + extent.volumeoffset() / blockGroupSize_ * blockGroupSize_; + tmp.set_blockgroupoffset(blockgroupOffset); + tmp.set_deallocatablesize(extent.length()); + tmp.add_inodeidlist(inode->inodeid()); + Key4DeallocatableBlockGroup key(fsId_, blockgroupOffset); + + auto st = kvStorage_->HSet( + nameGen_->GetDeallocatableBlockGroupTableName(), + key.SerializeToString(), tmp); + ASSERT_TRUE(st.ok()); + + (*demap)[blockgroupOffset] = std::move(tmp); + } + } + + protected: + std::string dataDir_; + std::shared_ptr kvStorage_; + std::shared_ptr inodeStorage_; + std::shared_ptr volumeSpaceManager_; + std::shared_ptr metaClient_; + std::shared_ptr nameGen_; + std::shared_ptr copysetNode_; + + std::shared_ptr inodeVolumeSpaceDeallocate_; + + uint64_t fsId_ = 1; + uint32_t partitionId_ = 1; + uint64_t blockGroupSize_ = 128 * 1024 * 1024; +}; + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_Status) { + ASSERT_FALSE(inodeVolumeSpaceDeallocate_->IsCanceled()); + + inodeVolumeSpaceDeallocate_->SetCanceled(); + ASSERT_TRUE(inodeVolumeSpaceDeallocate_->IsCanceled()); + + inodeVolumeSpaceDeallocate_->SetDeallocateTask(1024); + ASSERT_EQ(1024, inodeVolumeSpaceDeallocate_->GetDeallocateTask()); + + inodeVolumeSpaceDeallocate_->ResetDeallocateTask(); + ASSERT_FALSE(inodeVolumeSpaceDeallocate_->HasDeallocateTask()); +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_CalDeallocatableSpace) { + // test with empty kvstorage + inodeVolumeSpaceDeallocate_->CalDeallocatableSpace(); + + // test with one inode kvstorage + Inode inode; + VolumeExtentSlice slice; + PrepareOneDeallocatableInode(&inode, &slice); + EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(_, _, _)) + .WillOnce(Return(MetaStatusCode::OK)); + inodeVolumeSpaceDeallocate_->CalDeallocatableSpace(); +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocatableSpaceForInode) { + Inode inode; + VolumeExtentSlice slice; + PrepareOneDeallocatableInode(&inode, &slice); + auto key = Key4Inode(inode.fsid(), inode.inodeid()); + + DeallocatableBlockGroupMap increaseMap; + inodeVolumeSpaceDeallocate_->DeallocatableSpaceForInode(key, &increaseMap); + + ASSERT_EQ(increaseMap.size(), slice.extents_size()); + auto iter = slice.extents().begin(); + for (auto &item : increaseMap) { + auto expectedOffset = + iter->volumeoffset() / blockGroupSize_ * blockGroupSize_; + ASSERT_EQ(item.first, expectedOffset); + ASSERT_EQ(item.second.blockgroupoffset(), expectedOffset); + + auto &increase = item.second.increase(); + ASSERT_EQ(increase.increasedeallocatablesize(), iter->length()); + ASSERT_EQ(increase.inodeidlistadd()[0], inode.inodeid()); + ASSERT_EQ(increase.inodeidlistadd().size(), 1); + + iter++; + } +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocateOneBlockGroup) { + uint64_t blockGroupOffset = 0; + + // no decallocatable block group + inodeVolumeSpaceDeallocate_->DeallocateOneBlockGroup(blockGroupOffset); + + // one deallocatable block group + { + Inode inode; + VolumeExtentSlice slice; + DeallocatableBlockGroupMap demap; + PrepareDeallocatableGroup(&inode, &slice, &demap); + + EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(fsId_, _, _)) + .Times(2) + .WillRepeatedly(Return(MetaStatusCode::OK)); + EXPECT_CALL(*volumeSpaceManager_, + DeallocVolumeSpace(fsId_, blockGroupOffset, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _)) + .WillOnce( + Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &, + MetaServerClientDone *done) { + done->SetMetaStatusCode(MetaStatusCode::OK); + done->Run(); + })); + inodeVolumeSpaceDeallocate_->DeallocateOneBlockGroup(blockGroupOffset); + } +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_ProcessSepcifyInodeList) { + Inode inode; + VolumeExtentSlice slice; + uint64_t blockGroupOffset = 0; + DeallocatableBlockGroupMap demap; + PrepareDeallocatableGroup(&inode, &slice, &demap); + demap[blockGroupOffset].mutable_mark()->add_inodeidunderdeallocate( + inode.inodeid()); + + EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(fsId_, _, _)) + .Times(2) + .WillRepeatedly(Return(MetaStatusCode::OK)); + EXPECT_CALL(*volumeSpaceManager_, + DeallocVolumeSpace(fsId_, blockGroupOffset, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _)) + .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &, + MetaServerClientDone *done) { + done->SetMetaStatusCode(MetaStatusCode::OK); + done->Run(); + })); + + inodeVolumeSpaceDeallocate_->ProcessSepcifyInodeList(blockGroupOffset, + &demap); + ASSERT_TRUE( + demap[blockGroupOffset].mark().inodeidunderdeallocate().empty()); +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocateInode) { + Inode inode; + VolumeExtentSlice slice; + DeallocatableBlockGroupMap demap; + PrepareDeallocatableGroup(&inode, &slice, &demap); + uint64_t blockGroupOffset = 0; + uint64_t decrease = 0; + Uint64Vec inodelist; + inodelist.Add(inode.inodeid()); + + EXPECT_CALL(*volumeSpaceManager_, + DeallocVolumeSpace(fsId_, blockGroupOffset, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _)) + .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &, + MetaServerClientDone *done) { + done->SetMetaStatusCode(MetaStatusCode::OK); + done->Run(); + })); + inodeVolumeSpaceDeallocate_->DeallocateInode(blockGroupOffset, inodelist, + &decrease); + ASSERT_EQ(decrease, slice.extents(0).length()); +} + +TEST_F(InodeVolumeSpaceDeallocateTest, Test_UpdateDeallocateInodeExtentSlice) { + Inode inode; + VolumeExtentSlice slice; + PrepareOneDeallocatableInode(&inode, &slice); + uint64_t blockGroupOffset = 0; + uint64_t decrease = 0; + VolumeExtentSliceList slicelist; + std::vector deallocatableVolumeSpace; + + auto st = + inodeStorage_->GetAllVolumeExtent(fsId_, inode.inodeid(), &slicelist); + ASSERT_EQ(st, MetaStatusCode::OK); + + // deallocate blockGroupOffset = 0 + { + inodeVolumeSpaceDeallocate_->UpdateDeallocateInodeExtentSlice( + blockGroupOffset, &decrease, &slicelist, &deallocatableVolumeSpace); + + ASSERT_EQ(slice.extents(0).length(), decrease); + ASSERT_EQ(deallocatableVolumeSpace.size(), 1); + ASSERT_EQ(deallocatableVolumeSpace[0].offset, + slice.extents(0).volumeoffset()); + ASSERT_EQ(deallocatableVolumeSpace[0].len, slice.extents(0).length()); + ASSERT_EQ(slice.extents_size() - 1, slicelist.slices(0).extents_size()); + + deallocatableVolumeSpace.clear(); + decrease = 0; + } + + // deallocate blockGroupOffset = blockGroupSize + { + blockGroupOffset = blockGroupSize_; + inodeVolumeSpaceDeallocate_->UpdateDeallocateInodeExtentSlice( + blockGroupOffset, &decrease, &slicelist, &deallocatableVolumeSpace); + + ASSERT_EQ(slice.extents(1).length(), decrease); + ASSERT_EQ(deallocatableVolumeSpace.size(), 1); + ASSERT_EQ(deallocatableVolumeSpace[0].offset, + slice.extents(1).volumeoffset()); + ASSERT_EQ(deallocatableVolumeSpace[0].len, slice.extents(1).length()); + ASSERT_EQ(slice.extents_size() - 2, slicelist.slices(0).extents_size()); + } +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/test/metaserver/space/mock_inode_volume_space_deallocate.h b/curvefs/test/metaserver/space/mock_inode_volume_space_deallocate.h new file mode 100644 index 0000000000..83f5577136 --- /dev/null +++ b/curvefs/test/metaserver/space/mock_inode_volume_space_deallocate.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 20:09:45 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_TEST_METASERVER_SPACE_MOCK_INODE_VOLUME_SPACE_DEALLOCATE_H_ +#define CURVEFS_TEST_METASERVER_SPACE_MOCK_INODE_VOLUME_SPACE_DEALLOCATE_H_ + +#include + +#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h" + +namespace curvefs { +namespace metaserver { + +class MockInodeVolumeSpaceDeallocate : public InodeVolumeSpaceDeallocate { + public: + MockInodeVolumeSpaceDeallocate() + : InodeVolumeSpaceDeallocate(InodeVolumeSpaceDeallocateOption{}, + nullptr, 0, 0) {} + MOCK_METHOD0(CalDeallocatableSpace, void()); +}; + +} // namespace metaserver +} // namespace curvefs +#endif // CURVEFS_TEST_METASERVER_SPACE_MOCK_INODE_VOLUME_SPACE_DEALLOCATE_H_ diff --git a/curvefs/test/metaserver/space/mock_volume_space_manager.h b/curvefs/test/metaserver/space/mock_volume_space_manager.h new file mode 100644 index 0000000000..1f30b45b24 --- /dev/null +++ b/curvefs/test/metaserver/space/mock_volume_space_manager.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 18 16:34:03 CST 2023 + * Author: lixiaocui + */ + +#include +#include +#include "curvefs/src/metaserver/space/volume_space_manager.h" + +#ifndef CURVEFS_TEST_METASERVER_SPACE_MOCK_VOLUME_SPACE_MANAGER_H_ +#define CURVEFS_TEST_METASERVER_SPACE_MOCK_VOLUME_SPACE_MANAGER_H_ + +namespace curvefs { +namespace metaserver { + +class MockVolumeSpaceManager : public VolumeSpaceManager { + public: + MOCK_METHOD3(DeallocVolumeSpace, + bool(uint32_t, uint64_t, const std::vector &)); + MOCK_METHOD1(Destroy, void(uint32_t)); + MOCK_METHOD1(GetBlockGroupSize, uint64_t(uint32_t)); +}; +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_TEST_METASERVER_SPACE_MOCK_VOLUME_SPACE_MANAGER_H_ diff --git a/curvefs/test/metaserver/space/utils.h b/curvefs/test/metaserver/space/utils.h new file mode 100644 index 0000000000..55bfa4e864 --- /dev/null +++ b/curvefs/test/metaserver/space/utils.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wed Apr 26 10:03:18 CST 2023 + * Author: lixiaocui + */ + +#ifndef CURVEFS_TEST_METASERVER_SPACE_UTILS_H_ +#define CURVEFS_TEST_METASERVER_SPACE_UTILS_H_ + +#include + +namespace curvefs { +namespace metaserver { +inline std::string execShell(const std::string &cmd) { + std::array buffer; + std::string result; + std::unique_ptr pipe(popen(cmd.c_str(), "r"), + pclose); + if (!pipe) { + throw std::runtime_error("popen() failed!"); + } + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { + result += buffer.data(); + } + return result; +} +} // namespace metaserver +} // namespace curvefs + +#endif // CURVEFS_TEST_METASERVER_SPACE_UTILS_H_ diff --git a/curvefs/test/metaserver/space/volume_deallocate_manager_test.cpp b/curvefs/test/metaserver/space/volume_deallocate_manager_test.cpp new file mode 100644 index 0000000000..bcc6b7e3cc --- /dev/null +++ b/curvefs/test/metaserver/space/volume_deallocate_manager_test.cpp @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Apr 25 19:35:13 CST 2023 + * Author: lixiaocui + */ + +#include + +#include "curvefs/test/metaserver/storage/utils.h" +#include "curvefs/src/metaserver/storage/rocksdb_storage.h" +#include "curvefs/src/metaserver/space/volume_deallocate_manager.h" +#include "curvefs/test/metaserver/space/mock_volume_space_manager.h" +#include "curvefs/test/client/mock_metaserver_client.h" +#include "src/fs/ext4_filesystem_impl.h" +#include "curvefs/test/metaserver/space/utils.h" +#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" + + +namespace curvefs { +namespace metaserver { + +using curvefs::client::rpcclient::MockMetaServerClient; +using curvefs::metaserver::storage::RandomStoragePath; +using curvefs::metaserver::storage::RocksDBStorage; +using curvefs::metaserver::storage::StorageOptions; + +using ::testing::_; +using ::testing::Return; + +namespace { +auto localfs = curve::fs::Ext4FileSystemImpl::getInstance(); +} + +TEST(VolumeDeallocateManager, Run) { + auto &instance = VolumeDeallocateManager::GetInstance(); + VolumeDeallocateWorkerQueueOption workerOpt; + workerOpt.enable = true; + workerOpt.workerNum = 10; + + VolumeDeallocateExecuteOption executeOpt; + auto volumeSpaceManager = std::make_shared(); + auto metaClient = std::make_shared(); + auto copyset = std::make_shared(); + executeOpt.volumeSpaceManager = volumeSpaceManager; + executeOpt.metaClient = metaClient; + executeOpt.batchClean = 10; + + // double run + { + instance.Init(workerOpt, executeOpt); + instance.Run(); + instance.Run(); + instance.Stop(); + } + + // register and run + { + instance.Run(); + + // prepare task + auto dataDir = RandomStoragePath(); + StorageOptions options; + options.dataDir = dataDir; + options.localFileSystem = localfs.get(); + auto kvStorage = std::make_shared(options); + ASSERT_TRUE(kvStorage->Open()); + + uint64_t fsId = 1; + uint64_t blockGroupSize = 128 * 1024 * 1024; + + EXPECT_CALL(*volumeSpaceManager, GetBlockGroupSize(_)) + .WillRepeatedly(Return(blockGroupSize)); + EXPECT_CALL(*copyset, IsLeaderTerm()) + .WillOnce(Return(false)) + .WillRepeatedly(Return(true)); + + std::thread t([&]() { + for (int i = 1; i <= 100; i++) { + auto nameGen = std::make_shared(i); + auto inodeStorage = + std::make_shared(kvStorage, nameGen, 0); + + VolumeDeallocateCalOption calOpt; + calOpt.kvStorage = kvStorage; + calOpt.inodeStorage = inodeStorage; + calOpt.nameGen = nameGen; + + InodeVolumeSpaceDeallocate task(fsId, i, copyset); + task.Init(calOpt); + + instance.Register(task); + + if (i >= 2 && i <= 20) { + instance.Cancel(i - 1); + } + + if (i > 20 && i < 40) { + instance.HasDeallocate(); + instance.Deallocate(i - 1, 0); + } + } + }); + + // wait and stop + sleep(30); + t.join(); + ASSERT_FALSE(instance.HasDeallocate()); + instance.Stop(); + + ASSERT_TRUE(kvStorage->Close()); + auto output = execShell("rm -rf " + dataDir); + ASSERT_EQ(output.size(), 0); + } +} + +} // namespace metaserver +} // namespace curvefs diff --git a/curvefs/test/volume/block_group_loader_test.cpp b/curvefs/test/volume/block_group_loader_test.cpp index 2b7af2b319..3e97f2c775 100644 --- a/curvefs/test/volume/block_group_loader_test.cpp +++ b/curvefs/test/volume/block_group_loader_test.cpp @@ -51,11 +51,24 @@ class BlockGroupBitmapLoaderTest : public ::testing::Test { option_.bitmapAllocatorOption.sizePerBit = 4 * kMiB; option_.bitmapAllocatorOption.smallAllocProportion = 0; + BlockGroup blockGroup; + GetBlockGroup(false, &blockGroup); + loader_ = absl::make_unique( - &blockDev_, kBlockSize, kBlockGroupOffset, kBlockGroupSize, - kBitmapLocation, false, option_); + &blockDev_, kBlockSize, option_, blockGroup); } + void GetBlockGroup(bool available, BlockGroup *blockGroup) { + blockGroup->set_offset(kBlockGroupOffset); + blockGroup->set_size(kBlockGroupSize); + blockGroup->set_bitmaplocation(kBitmapLocation); + if (available) { + blockGroup->set_available(kBlockGroupSize); + } else { + blockGroup->set_available(0); + } + } + protected: MockBlockDeviceClient blockDev_; AllocatorOption option_; @@ -93,9 +106,10 @@ TEST_F(BlockGroupBitmapLoaderTest, LoadTest_Success) { } TEST_F(BlockGroupBitmapLoaderTest, LoadTest_LoadFromACleanBlockGroup) { - loader_ = absl::make_unique( - &blockDev_, kBlockSize, kBlockGroupOffset, kBlockGroupSize, - kBitmapLocation, true, option_); + BlockGroup blockGroup; + GetBlockGroup(true, &blockGroup); + loader_ = absl::make_unique(&blockDev_, kBlockSize, + option_, blockGroup); EXPECT_CALL(blockDev_, Read(_, _, _)) .Times(0); @@ -138,9 +152,10 @@ TEST_F(BlockGroupBitmapLoaderTest, ISSUE_1457) { int count = 100; while (count-- > 0) { + BlockGroup blockGroup; + GetBlockGroup(false, &blockGroup); loader_ = absl::make_unique( - &blockDev_, kBlockSize, kBlockGroupOffset, kBlockGroupSize, - kBitmapLocation, false, option_); + &blockDev_, kBlockSize, option_, blockGroup); AllocatorAndBitmapUpdater out; EXPECT_TRUE(loader_->Load(&out)); diff --git a/curvefs/test/volume/mock/mock_space_manager.h b/curvefs/test/volume/mock/mock_space_manager.h index 25fa899ff3..b85aad26f2 100644 --- a/curvefs/test/volume/mock/mock_space_manager.h +++ b/curvefs/test/volume/mock/mock_space_manager.h @@ -35,9 +35,11 @@ namespace volume { class MockSpaceManager : public SpaceManager { public: MOCK_METHOD3(Alloc, - bool(uint32_t, const AllocateHint&, std::vector*)); - MOCK_METHOD1(DeAlloc, bool(const std::vector&)); + bool(uint32_t, const AllocateHint &, std::vector *)); + MOCK_METHOD2(DeAlloc, bool(uint64_t, const std::vector &)); MOCK_METHOD0(Shutdown, bool()); + MOCK_METHOD0(Run, void()); + MOCK_METHOD0(GetBlockGroupSize, uint64_t()); }; } // namespace volume diff --git a/curvefs/test/volume/space_manager_test.cpp b/curvefs/test/volume/space_manager_test.cpp index 5be8b57943..6a23a30587 100644 --- a/curvefs/test/volume/space_manager_test.cpp +++ b/curvefs/test/volume/space_manager_test.cpp @@ -94,6 +94,9 @@ class SpaceManagerImplTest : public ::testing::Test { opt_.allocatorOption.bitmapAllocatorOption.sizePerBit = 0; opt_.allocatorOption.bitmapAllocatorOption.smallAllocProportion = 0; + opt_.threshold = 0.5; + opt_.releaseInterSec = 0; + mdsClient_ = std::make_shared(); devClient_ = std::make_shared(); spaceManager_.reset(new SpaceManagerImpl(opt_, mdsClient_, devClient_)); @@ -144,6 +147,12 @@ TEST_F(SpaceManagerImplTest, TestAlloc) { ASSERT_EQ(opt_.blockGroupManagerOption.blockGroupSize / kBlockSize - 1, extents.size()); + + spaceManager_->Run(); + sleep(1); + EXPECT_CALL(*mdsClient_, ReleaseVolumeBlockGroup(_, _, _)) + .WillOnce(Return(SpaceErrCode::SpaceErrUnknown)); + ASSERT_FALSE(spaceManager_->Shutdown()); } TEST_F(SpaceManagerImplTest, TestMultiThreadAlloc) { @@ -199,6 +208,14 @@ TEST_F(SpaceManagerImplTest, TestMultiThreadAlloc) { } ASSERT_EQ(kBlockGroupSize, totalAllocated); + + EXPECT_CALL(*mdsClient_, ReleaseVolumeBlockGroup(_, _, _)) + .WillOnce(Return(SpaceErrCode::SpaceOk)); + spaceManager_->Run(); + sleep(1); + EXPECT_CALL(*mdsClient_, ReleaseVolumeBlockGroup(_, _, _)) + .WillOnce(Return(SpaceErrCode::SpaceOk)); + ASSERT_TRUE(spaceManager_->Shutdown()); } } // namespace volume diff --git a/docs/cn/curvefs_volume_space_deallocate.md b/docs/cn/curvefs_volume_space_deallocate.md index 0b77f05326..38e6873c27 100644 --- a/docs/cn/curvefs_volume_space_deallocate.md +++ b/docs/cn/curvefs_volume_space_deallocate.md @@ -23,7 +23,7 @@ message VolumeExtentSlice { repeated VolumeExtent extents = 2; } -message VolumeExtentList { +message VolumeExtenSlicetList { repeated VolumeExtentSlice slices = 1; } ``` diff --git a/tools-v2/mk-proto.sh b/tools-v2/mk-proto.sh index 0659d0b923..300f144cdb 100644 --- a/tools-v2/mk-proto.sh +++ b/tools-v2/mk-proto.sh @@ -77,6 +77,7 @@ protoc --go_out=proto --proto_path=.. \ protoc --go_out=proto --proto_path=.. \ --go_opt=Mcurvefs/proto/common.proto=github.com/opencurve/curve/tools-v2/proto/curvefs/proto/common \ --go_opt=Mproto/heartbeat.proto=github.com/opencurve/curve/tools-v2/proto/proto/heartbeat \ + --go_opt=Mcurvefs/proto/metaserver.proto=github.com/opencurve/curve/tools-v2/proto/curvefs/proto/metaserver \ ../curvefs/proto/heartbeat.proto ### curvefs/proto/mds.proto