From 969a67587996e1e0bc95a04e4e00c23b45a0f678 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Wed, 23 Jun 2021 20:04:17 +0800 Subject: [PATCH] client/chunkserver/curve-nbd: support 512 aligned IO requests --- conf/chunkserver.conf.example | 2 + conf/client.conf | 7 + conf/cs_client.conf | 7 + conf/py_client.conf | 7 + conf/snap_client.conf | 7 + .../roles/generate_config/defaults/main.yml | 3 + .../templates/chunkserver.conf.j2 | 2 + .../generate_config/templates/client.conf.j2 | 7 + .../local/chunkserver/conf/chunkserver.conf.0 | 2 + .../local/chunkserver/conf/chunkserver.conf.1 | 2 + .../local/chunkserver/conf/chunkserver.conf.2 | 2 + include/chunkserver/chunkserver_common.h | 3 - nbd/src/NBDController.cpp | 6 +- nbd/src/define.h | 3 +- nbd/src/main.cpp | 1 + nbd/src/util.cpp | 12 +- robot/curve_robot.txt | 37 +- src/chunkserver/chunk_service.cpp | 15 +- src/chunkserver/chunkserver.cpp | 12 + .../datastore/chunkserver_chunkfile.cpp | 45 +- .../datastore/chunkserver_chunkfile.h | 17 +- src/chunkserver/datastore/define.h | 2 + src/client/client_config.cpp | 29 + src/client/config_info.h | 8 +- src/client/libcurve_file.cpp | 8 + src/client/libcurve_file.h | 5 +- src/client/metacache.h | 4 + src/client/request_closure.cpp | 131 +++++ src/client/request_closure.h | 64 +- src/client/request_context.h | 22 +- src/client/request_scheduler.cpp | 26 +- src/client/request_scheduler.h | 4 +- src/client/splitor.cpp | 58 +- src/client/splitor.h | 16 +- src/common/configuration.cpp | 5 + src/common/configuration.h | 1 + src/common/fast_align.h | 61 ++ test/chunkserver/chunk_service_test.cpp | 2 + test/chunkserver/chunk_service_test2.cpp | 2 + .../chunkserver/chunkserver_snapshot_test.cpp | 2 + .../datastore/datastore_mock_unittest.cpp | 354 ++++++++++++ ...raftsnapshot_chunkfilepool_integration.cpp | 2 + test/client/BUILD | 18 +- test/client/copyset_client_test.cpp | 6 + test/client/iotracker_align_test.cpp | 545 ++++++++++++++++++ test/client/mock/mock_chunk_service.h | 48 ++ test/client/mock/mock_meta_cache.h | 3 + test/client/mock/mock_request_context.h | 41 +- test/client/splitor_test.cpp | 267 +++++++++ test/common/fast_align_test.cpp | 54 ++ .../chunkserver/chunkserver_basic_test.cpp | 1 + .../chunkserver_concurrent_test.cpp | 1 + test/integration/common/chunkservice_op.cpp | 1 + .../raft/raft_config_change_test.cpp | 1 + .../raft/raft_log_replication_test.cpp | 2 + test/integration/raft/raft_snapshot_test.cpp | 2 + test/integration/raft/raft_vote_test.cpp | 2 + 57 files changed, 1876 insertions(+), 121 deletions(-) create mode 100644 src/common/fast_align.h create mode 100644 test/client/iotracker_align_test.cpp create mode 100644 test/client/mock/mock_chunk_service.h create mode 100644 test/client/splitor_test.cpp create mode 100644 test/common/fast_align_test.cpp diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example index ce1b3f10e5..bd8055576a 100644 --- a/conf/chunkserver.conf.example +++ b/conf/chunkserver.conf.example @@ -14,6 +14,8 @@ global.chunk_size=16777216 global.meta_page_size=4096 # clone chunk允许的最长location长度 global.location_limit=3000 +# minimum alignment for io request +global.min_io_alignment=512 # # MDS settings diff --git a/conf/client.conf b/conf/client.conf index 971e90fca9..559ff73f83 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -164,3 +164,10 @@ discard.enable=true discard.granularity=4096 # discard cleanup task delay times in millisecond discard.taskDelayMs=60000 + +##### alignment ##### +# default alignment +global.alignment.commonVolume=512 +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume=4096 diff --git a/conf/cs_client.conf b/conf/cs_client.conf index 98e00e77f9..1e98564b80 100644 --- a/conf/cs_client.conf +++ b/conf/cs_client.conf @@ -159,3 +159,10 @@ discard.enable=false discard.granularity=4096 # discard cleanup task delay times in millisecond discard.taskDelayMs=60000 + +##### alignment ##### +# default alignment +global.alignment.commonVolume=512 +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume=4096 diff --git a/conf/py_client.conf b/conf/py_client.conf index 0a681aa0b1..3e82865d36 100644 --- a/conf/py_client.conf +++ b/conf/py_client.conf @@ -153,3 +153,10 @@ discard.enable=false discard.granularity=4096 # discard cleanup task delay times in millisecond discard.taskDelayMs=60000 + +##### alignment ##### +# default alignment +global.alignment.commonVolume=512 +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume=4096 diff --git a/conf/snap_client.conf b/conf/snap_client.conf index 4e11946a57..2c410d3c62 100644 --- a/conf/snap_client.conf +++ b/conf/snap_client.conf @@ -159,3 +159,10 @@ discard.enable=false discard.granularity=4096 # discard cleanup task delay times in millisecond discard.taskDelayMs=60000 + +##### alignment ##### +# default alignment +global.alignment.commonVolume=512 +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume=4096 diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml index 068c35ed59..fe801b1d14 100644 --- a/curve-ansible/roles/generate_config/defaults/main.yml +++ b/curve-ansible/roles/generate_config/defaults/main.yml @@ -160,6 +160,7 @@ chunkserver_walfilepool_retry_times: 5 chunkserver_trash_expire_after_sec: 300 chunkserver_trash_scan_period_sec: 120 chunkserver_common_log_dir: ./runlog/ +chunkserver_min_io_alignment: 512 # 快照克隆配置默认值 snap_client_config_path: /etc/curve/snap_client.conf @@ -239,6 +240,8 @@ client_throttle_enable: false client_discard_enable: true client_discard_granularity: 4096 client_discard_task_delay_ms: 60000 +client_alignment_common: 512 +client_alignment_clone: 4096 # nebd默认配置 client_config_path: /etc/curve/client.conf diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 index 1ff5aac4e4..34dea8bc4c 100644 --- a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 @@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }} global.meta_page_size={{ chunkserver_meta_page_size }} # clone chunk允许的最长location长度 global.location_limit={{ chunkserver_location_limit }} +# minimum alignment for io request +global.min_io_alignment={{ chunkserver_min_io_alignment }} # # MDS settings diff --git a/curve-ansible/roles/generate_config/templates/client.conf.j2 b/curve-ansible/roles/generate_config/templates/client.conf.j2 index 08d4413780..c9ab27029c 100644 --- a/curve-ansible/roles/generate_config/templates/client.conf.j2 +++ b/curve-ansible/roles/generate_config/templates/client.conf.j2 @@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }} discard.granularity={{ client_discard_granularity }} # discard cleanup task delay times in millisecond discard.taskDelayMs={{ client_discard_task_delay_ms }} + +##### alignment ##### +# default alignment +global.alignment.commonVolume={{ client_alignment_common }} +# alignment for clone volume +# default is 4096, because lazy clone chunk bitmap granularity is 4096 +global.alignment.cloneVolume={{ client_alignment_clone }} diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0 index c4e8cd2ef1..dc5a40c31a 100644 --- a/deploy/local/chunkserver/conf/chunkserver.conf.0 +++ b/deploy/local/chunkserver/conf/chunkserver.conf.0 @@ -28,6 +28,8 @@ global.chunk_size=16777216 global.meta_page_size=4096 global.location_limit=3000 +global.min_io_alignment=512 + # # MDS settings # diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1 index 2a423f3e59..e71bcc195c 100644 --- a/deploy/local/chunkserver/conf/chunkserver.conf.1 +++ b/deploy/local/chunkserver/conf/chunkserver.conf.1 @@ -28,6 +28,8 @@ global.chunk_size=16777216 global.meta_page_size=4096 global.location_limit=3000 +global.min_io_alignment=512 + # # MDS settings # diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2 index e26089aadc..68b61867b0 100644 --- a/deploy/local/chunkserver/conf/chunkserver.conf.2 +++ b/deploy/local/chunkserver/conf/chunkserver.conf.2 @@ -28,6 +28,8 @@ global.chunk_size=16777216 global.meta_page_size=4096 global.location_limit=3000 +global.min_io_alignment=512 + # # MDS settings # diff --git a/include/chunkserver/chunkserver_common.h b/include/chunkserver/chunkserver_common.h index aaa05c2333..4a03a89ee6 100644 --- a/include/chunkserver/chunkserver_common.h +++ b/include/chunkserver/chunkserver_common.h @@ -123,9 +123,6 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId, } #define ToGroupIdStr ToGroupIdString -// TODO(wudmeiao): 是否需要考虑可配置 -const uint32_t kOpRequestAlignSize = 4096; - } // namespace chunkserver } // namespace curve diff --git a/nbd/src/NBDController.cpp b/nbd/src/NBDController.cpp index 42d8dad525..4c7f1965fa 100644 --- a/nbd/src/NBDController.cpp +++ b/nbd/src/NBDController.cpp @@ -47,7 +47,7 @@ int IOController::InitDevAttr(NBDConfig* config, uint64_t size, int ret = -1; do { - ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE); + ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, config->block_size); if (ret < 0) { break; } @@ -259,7 +259,7 @@ int NetLinkController::SetUp(NBDConfig* config, int sockfd, if (index < 0) { return index; } - ret = check_block_size(index, CURVE_NBD_BLKSIZE); + ret = check_block_size(index, config->block_size); if (ret < 0) { return ret; } @@ -393,7 +393,7 @@ int NetLinkController::ConnectInternal(NBDConfig* config, int sockfd, NLA_PUT_U64(msg, NBD_ATTR_TIMEOUT, config->timeout); } NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size); - NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, CURVE_NBD_BLKSIZE); + NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, config->block_size); NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags); sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS); diff --git a/nbd/src/define.h b/nbd/src/define.h index 3c23d7191e..0e28aa3795 100644 --- a/nbd/src/define.h +++ b/nbd/src/define.h @@ -55,7 +55,6 @@ namespace nbd { #define HELP_INFO 1 #define VERSION_INFO 2 -#define CURVE_NBD_BLKSIZE 4096UL // CURVE后端当前支持4096大小对齐的IO #define NBD_MAX_PATH "/sys/module/nbd/parameters/nbds_max" #define PROCESS_NAME "curve-nbd" @@ -88,6 +87,8 @@ struct NBDConfig { int retry_times = 25; // unmap重试之间的睡眠间隔 int sleep_ms = 200; + // device's block size + int block_size = 4096; }; // 用户命令类型 diff --git a/nbd/src/main.cpp b/nbd/src/main.cpp index 2d73a7e185..3c70ef4eb6 100644 --- a/nbd/src/main.cpp +++ b/nbd/src/main.cpp @@ -87,6 +87,7 @@ static void Usage() { << " --max_part Override for module param max_part\n" << " --timeout Set nbd request timeout\n" << " --try-netlink Use the nbd netlink interface\n" + << " --block-size NBD Devices's block size, default is 4096, support 512 and 4096\n" // NOLINT << "Unmap options:\n" << " -f, --force Force unmap even if the device is mounted\n" // NOLINT << " --retry_times The number of retries waiting for the process to exit\n" // NOLINT diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp index 0f1ebd7853..ab90104a11 100644 --- a/nbd/src/util.cpp +++ b/nbd/src/util.cpp @@ -173,6 +173,17 @@ int parse_args(std::vector& args, std::ostream *err_msg, // NOLIN *err_msg << "curve-nbd: Invalid argument for sleep_ms!"; return -EINVAL; } + } else if (argparse_witharg(args, i, &cfg->block_size, err, "--block_size", (char*)(NULL))) { // NOLINT + if (!err.str().empty()) { + *err_msg << "curve-nbd: " << err.str(); + return -EINVAL; + } + + if (cfg->block_size != 512 && cfg->block_size != 4096) { + *err_msg << "curve-nbd: Invalid block size, only support 512 " + "or 4096"; + return -EINVAL; + } } else { ++i; } @@ -322,7 +333,6 @@ int check_size_from_file(const std::string& path, uint64_t expected_size) { uint64_t size = 0; ifs >> size; - size *= CURVE_NBD_BLKSIZE; if (size == 0) { // Newer kernel versions will report real size only after nbd diff --git a/robot/curve_robot.txt b/robot/curve_robot.txt index 8709a96b6e..33639de83f 100644 --- a/robot/curve_robot.txt +++ b/robot/curve_robot.txt @@ -1449,11 +1449,11 @@ read write iosize 512 ${fd} open libcurve file ${new_fd} Convert To Integer ${fd} ${rc} write libcurve file error ${new_fd} length=512 - ${expect_rc} evaluate int(-22) - should be equal ${rc} ${expect_rc} - ${rc} read libcurve file ${new_fd} length=512 - ${expect_rc} evaluate int(-22) + ${expect_rc} evaluate int(512) should be equal ${rc} ${expect_rc} + ${content} read libcurve file ${new_fd} length=512 + ${expect_content} evaluate str("a")*512 + should be equal ${content} ${expect_content} [Teardown] file clean ${new_fd} @@ -1480,11 +1480,11 @@ read write iosize 512 ${fd} open libcurve file ${new_fd} Convert To Integer ${fd} ${rc} write libcurve file error ${new_fd} length=512 - ${expect_rc} evaluate int(-22) - should be equal ${rc} ${expect_rc} - ${rc} read libcurve file ${new_fd} length=512 - ${expect_rc} evaluate int(-22) + ${expect_rc} evaluate int(512) should be equal ${rc} ${expect_rc} + ${content} read libcurve file ${new_fd} length=512 + ${expect_content} evaluate str("a")*512 + should be equal ${content} ${expect_content} [Teardown] file clean ${new_fd} read write iosize 1k @@ -1493,9 +1493,22 @@ read write iosize 1k ${fd} open libcurve file ${new_fd} Convert To Integer ${fd} ${rc} write libcurve file error ${new_fd} length=1024 + ${expect_rc} evaluate int(1024) + should be equal ${rc} ${expect_rc} + ${content} read libcurve file ${new_fd} length=1024 + ${expect_content} evaluate str("a")*1024 + should be equal ${content} ${expect_content} + [Teardown] file clean ${new_fd} + +read write iosize not 512 alignment + [Tags] P0 base first release + [Setup] file init + ${fd} open libcurve file + ${new_fd} Convert To Integer ${fd} + ${rc} write libcurve file error ${new_fd} length=511 ${expect_rc} evaluate int(-22) should be equal ${rc} ${expect_rc} - ${rc} read libcurve file ${new_fd} length=1024 + ${rc} read libcurve file ${new_fd} length=511 ${expect_rc} evaluate int(-22) should be equal ${rc} ${expect_rc} [Teardown] file clean ${new_fd} @@ -2552,9 +2565,3 @@ check chunkserver should be exsits start mds with sleep start mds Sleep 5s - - - - - - diff --git a/src/chunkserver/chunk_service.cpp b/src/chunkserver/chunk_service.cpp index d70138b7b4..aeeaa3e031 100755 --- a/src/chunkserver/chunk_service.cpp +++ b/src/chunkserver/chunk_service.cpp @@ -36,6 +36,8 @@ #include "src/chunkserver/op_request.h" #include "src/chunkserver/chunk_service_closure.h" +#include "src/common/fast_align.h" + namespace curve { namespace chunkserver { @@ -541,17 +543,8 @@ bool ChunkServiceImpl::CheckRequestOffsetAndLength(uint32_t offset, return false; } - // 检查offset是否对齐 - if (offset % kOpRequestAlignSize != 0) { - return false; - } - - // 检查len是否对齐 - if (len % kOpRequestAlignSize != 0) { - return false; - } - - return true; + return common::is_aligned(offset, FLAGS_minIoAlignment) && + common::is_aligned(len, FLAGS_minIoAlignment); } } // namespace chunkserver diff --git a/src/chunkserver/chunkserver.cpp b/src/chunkserver/chunkserver.cpp index 1249a79e5b..e70acd696e 100644 --- a/src/chunkserver/chunkserver.cpp +++ b/src/chunkserver/chunkserver.cpp @@ -106,6 +106,12 @@ int ChunkServer::Run(int argc, char** argv) { // ============================初始化各模块==========================// LOG(INFO) << "Initializing ChunkServer modules"; + LOG_IF(FATAL, !conf.GetUInt32Value("global.min_io_alignment", + &FLAGS_minIoAlignment)) + << "Failed to get global.min_io_alignment"; + LOG_IF(FATAL, !common::is_aligned(FLAGS_minIoAlignment, 512)) + << "minIoAlignment should align to 512"; + // 优先初始化 metric 收集模块 ChunkServerMetricOptions metricOptions; InitMetricOptions(&conf, &metricOptions); @@ -559,6 +565,8 @@ void ChunkServer::InitCopysetNodeOptions( ©setNodeOptions->maxChunkSize)); LOG_IF(FATAL, !conf->GetUInt32Value("global.location_limit", ©setNodeOptions->locationLimit)); + LOG_IF(FATAL, !conf->GetUInt32Value("global.meta_page_size", + ©setNodeOptions->pageSize)); LOG_IF(FATAL, !conf->GetUInt32Value("copyset.load_concurrency", ©setNodeOptions->loadConcurrency)); LOG_IF(FATAL, !conf->GetUInt32Value("copyset.check_retrytimes", @@ -821,6 +829,10 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) { conf->SetIntValue("copyset.load_concurrency", FLAGS_copysetLoadConcurrency); } + + if (GetCommandLineFlagInfo("minIoAlignment", &info) && !info.is_default) { + conf->SetUInt32Value("global.min_io_alignment", FLAGS_minIoAlignment); + } } int ChunkServer::GetChunkServerMetaFromLocal( diff --git a/src/chunkserver/datastore/chunkserver_chunkfile.cpp b/src/chunkserver/datastore/chunkserver_chunkfile.cpp index 5287dbda08..ef3d71fe22 100644 --- a/src/chunkserver/datastore/chunkserver_chunkfile.cpp +++ b/src/chunkserver/datastore/chunkserver_chunkfile.cpp @@ -30,6 +30,19 @@ namespace curve { namespace chunkserver { +namespace { + +bool ValidMinIoAlignment(const char* flagname, uint32_t value) { + return common::is_aligned(value, 512); +} + +} // namespace + +DEFINE_uint32(minIoAlignment, 512, + "minimum alignment for io request, must align to 512"); + +DEFINE_validator(minIoAlignment, ValidMinIoAlignment); + ChunkFileMetaPage::ChunkFileMetaPage(const ChunkFileMetaPage& metaPage) { version = metaPage.version; sn = metaPage.sn; @@ -276,13 +289,16 @@ CSErrorCode CSChunkFile::Write(SequenceNum sn, size_t length, uint32_t* cost) { WriteLockGuard writeGuard(rwLock_); - if (!CheckOffsetAndLength(offset, length)) { + if (!CheckOffsetAndLength( + offset, length, isCloneChunk_ ? pageSize_ : FLAGS_minIoAlignment)) { LOG(ERROR) << "Write chunk failed, invalid offset or length." << "ChunkID: " << chunkId_ << ", offset: " << offset << ", length: " << length << ", page size: " << pageSize_ - << ", chunk size: " << size_; + << ", chunk size: " << size_ + << ", align: " + << (isCloneChunk_ ? pageSize_ : FLAGS_minIoAlignment); return CSErrorCode::InvalidArgError; } // Curve will ensure that all previous requests arrive or time out @@ -390,19 +406,20 @@ CSErrorCode CSChunkFile::Write(SequenceNum sn, CSErrorCode CSChunkFile::Paste(const char * buf, off_t offset, size_t length) { WriteLockGuard writeGuard(rwLock_); - if (!CheckOffsetAndLength(offset, length)) { + // If it is not a clone chunk, return success directly + if (!isCloneChunk_) { + return CSErrorCode::Success; + } + if (!CheckOffsetAndLength(offset, length, pageSize_)) { LOG(ERROR) << "Paste chunk failed, invalid offset or length." << "ChunkID: " << chunkId_ << ", offset: " << offset << ", length: " << length << ", page size: " << pageSize_ - << ", chunk size: " << size_; + << ", chunk size: " << size_ + << ", align: " << pageSize_; return CSErrorCode::InvalidArgError; } - // If it is not a clone chunk, return success directly - if (!isCloneChunk_) { - return CSErrorCode::Success; - } // The request above must be pagesize aligned // the starting page index number of the paste area @@ -446,13 +463,16 @@ CSErrorCode CSChunkFile::Paste(const char * buf, off_t offset, size_t length) { CSErrorCode CSChunkFile::Read(char * buf, off_t offset, size_t length) { ReadLockGuard readGuard(rwLock_); - if (!CheckOffsetAndLength(offset, length)) { + if (!CheckOffsetAndLength( + offset, length, isCloneChunk_ ? pageSize_ : FLAGS_minIoAlignment)) { LOG(ERROR) << "Read chunk failed, invalid offset or length." << "ChunkID: " << chunkId_ << ", offset: " << offset << ", length: " << length << ", page size: " << pageSize_ - << ", chunk size: " << size_; + << ", chunk size: " << size_ + << ", align: " + << (isCloneChunk_ ? pageSize_ : FLAGS_minIoAlignment); return CSErrorCode::InvalidArgError; } @@ -501,13 +521,14 @@ CSErrorCode CSChunkFile::ReadSpecifiedChunk(SequenceNum sn, off_t offset, size_t length) { ReadLockGuard readGuard(rwLock_); - if (!CheckOffsetAndLength(offset, length)) { + if (!CheckOffsetAndLength(offset, length, pageSize_)) { LOG(ERROR) << "Read specified chunk failed, invalid offset or length." << "ChunkID: " << chunkId_ << ", offset: " << offset << ", length: " << length << ", page size: " << pageSize_ - << ", chunk size: " << size_; + << ", chunk size: " << size_ + << ", align: " << pageSize_; return CSErrorCode::InvalidArgError; } // If the sequence equals the sequence of the current chunk, diff --git a/src/chunkserver/datastore/chunkserver_chunkfile.h b/src/chunkserver/datastore/chunkserver_chunkfile.h index c9e4378ea5..87166f1d28 100644 --- a/src/chunkserver/datastore/chunkserver_chunkfile.h +++ b/src/chunkserver/datastore/chunkserver_chunkfile.h @@ -41,6 +41,8 @@ #include "src/chunkserver/datastore/define.h" #include "src/chunkserver/datastore/file_pool.h" +#include "src/common/fast_align.h" + namespace curve { namespace chunkserver { @@ -356,23 +358,14 @@ class CSChunkFile { return rc; } - inline bool CheckOffsetAndLength(off_t offset, size_t len) { + inline bool CheckOffsetAndLength(off_t offset, size_t len, size_t align) { // Check if offset+len is out of bounds if (offset + len > size_) { return false; } - // Check if the offset is aligned - if (offset % pageSize_ != 0) { - return false; - } - - // Check if len is aligned - if (len % pageSize_ != 0) { - return false; - } - - return true; + return common::is_aligned(offset, align) && + common::is_aligned(len, align); } private: diff --git a/src/chunkserver/datastore/define.h b/src/chunkserver/datastore/define.h index 6cf1255bdf..8ffe3d1e5c 100644 --- a/src/chunkserver/datastore/define.h +++ b/src/chunkserver/datastore/define.h @@ -40,6 +40,8 @@ const uint8_t FORMAT_VERSION = 1; const uint8_t FORMAT_VERSION_V2 = 2; const SequenceNum kInvalidSeq = 0; +DECLARE_uint32(minIoAlignment); + // define error code enum CSErrorCode { // success diff --git a/src/client/client_config.cpp b/src/client/client_config.cpp index e30d88dabe..4efe02af1d 100644 --- a/src/client/client_config.cpp +++ b/src/client/client_config.cpp @@ -28,6 +28,7 @@ #include #include "src/common/net_common.h" +#include "src/common/fast_align.h" #include "src/common/string_util.h" #define RETURN_IF_FALSE(x) \ @@ -37,6 +38,9 @@ namespace curve { namespace client { + +extern uint32_t kMinIOAlignment; + int ClientConfig::Init(const char* configpath) { conf_.SetConfigPath(configpath); @@ -271,6 +275,31 @@ int ClientConfig::Init(const char* configpath) { LOG_IF(ERROR, ret == false) << "config no discard.taskDelayMs info"; RETURN_IF_FALSE(ret); + ret = conf_.GetUInt32Value( + "global.alignment.commonVolume", + &fileServiceOption_.ioOpt.ioSplitOpt.alignment.commonVolume); + LOG_IF(ERROR, ret == false) + << "config no global.alignment.commonVolume info"; + RETURN_IF_FALSE(ret); + + ret = conf_.GetUInt32Value( + "global.alignment.cloneVolume", + &fileServiceOption_.ioOpt.ioSplitOpt.alignment.cloneVolume); + LOG_IF(ERROR, ret == false) + << "config no global.alignment.cloneVolume info"; + RETURN_IF_FALSE(ret); + + if (!common::is_aligned( + fileServiceOption_.ioOpt.ioSplitOpt.alignment.commonVolume, 512) || + !common::is_aligned( + fileServiceOption_.ioOpt.ioSplitOpt.alignment.cloneVolume, 512)) { + LOG(ERROR) << "global.alignment.commonVolume and " + "global.alignment.cloneVolume must align to 512"; + RETURN_IF_FALSE(false); + } + + kMinIOAlignment = + fileServiceOption_.ioOpt.ioSplitOpt.alignment.commonVolume; return 0; } diff --git a/src/client/config_info.h b/src/client/config_info.h index 8f8c4e6161..8a02fb862f 100644 --- a/src/client/config_info.h +++ b/src/client/config_info.h @@ -69,7 +69,7 @@ struct MetaServerOption { uint32_t mdsMaxFailedTimesBeforeChangeMDS = 5; /** - * When the failed times except RPC error + * When the failed times except RPC error * greater than mdsNormalRetryTimesBeforeTriggerWait, * it will trigger wait strategy, and sleep long time before retry */ @@ -203,6 +203,11 @@ struct MetaCacheOption { ChunkServerUnstableOption chunkserverUnstableOption; }; +struct AlignmentOption { + uint32_t commonVolume = 512; + uint32_t cloneVolume = 4096; +}; + /** * IO 拆分模块配置信息 * @fileIOSplitMaxSizeKB: 用户下发IO大小client没有限制，但是client会将用户的IO进行拆分， @@ -210,6 +215,7 @@ struct MetaCacheOption { */ struct IOSplitOption { uint64_t fileIOSplitMaxSizeKB = 64; + AlignmentOption alignment; }; /** diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index 5dc762ddc7..f9cf0e9fc6 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -44,6 +44,7 @@ #include "src/common/net_common.h" #include "src/common/uuid.h" #include "src/common/string_util.h" +#include "src/common/fast_align.h" #define PORT_LIMIT 65535 @@ -62,6 +63,8 @@ namespace client { using curve::common::ReadLockGuard; using curve::common::WriteLockGuard; +uint32_t kMinIOAlignment = 512; + static const int PROCESS_NAME_MAX = 32; static char g_processname[PROCESS_NAME_MAX]; @@ -107,6 +110,11 @@ FileClient::FileClient() inited_(false), openedFileNum_(common::ToHexString(this)) {} +bool FileClient::CheckAligned(off_t offset, size_t length) const { + return common::is_aligned(offset, kMinIOAlignment) && + common::is_aligned(length, kMinIOAlignment); +} + int FileClient::Init(const std::string& configpath) { if (inited_) { LOG(WARNING) << "already inited!"; diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h index bb835cae60..156d386200 100644 --- a/src/client/libcurve_file.h +++ b/src/client/libcurve_file.h @@ -318,10 +318,7 @@ class FileClient { private: bool StartDummyServer(); - bool CheckAligned(off_t offset, size_t length) const { - return (offset % IO_ALIGNED_BLOCK_SIZE == 0) && - (length % IO_ALIGNED_BLOCK_SIZE == 0); - } + bool CheckAligned(off_t offset, size_t length) const; private: BthreadRWLock rwlock_; diff --git a/src/client/metacache.h b/src/client/metacache.h index 21cf9ad54a..9ff41ad768 100644 --- a/src/client/metacache.h +++ b/src/client/metacache.h @@ -228,6 +228,10 @@ class MetaCache { fileInfo_.filestatus = status; } + bool IsCloneFile() const { + return fileInfo_.filestatus == FileStatus::CloneMetaInstalled; + } + /** * 获取对应的copyset的LeaderMayChange标志 */ diff --git a/src/client/request_closure.cpp b/src/client/request_closure.cpp index 54fe46645c..52a3882351 100644 --- a/src/client/request_closure.cpp +++ b/src/client/request_closure.cpp @@ -22,6 +22,8 @@ #include "src/client/request_closure.h" +#include + #include "src/client/io_tracker.h" #include "src/client/iomanager.h" #include "src/client/request_context.h" @@ -52,5 +54,134 @@ void RequestClosure::ReleaseInflightRPCToken() { } } +PaddingReadClosure::PaddingReadClosure(RequestContext* requestCtx, + RequestScheduler* scheduler) + : RequestClosure(requestCtx), alignedCtx_(nullptr), scheduler_(scheduler) { + GenAlignedRequest(); +} + +void PaddingReadClosure::Run() { + std::unique_ptr selfGuard(this); + std::unique_ptr ctxGuard(alignedCtx_); + + const int errCode = GetErrorCode(); + if (errCode != 0) { + HandleError(errCode); + return; + } + + switch (reqCtx_->optype_) { + case OpType::READ: + HandleRead(); + break; + case OpType::WRITE: + HandleWrite(); + break; + default: + HandleError(-1); + LOG(ERROR) << "Unexpected original request optype: " + << static_cast(reqCtx_->optype_) + << ", request context: " << *reqCtx_; + } +} + +void PaddingReadClosure::HandleRead() { + brpc::ClosureGuard doneGuard(reqCtx_->done_); + + // copy data to original read request + auto nc = alignedCtx_->readData_.append_to( + &reqCtx_->readData_, reqCtx_->rawlength_, + reqCtx_->offset_ - alignedCtx_->offset_); + + if (nc != reqCtx_->rawlength_) { + LOG(ERROR) << "Copy read data failed, coyied bytes: " << nc + << ", expected bytes: " << reqCtx_->rawlength_; + reqCtx_->done_->SetFailed(-1); + } else { + reqCtx_->done_->SetFailed(0); + } +} + +void PaddingReadClosure::HandleWrite() { + // padding data + butil::IOBuf alignedData; + uint64_t bytes = 0; + uint64_t pos = 0; + + switch (reqCtx_->padding.type) { + case RequestContext::Padding::Left: + alignedCtx_->readData_.append_to( + &alignedData, reqCtx_->offset_ - alignedCtx_->offset_); + reqCtx_->writeData_.append_to(&alignedData, reqCtx_->rawlength_); + + reqCtx_->offset_ = alignedCtx_->offset_; + reqCtx_->rawlength_ = alignedData.size(); + break; + case RequestContext::Padding::Right: + reqCtx_->writeData_.append_to(&alignedData, reqCtx_->rawlength_); + bytes = (alignedCtx_->offset_ + alignedCtx_->rawlength_) - + (reqCtx_->offset_ + reqCtx_->rawlength_); + alignedCtx_->readData_.append_to(&alignedData, bytes, + alignedCtx_->rawlength_ - bytes); + + reqCtx_->rawlength_ = alignedData.size(); + break; + case RequestContext::Padding::ALL: + bytes = reqCtx_->offset_ - alignedCtx_->offset_; + alignedCtx_->readData_.append_to(&alignedData, bytes); + pos += bytes; + reqCtx_->writeData_.append_to(&alignedData, reqCtx_->rawlength_); + pos += reqCtx_->rawlength_; + bytes = alignedCtx_->rawlength_ - bytes; + alignedCtx_->readData_.append_to(&alignedData, bytes, pos); + + reqCtx_->offset_ = alignedCtx_->offset_; + reqCtx_->rawlength_ = alignedData.size(); + break; + default: + break; + } + + // mark original request aligned and swap data + reqCtx_->padding.aligned = true; + reqCtx_->writeData_.swap(alignedData); + + // reschedule original requst + int ret = scheduler_->ReSchedule(reqCtx_); + if (ret != 0) { + LOG(ERROR) << "Reschedule original request failed"; + reqCtx_->done_->SetFailed(-1); + reqCtx_->done_->Run(); + } +} + +void PaddingReadClosure::GenAlignedRequest() { + alignedCtx_ = new RequestContext(); + + alignedCtx_->optype_ = OpType::READ; // set to read + alignedCtx_->padding.aligned = true; // set to aligned + + // copy request context + alignedCtx_->idinfo_ = reqCtx_->idinfo_; + alignedCtx_->offset_ = reqCtx_->padding.offset; + alignedCtx_->rawlength_ = reqCtx_->padding.length; + alignedCtx_->done_ = this; + alignedCtx_->seq_ = reqCtx_->seq_; + alignedCtx_->appliedindex_ = reqCtx_->appliedindex_; + alignedCtx_->chunksize_ = reqCtx_->chunksize_; + alignedCtx_->location_ = reqCtx_->location_; + alignedCtx_->sourceInfo_ = reqCtx_->sourceInfo_; + alignedCtx_->correctedSeq_ = reqCtx_->correctedSeq_; + alignedCtx_->id_ = RequestContext::GetNextRequestContextId(); +} + +void PaddingReadClosure::HandleError(int errCode) { + brpc::ClosureGuard doneGuard(reqCtx_->done_); + reqCtx_->done_->SetFailed(errCode); + + LOG(ERROR) << "Padding read request failed, request: " << *alignedCtx_ + << ", error: " << errCode; +} + } // namespace client } // namespace curve diff --git a/src/client/request_closure.h b/src/client/request_closure.h index a385d5c98a..07de4e2558 100644 --- a/src/client/request_closure.h +++ b/src/client/request_closure.h @@ -28,16 +28,15 @@ #include "include/curve_compiler_specific.h" #include "src/client/client_common.h" -#include "src/client/client_metric.h" -#include "src/client/inflight_controller.h" -#include "src/common/concurrent/concurrent.h" namespace curve { namespace client { +class FileMetric; class IOTracker; -class RequestContext; class IOManager; +class RequestContext; +class RequestScheduler; class CURVE_CACHELINE_ALIGNMENT RequestClosure : public ::google::protobuf::Closure { @@ -149,6 +148,13 @@ class CURVE_CACHELINE_ALIGNMENT RequestClosure return suspendRPC_; } + protected: + // request context of this closure + RequestContext* reqCtx_ = nullptr; + + // iotracker of current request context + IOTracker* tracker_ = nullptr; + private: // suspend io标志 bool suspendRPC_ = false; @@ -159,12 +165,6 @@ class CURVE_CACHELINE_ALIGNMENT RequestClosure // 当前request的错误码 int errcode_ = -1; - // 当前request的tracker信息 - IOTracker* tracker_ = nullptr; - - // closure的request信息 - RequestContext* reqCtx_ = nullptr; - // metric信息 FileMetric* metric_ = nullptr; @@ -178,6 +178,50 @@ class CURVE_CACHELINE_ALIGNMENT RequestClosure uint64_t nextTimeoutMS_ = 0; }; +// PaddingReadClosure is used to process unaligned request +// it wraps the original unaligned request and forms an aligned request +class PaddingReadClosure : public RequestClosure { + public: + PaddingReadClosure(RequestContext* requestCtx, RequestScheduler* scheduler); + + void Run() override; + + RequestContext* GetReqCtx() override { + return alignedCtx_; + } + + RequestContext* AlignedRequest() const { + return alignedCtx_; + } + + private: + /** + * @brief Called in Run(), handle original read request + */ + void HandleRead(); + + /** + * @brief Called in Run(), handle original write request + */ + void HandleWrite(); + + /** + * @brief Called when error occurs + */ + void HandleError(int errCode); + + /** + * @brief Generate an aligned request based on RequestClosure::reqCtx_ + */ + void GenAlignedRequest(); + + private: + // aligned request context based on RequestClosure::reqCtx_ + RequestContext* alignedCtx_; + + RequestScheduler* scheduler_; +}; + } // namespace client } // namespace curve diff --git a/src/client/request_context.h b/src/client/request_context.h index 1bd03aed05..ee000916e3 100644 --- a/src/client/request_context.h +++ b/src/client/request_context.h @@ -59,6 +59,20 @@ inline std::ostream& operator<<(std::ostream& os, } struct CURVE_CACHELINE_ALIGNMENT RequestContext { + struct Padding { + enum PaddingType { + None, + Left, + Right, + ALL + }; + + bool aligned = true; + PaddingType type = None; + off_t offset = 0; + size_t length = 0; + }; + RequestContext() : id_(GetNextRequestContextId()) {} ~RequestContext() = default; @@ -113,6 +127,8 @@ struct CURVE_CACHELINE_ALIGNMENT RequestContext { // 当前request context id uint64_t id_ = 0; + Padding padding; + static RequestContext* NewInitedRequestContext() { RequestContext* ctx = new (std::nothrow) RequestContext(); if (ctx && ctx->Init()) { @@ -124,12 +140,12 @@ struct CURVE_CACHELINE_ALIGNMENT RequestContext { } } - private: - static std::atomic requestId; - static uint64_t GetNextRequestContextId() { return requestId.fetch_add(1, std::memory_order_relaxed); } + + private: + static std::atomic requestId; }; inline std::ostream& operator<<(std::ostream& os, diff --git a/src/client/request_scheduler.cpp b/src/client/request_scheduler.cpp index ee60923d9b..a0e7e585ef 100644 --- a/src/client/request_scheduler.cpp +++ b/src/client/request_scheduler.cpp @@ -148,7 +148,11 @@ void RequestScheduler::Process() { BBQItem item = queue_.TakeFront(); if (!item.IsStop()) { RequestContext* req = item.Item(); - ProcessOne(req); + if (req->padding.aligned) { + ProcessAligned(req); + } else { + ProcessUnaligned(req); + } } else { /** * 一旦遇到stop item，所有线程都可以退出，因为此时 @@ -159,7 +163,7 @@ void RequestScheduler::Process() { } } -void RequestScheduler::ProcessOne(RequestContext* ctx) { +void RequestScheduler::ProcessAligned(RequestContext* ctx) { brpc::ClosureGuard guard(ctx->done_); switch (ctx->optype_) { @@ -202,5 +206,23 @@ void RequestScheduler::ProcessOne(RequestContext* ctx) { } } +void RequestScheduler::ProcessUnaligned(RequestContext* ctx) { + brpc::ClosureGuard doneGuard(ctx->done_); + if (ctx->optype_ != OpType::READ && ctx->optype_ != OpType::WRITE) { + ctx->done_->SetFailed(-1); + LOG(ERROR) << "unexpected op type: " << static_cast(ctx->optype_); + return; + } + + PaddingReadClosure* p = new PaddingReadClosure(ctx, this); + int ret = ReSchedule(p->AlignedRequest()); + if (ret == 0) { + doneGuard.release(); + } else { + ctx->done_->SetFailed(-1); + LOG(ERROR) << "ReSchedule failed"; + } +} + } // namespace client } // namespace curve diff --git a/src/client/request_scheduler.h b/src/client/request_scheduler.h index c4ad511480..fa87a8fa12 100644 --- a/src/client/request_scheduler.h +++ b/src/client/request_scheduler.h @@ -138,7 +138,9 @@ class RequestScheduler : public Uncopyable { */ void Process(); - void ProcessOne(RequestContext* ctx); + void ProcessAligned(RequestContext* ctx); + + void ProcessUnaligned(RequestContext* ctx); void WaitValidSession() { // lease续约失败的时候需要阻塞IO直到续约成功 diff --git a/src/client/splitor.cpp b/src/client/splitor.cpp index 2d7764a9f8..23f9f228cc 100644 --- a/src/client/splitor.cpp +++ b/src/client/splitor.cpp @@ -32,7 +32,7 @@ #include "src/client/mds_client.h" #include "src/client/metacache_struct.h" #include "src/client/request_closure.h" -#include "src/common/location_operator.h" +#include "src/common/fast_align.h" namespace curve { namespace client { @@ -88,8 +88,15 @@ int Splitor::SingleChunkIO2ChunkRequests( uint64_t currentOffset = offset; uint64_t leftLength = length; while (leftLength > 0) { + RequestContext::Padding padding; + padding.aligned = true; // TODO(wuhanqing): add test case for normal file // NOLINT uint64_t requestLength = std::min(leftLength, maxSplitSizeBytes); + if (metaCache->IsCloneFile()) { + requestLength = ProcessUnalignedRequests(currentOffset, + requestLength, &padding); + } + RequestContext* newreqNode = RequestContext::NewInitedRequestContext(); if (newreqNode == nullptr) { return -1; @@ -109,6 +116,7 @@ int Splitor::SingleChunkIO2ChunkRequests( newreqNode->rawlength_ = requestLength; newreqNode->optype_ = iotracker->Optype(); newreqNode->idinfo_ = idinfo; + newreqNode->padding = padding; newreqNode->done_->SetIOTracker(iotracker); targetlist->push_back(newreqNode); @@ -393,6 +401,54 @@ bool Splitor::MarkDiscardBitmap(IOTracker* iotracker, FileSegment* fileSegment, return true; } +uint64_t Splitor::ProcessUnalignedRequests(const off_t currentOffset, + const uint64_t requestLength, + RequestContext::Padding* padding) { + uint64_t length = requestLength; + uint64_t currentEndOff = currentOffset + requestLength; + uint64_t alignedStartOffset = + common::align_up(currentOffset, iosplitopt_.alignment.cloneVolume); + uint64_t alignedEndOffset = + common::align_down(currentEndOff, iosplitopt_.alignment.cloneVolume); + + if (currentOffset == alignedStartOffset && + currentEndOff == alignedEndOffset) { + padding->aligned = true; + } else { + if (currentOffset == alignedStartOffset) { + padding->aligned = false; + padding->type = RequestContext::Padding::Right; + padding->offset = alignedEndOffset; + padding->length = iosplitopt_.alignment.cloneVolume; + } else if (currentEndOff == alignedStartOffset) { + padding->aligned = false; + padding->type = RequestContext::Padding::Left; + padding->offset = common::align_down( + currentOffset, iosplitopt_.alignment.cloneVolume); + padding->length = iosplitopt_.alignment.cloneVolume; + } else { + if (alignedEndOffset > alignedStartOffset) { + length = alignedEndOffset - currentOffset; + padding->aligned = false; + padding->type = RequestContext::Padding::Left; + padding->offset = common::align_down( + currentOffset, iosplitopt_.alignment.cloneVolume); + padding->length = iosplitopt_.alignment.cloneVolume; + } else { + padding->aligned = false; + padding->type = RequestContext::Padding::ALL; + padding->offset = common::align_down( + currentOffset, iosplitopt_.alignment.cloneVolume); + padding->length = (alignedStartOffset == alignedEndOffset) + ? 2 * iosplitopt_.alignment.cloneVolume + : iosplitopt_.alignment.cloneVolume; + } + } + } + + return length; +} + RequestSourceInfo Splitor::CalcRequestSourceInfo(IOTracker* ioTracker, MetaCache* metaCache, ChunkIndex chunkIdx) { diff --git a/src/client/splitor.h b/src/client/splitor.h index 4860d95912..d5a274d9cd 100644 --- a/src/client/splitor.h +++ b/src/client/splitor.h @@ -24,19 +24,21 @@ #include -#include #include +#include -#include "src/client/metacache.h" -#include "src/client/io_tracker.h" +#include "src/client/client_common.h" #include "src/client/config_info.h" #include "src/client/request_context.h" -#include "src/client/client_common.h" -#include "src/client/client_config.h" namespace curve { namespace client { +class MetaCache; +class MDSClient; +class IOTracker; +class FileSegment; + class Splitor { public: static void Init(const IOSplitOption& ioSplitOpt); @@ -138,6 +140,10 @@ class Splitor { uint64_t offset, uint64_t len); + static uint64_t ProcessUnalignedRequests(const off_t currentOffset, + const uint64_t requestLength, + RequestContext::Padding* padding); + private: // IO拆分模块所使用的配置信息 static IOSplitOption iosplitopt_; diff --git a/src/common/configuration.cpp b/src/common/configuration.cpp index 3536869ec7..542574f4b5 100644 --- a/src/common/configuration.cpp +++ b/src/common/configuration.cpp @@ -180,6 +180,11 @@ void Configuration::SetUInt64Value( SetValue(key, std::to_string(value)); } +void Configuration::SetUInt32Value(const std::string &key, + const uint32_t value) { + SetValue(key, std::to_string(value)); +} + bool Configuration::GetInt64Value(const std::string& key, int64_t* out) { std::string res; if (GetValue(key, &res)) { diff --git a/src/common/configuration.h b/src/common/configuration.h index 9a43b10a91..778db35bf9 100644 --- a/src/common/configuration.h +++ b/src/common/configuration.h @@ -82,6 +82,7 @@ class Configuration { bool GetUInt32Value(const std::string &key, uint32_t *out); bool GetUInt64Value(const std::string &key, uint64_t *out); void SetIntValue(const std::string &key, const int value); + void SetUInt32Value(const std::string &key, const uint32_t value); void SetUInt64Value(const std::string &key, const uint64_t value); bool GetInt64Value(const std::string& key, int64_t* out); diff --git a/src/common/fast_align.h b/src/common/fast_align.h new file mode 100644 index 0000000000..5906a3fad6 --- /dev/null +++ b/src/common/fast_align.h @@ -0,0 +1,61 @@ +/* +Copyright 2015 Glen Joseph Fernandes +(glenjofe@gmail.com) +Distributed under the Boost Software License, Version 1.0. +(http://www.boost.org/LICENSE_1_0.txt) +*/ + +#ifndef SRC_COMMON_FAST_ALIGN_H_ +#define SRC_COMMON_FAST_ALIGN_H_ + +#include + +namespace curve { +namespace common { + +namespace detail { + +template +struct not_pointer { + typedef U type; +}; + +template +struct not_pointer {}; + +} // namespace detail + +template +constexpr inline typename detail::not_pointer::type align_down( + T value, std::size_t alignment) noexcept { + return T(value & ~T(alignment - 1)); +} + +inline void* align_down(void* ptr, std::size_t alignment) noexcept { + return (void*)(align_down((std::size_t)ptr, alignment)); // NOLINT +} + +template +constexpr inline typename detail::not_pointer::type align_up( + T value, std::size_t alignment) noexcept { + return T((value + (T(alignment) - 1)) & ~T(alignment - 1)); +} + +inline void* align_up(void* ptr, std::size_t alignment) noexcept { + return (void*)(align_up((std::size_t)ptr, alignment)); // NOLINT +} + +template +constexpr inline typename detail::not_pointer::type is_aligned( + T value, std::size_t alignment) noexcept { + return (value & (T(alignment) - 1)) == 0; +} + +inline bool is_aligned(const void* ptr, std::size_t alignment) noexcept { + return is_aligned((std::size_t)ptr, alignment); // NOLINT +} + +} // namespace common +} // namespace curve + +#endif // SRC_COMMON_FAST_ALIGN_H_ diff --git a/test/chunkserver/chunk_service_test.cpp b/test/chunkserver/chunk_service_test.cpp index b00107fb02..6f6a970448 100644 --- a/test/chunkserver/chunk_service_test.cpp +++ b/test/chunkserver/chunk_service_test.cpp @@ -44,6 +44,8 @@ namespace chunkserver { using curve::common::UUIDGenerator; +const uint32_t kOpRequestAlignSize = 4096; + class ChunkserverTest : public testing::Test { protected: virtual void SetUp() { diff --git a/test/chunkserver/chunk_service_test2.cpp b/test/chunkserver/chunk_service_test2.cpp index 7fcf4fbd07..7adca675da 100644 --- a/test/chunkserver/chunk_service_test2.cpp +++ b/test/chunkserver/chunk_service_test2.cpp @@ -44,6 +44,8 @@ namespace chunkserver { using curve::common::UUIDGenerator; +const uint32_t kOpRequestAlignSize = 4096; + class ChunkService2Test : public testing::Test { protected: virtual void SetUp() { diff --git a/test/chunkserver/chunkserver_snapshot_test.cpp b/test/chunkserver/chunkserver_snapshot_test.cpp index 43e29743c4..3314cadc51 100644 --- a/test/chunkserver/chunkserver_snapshot_test.cpp +++ b/test/chunkserver/chunkserver_snapshot_test.cpp @@ -41,6 +41,8 @@ using curve::fs::LocalFileSystem; using curve::fs::LocalFsFactory; using curve::fs::FileSystemType; +const uint32_t kOpRequestAlignSize = 4096; + class ChunkServerSnapshotTest : public testing::Test { protected: virtual void SetUp() { diff --git a/test/chunkserver/datastore/datastore_mock_unittest.cpp b/test/chunkserver/datastore/datastore_mock_unittest.cpp index cda70b63fa..232fd3257d 100644 --- a/test/chunkserver/datastore/datastore_mock_unittest.cpp +++ b/test/chunkserver/datastore/datastore_mock_unittest.cpp @@ -52,6 +52,7 @@ using ::testing::ReturnArg; using ::testing::ElementsAre; using ::testing::SetArgPointee; using ::testing::SetArrayArgument; +using ::testing::AtLeast; using std::shared_ptr; using std::make_shared; @@ -4182,5 +4183,358 @@ TEST_F(CSDataStore_test, GetStatusTest) { .Times(1); } +TEST_F(CSDataStore_test, CloneChunkUnAlignedTest) { + // initialize + FakeEnv(); + EXPECT_TRUE(dataStore->Initialize()); + + ChunkID id = 3; + SequenceNum sn = 2; + SequenceNum correctedSn = 3; + off_t offset = 0; + size_t length = PAGE_SIZE; + char buf[length]; // NOLINT + memset(buf, 0, sizeof(buf)); + CSChunkInfo info; + // 创建 clone chunk + { + char chunk3MetaPage[PAGE_SIZE]; + memset(chunk3MetaPage, 0, sizeof(chunk3MetaPage)); + shared_ptr bitmap = + make_shared(CHUNK_SIZE / PAGE_SIZE); + FakeEncodeChunk(chunk3MetaPage, correctedSn, sn, bitmap, location); + // create new chunk and open it + string chunk3Path = string(baseDir) + "/" + + FileNameOperator::GenerateChunkFileName(id); + // expect call chunkfile pool GetFile + EXPECT_CALL(*lfs_, FileExists(chunk3Path)) + .WillOnce(Return(false)); + EXPECT_CALL(*fpool_, GetFileImpl(chunk3Path, NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*lfs_, Open(chunk3Path, _)) + .Times(1) + .WillOnce(Return(4)); + // will read metapage + EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE)) + .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage, + chunk3MetaPage + PAGE_SIZE), + Return(PAGE_SIZE))); + EXPECT_EQ(CSErrorCode::Success, + dataStore->CreateCloneChunk(id, + sn, + correctedSn, + CHUNK_SIZE, + location)); + ASSERT_EQ(CSErrorCode::Success, dataStore->GetChunkInfo(id, &info)); + ASSERT_EQ(2, info.curSn); + ASSERT_EQ(3, info.correctedSn); + ASSERT_EQ(0, info.snapSn); + ASSERT_EQ(true, info.isClone); + ASSERT_EQ(0, info.bitmap->NextClearBit(0)); + ASSERT_EQ(Bitmap::NO_POS, info.bitmap->NextSetBit(0)); + } + + // read/write/paste offset are not aligned to pagesize + { + EXPECT_CALL(*lfs_, Write(4, Matcher(_), _, _)).Times(0); + EXPECT_CALL(*lfs_, Read(4, _, _, _)).Times(0); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 1, length, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 512, length, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 1024, length, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 2048, length, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 3072, length, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 4095, length, nullptr)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 1, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 512, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 1024, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 2048, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 3072, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 4095, length)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 1, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 512, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 1024, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 2048, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 3072, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 4095, length)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 1, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 512, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 1024, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 2048, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 3072, length)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 4095, length)); + } + + // read/write/paste length are not aligned to pagesize + { + EXPECT_CALL(*lfs_, Write(4, Matcher(_), _, _)).Times(0); + EXPECT_CALL(*lfs_, Read(4, _, _, _)).Times(0); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 1, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 512, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 1024, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 2048, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 3072, nullptr)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 4095, nullptr)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 1)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 512)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 1024)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 2048)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 3072)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 4095)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 1)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 512)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 1024)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 2048)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 3072)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->PasteChunk(id, buf, 0, 4095)); + + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 1)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 512)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 1024)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 2048)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 3072)); + ASSERT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadSnapshotChunk(id, sn, buf, 0, 4095)); + } + + EXPECT_CALL(*lfs_, Close(1)) + .Times(1); + EXPECT_CALL(*lfs_, Close(2)) + .Times(1); + EXPECT_CALL(*lfs_, Close(3)) + .Times(1); + EXPECT_CALL(*lfs_, Close(4)) + .Times(1); +} + +TEST_F(CSDataStore_test, CloneChunkAlignedTest) { + // initialize + FakeEnv(); + EXPECT_TRUE(dataStore->Initialize()); + + ChunkID id = 3; + SequenceNum sn = 2; + SequenceNum correctedSn = 3; + off_t offset = 0; + size_t length = PAGE_SIZE; + char buf[length]; // NOLINT + memset(buf, 0, sizeof(buf)); + CSChunkInfo info; + // 创建 clone chunk + { + char chunk3MetaPage[PAGE_SIZE]; + memset(chunk3MetaPage, 0, sizeof(chunk3MetaPage)); + shared_ptr bitmap = + make_shared(CHUNK_SIZE / PAGE_SIZE); + FakeEncodeChunk(chunk3MetaPage, correctedSn, sn, bitmap, location); + // create new chunk and open it + string chunk3Path = string(baseDir) + "/" + + FileNameOperator::GenerateChunkFileName(id); + // expect call chunkfile pool GetFile + EXPECT_CALL(*lfs_, FileExists(chunk3Path)) + .WillOnce(Return(false)); + EXPECT_CALL(*fpool_, GetFileImpl(chunk3Path, NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*lfs_, Open(chunk3Path, _)) + .Times(1) + .WillOnce(Return(4)); + // will read metapage + EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE)) + .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage, + chunk3MetaPage + PAGE_SIZE), + Return(PAGE_SIZE))); + EXPECT_EQ(CSErrorCode::Success, + dataStore->CreateCloneChunk(id, + sn, + correctedSn, + CHUNK_SIZE, + location)); + ASSERT_EQ(CSErrorCode::Success, dataStore->GetChunkInfo(id, &info)); + ASSERT_EQ(2, info.curSn); + ASSERT_EQ(3, info.correctedSn); + ASSERT_EQ(0, info.snapSn); + ASSERT_EQ(true, info.isClone); + ASSERT_EQ(0, info.bitmap->NextClearBit(0)); + ASSERT_EQ(Bitmap::NO_POS, info.bitmap->NextSetBit(0)); + } + + sn = 3; + // write offset/length both aligned + { + EXPECT_CALL(*lfs_, Write(4, Matcher(_), 0, PAGE_SIZE)) + .Times(2); + EXPECT_CALL(*lfs_, Write(4, Matcher(_), + PAGE_SIZE + offset, length)) + .Times(1); + + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, offset, length, nullptr)); + } + + // read offset/length both aligned + { + EXPECT_CALL(*lfs_, Read(4, NotNull(), offset + PAGE_SIZE, length)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, offset, length)); + } + + // paste offset/length both aligned + { + EXPECT_EQ(CSErrorCode::Success, + dataStore->PasteChunk(id, buf, offset, length)); + } + + // readspecificchunk offset/length both aligned + { + EXPECT_CALL(*lfs_, Read(4, NotNull(), offset + PAGE_SIZE, length)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadSnapshotChunk(id, sn, buf, offset, length)); + } + + EXPECT_CALL(*lfs_, Close(1)) + .Times(1); + EXPECT_CALL(*lfs_, Close(2)) + .Times(1); + EXPECT_CALL(*lfs_, Close(3)) + .Times(1); + EXPECT_CALL(*lfs_, Close(4)) + .Times(1); +} + +TEST_F(CSDataStore_test, NormalChunkAlignmentTest) { + // initialize + FakeEnv(); + EXPECT_TRUE(dataStore->Initialize()); + + ChunkID id = 2; + SequenceNum sn = 2; + off_t offset = 0; + size_t length = 512; + char buf[length]; // NOLINT + memset(buf, 0, sizeof(buf)); + + // write unaligned test + { + EXPECT_CALL(*lfs_, Write(0, Matcher(_), _, _)) + .Times(0); + + EXPECT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 0, 511, nullptr)); + EXPECT_EQ(CSErrorCode::InvalidArgError, + dataStore->WriteChunk(id, sn, buf, 1, 512, nullptr)); + } + + // read unaligned test + { + EXPECT_CALL(*lfs_, Read(0, NotNull(), _, _)) + .Times(0); + + EXPECT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 0, 511)); + EXPECT_EQ(CSErrorCode::InvalidArgError, + dataStore->ReadChunk(id, sn, buf, 1, 512)); + } + + // write aligned test + { + EXPECT_CALL(*lfs_, Write(3, Matcher(_), _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(0)); + + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, 0, 512, nullptr)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, 0, 1024, nullptr)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, 0, 2048, nullptr)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, 0, 3072, nullptr)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->WriteChunk(id, sn, buf, 0, 4096, nullptr)); + } + + // read aligned test + { + EXPECT_CALL(*lfs_, Read(3, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(0)); + + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, 0, 512)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, 0, 1024)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, 0, 2048)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, 0, 3072)); + EXPECT_EQ(CSErrorCode::Success, + dataStore->ReadChunk(id, sn, buf, 0, 4096)); + } + + EXPECT_CALL(*lfs_, Close(1)) + .Times(1); + EXPECT_CALL(*lfs_, Close(2)) + .Times(1); + EXPECT_CALL(*lfs_, Close(3)) + .Times(1); +} + } // namespace chunkserver } // namespace curve diff --git a/test/chunkserver/raftsnapshot/raftsnapshot_chunkfilepool_integration.cpp b/test/chunkserver/raftsnapshot/raftsnapshot_chunkfilepool_integration.cpp index 6c5b8c0ee9..cb3b4fab7d 100644 --- a/test/chunkserver/raftsnapshot/raftsnapshot_chunkfilepool_integration.cpp +++ b/test/chunkserver/raftsnapshot/raftsnapshot_chunkfilepool_integration.cpp @@ -39,6 +39,8 @@ using curve::fs::LocalFileSystem; using curve::fs::LocalFsFactory; using curve::fs::FileSystemType; +const uint32_t kOpRequestAlignSize = 4096; + class RaftSnapFilePoolTest : public testing::Test { protected: virtual void SetUp() { diff --git a/test/client/BUILD b/test/client/BUILD index 9e42f9d809..20e0ffccf2 100644 --- a/test/client/BUILD +++ b/test/client/BUILD @@ -51,7 +51,8 @@ cc_test( "lease_executor_test.cpp", "request_sender_test.cpp", "mds_client_test.cpp", - "client_mdsclient_metacache_unittest.cpp" + "client_mdsclient_metacache_unittest.cpp", + "splitor_test.cpp" ] ), copts = COPTS, @@ -362,3 +363,18 @@ cc_test( "//test/client/mock:client_mock_lib", ] ) + +cc_test( + name = "client_splitor_test", + srcs = [ + "splitor_test.cpp" + ], + copts = COPTS, + defines = ["UNIT_TEST", "FIU_ENABLE"], + linkopts = ["-lfiu"], + deps = [ + "//include/client:include_client", + "//src/client:curve_client", + "@com_google_googletest//:gtest", + ] +) diff --git a/test/client/copyset_client_test.cpp b/test/client/copyset_client_test.cpp index 307c3070c2..eca25765a6 100644 --- a/test/client/copyset_client_test.cpp +++ b/test/client/copyset_client_test.cpp @@ -647,6 +647,7 @@ TEST_F(CopysetClientTest, write_error_test) { ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, reqDone->GetErrorCode()); } + /* 不是 leader，返回正确的 leader */ { RequestContext *reqCtx = new FakeRequestContext(); @@ -687,6 +688,7 @@ TEST_F(CopysetClientTest, write_error_test) { ASSERT_EQ(1, fm.writeRPC.redirectQps.count.get_value()); } + /* 不是 leader，没有返回 leader，刷新 meta cache 成功 */ { RequestContext *reqCtx = new FakeRequestContext(); @@ -724,6 +726,7 @@ TEST_F(CopysetClientTest, write_error_test) { ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, reqDone->GetErrorCode()); } + /* 不是 leader，没有返回 leader，刷新 meta cache 失败 */ { RequestContext *reqCtx = new FakeRequestContext(); @@ -764,6 +767,7 @@ TEST_F(CopysetClientTest, write_error_test) { ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, reqDone->GetErrorCode()); } + /* 不是 leader，但返回的是错误 leader */ { RequestContext *reqCtx = new FakeRequestContext(); @@ -808,6 +812,7 @@ TEST_F(CopysetClientTest, write_error_test) { reqDone->GetErrorCode()); ASSERT_EQ(3, fm.writeRPC.redirectQps.count.get_value()); } + /* copyset 不存在，更新 leader 依然失败 */ { RequestContext *reqCtx = new FakeRequestContext(); @@ -877,6 +882,7 @@ TEST_F(CopysetClientTest, write_error_test) { ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, reqDone->GetErrorCode()); } + scheduler.Fini(); } diff --git a/test/client/iotracker_align_test.cpp b/test/client/iotracker_align_test.cpp new file mode 100644 index 0000000000..d67ddbd574 --- /dev/null +++ b/test/client/iotracker_align_test.cpp @@ -0,0 +1,545 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tue Jun 22 19:26:16 CST 2021 + * Author: wuhanqing + */ + +#include +#include +#include +#include + +#include "src/client/client_common.h" +#include "src/client/io_tracker.h" +#include "src/client/splitor.h" +#include "test/client/mock/mock_chunk_service.h" +#include "test/client/mock/mock_mdsclient.h" +#include "test/client/mock/mock_meta_cache.h" +#include "test/client/mock/mock_request_scheduler.h" + +namespace curve { +namespace client { + +using ::testing::AllOf; +using ::testing::AtLeast; +using ::testing::Ge; +using ::testing::Le; +using ::testing::Matcher; +using ::testing::Return; +using ::testing::DoAll; +using ::testing::Invoke; +using ::testing::SetArgPointee; +using ::testing::SaveArgPointee; + +struct SaveReadRequst { + explicit SaveReadRequst(curve::chunkserver::ChunkRequest* r) : r_(r) {} + + void operator()(::google::protobuf::RpcController* cntl_base, + const chunkserver::ChunkRequest* request, + chunkserver::ChunkResponse* response, + google::protobuf::Closure* done) { + LOG(INFO) << "SaveReadRequest"; + LOG(INFO) << request->DebugString(); + + r_->CopyFrom(*request); + + brpc::Controller* cntl = static_cast(cntl_base); + cntl->response_attachment().resize(request->size(), 'x'); + response->set_status( + chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + done->Run(); + } + + curve::chunkserver::ChunkRequest* r_; +}; + +struct SaveWriteData { + explicit SaveWriteData(butil::IOBuf* data) : data_(data) {} + + void operator()(::google::protobuf::RpcController* cntl_base, + const chunkserver::ChunkRequest* request, + chunkserver::ChunkResponse* response, + google::protobuf::Closure* done) { + LOG(INFO) << "SaveWriteData"; + + brpc::Controller* cntl = static_cast(cntl_base); + *data_ = cntl->request_attachment(); + + response->set_status( + chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + done->Run(); + } + + butil::IOBuf* data_; +}; + +class IOTrackerAlignmentTest : public ::testing::Test { + void SetUp() override { + Splitor::Init(splitOpt_); + + mockMDSClient_.reset(new MockMDSClient()); + mockMetaCache_.reset(new MockMetaCache()); + + MetaCacheOption metaCacheOpt; + mockMetaCache_->Init(metaCacheOpt, mockMDSClient_.get()); + + fileInfo_.fullPathName = "/IOTrackerAlignmentTest"; + fileInfo_.length = 100 * GiB; + fileInfo_.segmentsize = 1 * GiB; + fileInfo_.chunksize = 16 * MiB; + fileInfo_.filestatus = FileStatus::CloneMetaInstalled; + + mockMetaCache_->UpdateFileInfo(fileInfo_); + + ASSERT_EQ(0, scheduler_.Init({}, mockMetaCache_.get())); + ASSERT_EQ(0, scheduler_.Run()); + + mockChunkService_.reset(new chunkserver::MockChunkService()); + + ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:21001", &csEp_)); + ASSERT_EQ(0, server_.AddService(mockChunkService_.get(), + brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server_.Start(csEp_, nullptr)); + } + + void TearDown() override { + scheduler_.Fini(); + + server_.Stop(0); + server_.Join(); + } + + protected: + FInfo fileInfo_; + std::unique_ptr metric; + std::unique_ptr mockMetaCache_; + std::unique_ptr mockMDSClient_; + std::unique_ptr mockChunkService_; + + RequestScheduler scheduler_; + + IOSplitOption splitOpt_; + brpc::Server server_; + ChunkServerID csId_; + butil::EndPoint csEp_; + FileMetric metric_{"/IOTrackerAlignmentTest"}; +}; + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite1) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 0; + uint64_t length = 2048; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(0, readRequest.offset()); + EXPECT_EQ(4096, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(2048, 'a'); + expected.resize(4096, 'x'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite2) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 0; + uint64_t length = 64 * KiB - 1024; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(60 * KiB, readRequest.offset()); + EXPECT_EQ(4096, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(length, 'a'); + expected.resize(length + 1024, 'x'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite3) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 2048; + uint64_t length = 2048; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(0, readRequest.offset()); + EXPECT_EQ(4096, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(2048, 'x'); + expected.resize(4096, 'a'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite4) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 1024; + uint64_t length = 64 * KiB - 1024; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(0, readRequest.offset()); + EXPECT_EQ(4096, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(1024, 'x'); + expected.resize(length + 1024, 'a'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite5) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 1024; + uint64_t length = 2048; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(0, readRequest.offset()); + EXPECT_EQ(4096, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(1024, 'x'); + expected.resize(3072, 'a'); + expected.resize(4096, 'x'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite6) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + curve::chunkserver::ChunkRequest readRequest; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .WillOnce(Invoke(SaveReadRequst(&readRequest))); + + butil::IOBuf writeData; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .WillOnce(Invoke(SaveWriteData(&writeData))); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 3072; + uint64_t length = 2048; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(0, readRequest.offset()); + EXPECT_EQ(8192, readRequest.size()); + EXPECT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequest.optype()); + + butil::IOBuf expected; + expected.resize(3072, 'x'); + expected.resize(3072 + 2048, 'a'); + expected.resize(8192, 'x'); + EXPECT_EQ(writeData, expected); +} + +TEST_F(IOTrackerAlignmentTest, TestUnalignedWrite7) { + EXPECT_CALL(*mockMetaCache_, GetChunkInfoByIndex(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke([](ChunkIndex idx, ChunkIDInfo_t* info) { + info->chunkExist = true; + info->lpid_ = 1; + info->cpid_ = 2; + info->cid_ = 3; + + return MetaCacheErrorType::OK; + })); + + EXPECT_CALL(*mockMetaCache_, GetLeader(_, _, _, _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<2>(csId_), SetArgPointee<3>(csEp_), Return(0))); + + std::vector readRequests; + std::mutex mtx; + EXPECT_CALL(*mockChunkService_, ReadChunk(_, _, _, _)) + .Times(2) + .WillRepeatedly(Invoke([&readRequests, &mtx]( + ::google::protobuf::RpcController* cntl_base, + const chunkserver::ChunkRequest* request, + chunkserver::ChunkResponse* response, + google::protobuf::Closure* done) { + { + std::lock_guard lock(mtx); + readRequests.push_back(*request); + } + + brpc::Controller* cntl = static_cast(cntl_base); + cntl->response_attachment().resize(request->size(), 'x'); + response->set_status( + chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + done->Run(); + })); + + std::vector writeDatas; + std::mutex mtx2; + EXPECT_CALL(*mockChunkService_, WriteChunk(_, _, _, _)) + .Times(2) + .WillRepeatedly(Invoke( + [&writeDatas, &mtx2](::google::protobuf::RpcController* cntl_base, + const chunkserver::ChunkRequest* request, + chunkserver::ChunkResponse* response, + google::protobuf::Closure* done) { + { + brpc::Controller* cntl = + static_cast(cntl_base); + std::lock_guard lock(mtx2); + writeDatas.push_back(cntl->request_attachment()); + } + + response->set_status( + chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + done->Run(); + })); + + IOTracker tracker(nullptr, mockMetaCache_.get(), &scheduler_, &metric_); + + butil::IOBuf fakeData; + uint64_t offset = 2048; + uint64_t length = 60 * KiB; + fakeData.resize(length, 'a'); + tracker.SetUserDataType(UserDataType::IOBuffer); + tracker.StartWrite(&fakeData, offset, length, mockMDSClient_.get(), + &fileInfo_); + + EXPECT_GE(tracker.Wait(), 0); + + EXPECT_EQ(2, readRequests.size()); + EXPECT_EQ(2, writeDatas.size()); + + if (0 == readRequests[0].offset()) { + ASSERT_EQ(0, readRequests[0].offset()); + ASSERT_EQ(4096, readRequests[0].size()); + ASSERT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequests[0].optype()); + + ASSERT_EQ(60 * KiB, readRequests[1].offset()); + ASSERT_EQ(4096, readRequests[1].size()); + ASSERT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequests[1].optype()); + } else { + ASSERT_EQ(0, readRequests[1].offset()); + ASSERT_EQ(4096, readRequests[1].size()); + ASSERT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequests[1].optype()); + + ASSERT_EQ(60 * KiB, readRequests[0].offset()); + ASSERT_EQ(4096, readRequests[0].size()); + ASSERT_EQ(curve::chunkserver::CHUNK_OP_READ, readRequests[0].optype()); + } + + if (writeDatas[0].size() == 60 *KiB) { + butil::IOBuf expected; + expected.resize(2048, 'x'); + expected.resize(60 * KiB, 'a'); + + ASSERT_EQ(expected, writeDatas[0]); + + expected.clear(); + expected.resize(2048, 'a'); + expected.resize(4096, 'x'); + ASSERT_EQ(expected, writeDatas[1]); + } else { + butil::IOBuf expected; + expected.resize(2048, 'x'); + expected.resize(60 * KiB, 'a'); + + ASSERT_EQ(expected, writeDatas[1]); + + expected.clear(); + expected.resize(2048, 'a'); + expected.resize(4096, 'x'); + ASSERT_EQ(expected, writeDatas[0]); + } + + // butil::IOBuf expected; + // expected.resize(2048, 'x'); + // expected.resize(60 * KiB + 2048, 'a'); + // expected.resize(64 * KiB, 'x'); + // EXPECT_EQ(writeData, expected); +} + +} // namespace client +} // namespace curve diff --git a/test/client/mock/mock_chunk_service.h b/test/client/mock/mock_chunk_service.h new file mode 100644 index 0000000000..e85170d4a9 --- /dev/null +++ b/test/client/mock/mock_chunk_service.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: Wed Jun 23 11:20:28 CST 2021 + * Author: wuhanqing + */ + +#ifndef TEST_CLIENT_MOCK_MOCK_CHUNK_SERVICE_H_ +#define TEST_CLIENT_MOCK_MOCK_CHUNK_SERVICE_H_ + +#include + +#include "proto/chunk.pb.h" + +namespace curve { +namespace chunkserver { + +class MockChunkService : public ChunkService { + public: + MOCK_METHOD4(WriteChunk, + void(::google::protobuf::RpcController *controller, + const ChunkRequest *request, ChunkResponse *response, + ::google::protobuf::Closure *done)); + MOCK_METHOD4(ReadChunk, + void(::google::protobuf::RpcController *controller, + const ChunkRequest *request, ChunkResponse *response, + ::google::protobuf::Closure *done)); +}; + +} // namespace chunkserver +} // namespace curve + +#endif // TEST_CLIENT_MOCK_MOCK_CHUNK_SERVICE_H_ diff --git a/test/client/mock/mock_meta_cache.h b/test/client/mock/mock_meta_cache.h index ca5aaf562d..ac9de4f378 100644 --- a/test/client/mock/mock_meta_cache.h +++ b/test/client/mock/mock_meta_cache.h @@ -67,6 +67,9 @@ class MockMetaCache : public MetaCache { MOCK_METHOD3(UpdateLeader, int(LogicPoolID, CopysetID, const butil::EndPoint &)); + MOCK_METHOD2(GetChunkInfoByIndex, + MetaCacheErrorType(ChunkIndex, ChunkIDInfo *)); + void DelegateToFake() { ON_CALL(*this, GetLeader(_, _, _, _, _, _)) .WillByDefault(Invoke(&fakeMetaCache_, &FakeMetaCache::GetLeader)); diff --git a/test/client/mock/mock_request_context.h b/test/client/mock/mock_request_context.h index 9248ccf32c..2c28e5aa9e 100644 --- a/test/client/mock/mock_request_context.h +++ b/test/client/mock/mock_request_context.h @@ -31,56 +31,27 @@ namespace curve { namespace client { -class FakeRequestContext : public RequestContext { - public: - FakeRequestContext() : RequestContext() {} - virtual ~FakeRequestContext() {} -}; +using FakeRequestContext = RequestContext; class FakeRequestClosure : public RequestClosure { public: - explicit FakeRequestClosure(curve::common::CountDownEvent *cond, - RequestContext *reqctx) - : cond_(cond), - RequestClosure(reqctx) { - reqCtx_ = reqctx; - } - virtual ~FakeRequestClosure() {} + FakeRequestClosure(curve::common::CountDownEvent *cond, + RequestContext *reqctx) + : RequestClosure(reqctx), cond_(cond) {} void Run() override { - if (0 == errcode_) { + if (0 == GetErrorCode()) { LOG(INFO) << "success"; } else { - LOG(INFO) << "errno: " << errcode_; + LOG(INFO) << "errno: " << GetErrorCode(); } if (nullptr != cond_) { cond_->Signal(); } } - void SetFailed(int err) override { - errcode_ = err; - } - - int GetErrorCode() override { - return errcode_; - } - - RequestContext *GetReqCtx() override { - return reqCtx_; - } - - IOTracker* GettIOTracker() { - return tracker_; - } - private: curve::common::CountDownEvent *cond_; - - private: - int errcode_ = -1; - IOTracker *tracker_; - RequestContext *reqCtx_; }; } // namespace client diff --git a/test/client/splitor_test.cpp b/test/client/splitor_test.cpp new file mode 100644 index 0000000000..e198e14123 --- /dev/null +++ b/test/client/splitor_test.cpp @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * File Created: Mon Jun 21 19:41:55 CST 2021 + * Author: wuhanqing + */ + +#include "src/client/splitor.h" + +#include + +#include + +#include "src/client/io_tracker.h" +#include "src/client/metacache.h" + +namespace curve { +namespace client { + +struct TestParams { + FileStatus fileStatus; + OpType opType; + uint64_t offset; + uint64_t length; + int expectedRequests; + int expectedUnAlignedRequests; +}; + +class SplitorAlignmentTest : public ::testing::TestWithParam { + protected: + void SetUp() override { + params_ = GetParam(); + + splitOpt_.alignment.commonVolume = 512; + splitOpt_.alignment.cloneVolume = 4096; + splitOpt_.fileIOSplitMaxSizeKB = 64; + Splitor::Init(splitOpt_); + + chunkIdInfo_.lpid_ = 1; + chunkIdInfo_.cpid_ = 2; + chunkIdInfo_.cid_ = 3; + chunkIdInfo_.chunkExist = true; + + metaCache_.SetLatestFileStatus(params_.fileStatus); + + iotracker_ = new IOTracker(nullptr, nullptr, nullptr, nullptr); + iotracker_->SetOpType(params_.opType); + + if (params_.opType == OpType::WRITE) { + std::string fakeData(params_.length, 'c'); + writeData_.append(fakeData); + } + } + + void TearDown() override { + delete iotracker_; + + for (auto& r : requests_) { + r->UnInit(); + delete r; + } + } + + int UnalignedRequests() const { + int count = 0; + for (auto& r : requests_) { + count += (r->padding.aligned ? 0 : 1); + } + + return count; + } + + protected: + TestParams params_; + IOSplitOption splitOpt_; + ChunkIDInfo chunkIdInfo_; + MetaCache metaCache_; + IOTracker* iotracker_; + butil::IOBuf writeData_; + std::vector requests_; +}; + +TEST_P(SplitorAlignmentTest, Test) { + EXPECT_EQ(0, Splitor::SingleChunkIO2ChunkRequests( + iotracker_, &metaCache_, &requests_, chunkIdInfo_, + params_.opType == OpType::WRITE ? &writeData_ : nullptr, + params_.offset, params_.length, 0)); + + EXPECT_EQ(params_.expectedRequests, requests_.size()); + EXPECT_EQ(params_.expectedUnAlignedRequests, UnalignedRequests()); +} + +INSTANTIATE_TEST_CASE_P( + SplitorTest, SplitorAlignmentTest, + ::testing::Values( + // case 1. start offset is aligned, end offset is not aligned + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 0, + .length = 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 4096, + .length = 4096 + 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 10ull * 1024 * 1024, + .length = 64ul * 1024 - 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 8ull * 1024 * 1024, + .length = 64ul * 1024 + 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 0, + .length = 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 4096, + .length = 4096 + 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 10ull * 1024 * 1024, + .length = 64ul * 1024 - 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 8ull * 1024 * 1024, + .length = 64ul * 1024 + 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 1}, + + // case 2. start offset is not aligned, end offset is aligned + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 2048, + .length = 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 2048, + .length = 4096 + 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 10ull * 1024 * 1024 + 1024, + .length = 64ul * 1024 - 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 8ull * 1024 * 1024 - 1024, + .length = 64ul * 1024 + 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 2048, + .length = 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 2048, + .length = 4096 + 2048, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 10ull * 1024 * 1024 + 1024, + .length = 64ul * 1024 - 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 8ull * 1024 * 1024 - 1024, + .length = 64ul * 1024 + 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 1}, + + // case 3. both start offset and end offset are unaligned + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 2048, + .length = 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 2048, + .length = 4096, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 2 * 1024, + .length = 60ull * 1024 - 4 * 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 2}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::WRITE, + .offset = 8ull * 1024 * 1024 + 2 * 1024, + .length = 128ul * 1024 - 4 * 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 2}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 2048, + .length = 1024, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 2048, + .length = 4096, + .expectedRequests = 1, + .expectedUnAlignedRequests = 1}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 2 * 1024, + .length = 60ull * 1024 - 4 * 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 2}, + TestParams{.fileStatus = FileStatus::CloneMetaInstalled, + .opType = OpType::READ, + .offset = 8ull * 1024 * 1024 + 2 * 1024, + .length = 128ul * 1024 - 4 * 1024, + .expectedRequests = 2, + .expectedUnAlignedRequests = 2})); + +} // namespace client +} // namespace curve + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/common/fast_align_test.cpp b/test/common/fast_align_test.cpp new file mode 100644 index 0000000000..000161efe7 --- /dev/null +++ b/test/common/fast_align_test.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2020 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: Mon Jul 5 19:53:34 CST 2021 + * Author: wuhanqing + */ + +#include "src/common/fast_align.h" + +#include + +namespace curve { +namespace common { + +TEST(FastAlignTest, TestAlignUp) { + ASSERT_EQ(0, align_up(0, 512)); + ASSERT_EQ(512, align_up(1, 512)); + ASSERT_EQ(512, align_up(511, 512)); + ASSERT_EQ(1024, align_up(513, 512)); +} + +TEST(FastAlignTest, TestAlignDown) { + ASSERT_EQ(0, align_down(0, 512)); + ASSERT_EQ(0, align_down(1, 512)); + ASSERT_EQ(0, align_down(511, 512)); + ASSERT_EQ(512, align_down(512, 512)); + ASSERT_EQ(512, align_down(513, 512)); +} + +TEST(FastAlignTest, TestIsAligned) { + ASSERT_TRUE(is_aligned(512, 512)); + ASSERT_TRUE(is_aligned(4096, 512)); + + ASSERT_FALSE(is_aligned(511, 512)); + ASSERT_FALSE(is_aligned(4095, 4096)); +} + +} // namespace common +} // namespace curve diff --git a/test/integration/chunkserver/chunkserver_basic_test.cpp b/test/integration/chunkserver/chunkserver_basic_test.cpp index cbf219b027..18e321b854 100644 --- a/test/integration/chunkserver/chunkserver_basic_test.cpp +++ b/test/integration/chunkserver/chunkserver_basic_test.cpp @@ -70,6 +70,7 @@ static char *chunkServerParams[1][16] = { butil::AtExitManager atExitManager; const int kChunkNum = 10; const ChunkSizeType kChunkSize = 16 * 1024 * 1024; +const uint32_t kOpRequestAlignSize = 4096; const PageSizeType kPageSize = kOpRequestAlignSize; class ChunkServerIoTest : public testing::Test { diff --git a/test/integration/chunkserver/chunkserver_concurrent_test.cpp b/test/integration/chunkserver/chunkserver_concurrent_test.cpp index 501ded79b4..b1407f0878 100644 --- a/test/integration/chunkserver/chunkserver_concurrent_test.cpp +++ b/test/integration/chunkserver/chunkserver_concurrent_test.cpp @@ -88,6 +88,7 @@ static char *chunkConcurrencyParams2[1][16] = { butil::AtExitManager atExitManager; const int kChunkNum = 10; const ChunkSizeType kChunkSize = 16 * 1024 * 1024; +const uint32_t kOpRequestAlignSize = 4096; const PageSizeType kPageSize = kOpRequestAlignSize; // chunk不从FilePool获取的chunkserver并发测试 diff --git a/test/integration/common/chunkservice_op.cpp b/test/integration/common/chunkservice_op.cpp index 28a451d928..db4710edfa 100644 --- a/test/integration/common/chunkservice_op.cpp +++ b/test/integration/common/chunkservice_op.cpp @@ -26,6 +26,7 @@ namespace curve { namespace chunkserver { +const uint32_t kOpRequestAlignSize = 4096; const PageSizeType kPageSize = kOpRequestAlignSize; int ChunkServiceOp::WriteChunk(struct ChunkServiceOpConf *opConf, diff --git a/test/integration/raft/raft_config_change_test.cpp b/test/integration/raft/raft_config_change_test.cpp index e207c6ad8b..913cf9d276 100644 --- a/test/integration/raft/raft_config_change_test.cpp +++ b/test/integration/raft/raft_config_change_test.cpp @@ -41,6 +41,7 @@ using curve::fs::LocalFsFactory; using curve::fs::FileSystemType; const char kRaftConfigChangeTestLogDir[] = "./runlog/RaftConfigChange"; +const uint32_t kOpRequestAlignSize = 4096; static char* raftConfigParam[5][16] = { { diff --git a/test/integration/raft/raft_log_replication_test.cpp b/test/integration/raft/raft_log_replication_test.cpp index dc974e3cd2..bf0e5eb8b5 100644 --- a/test/integration/raft/raft_log_replication_test.cpp +++ b/test/integration/raft/raft_log_replication_test.cpp @@ -41,6 +41,8 @@ using curve::fs::LocalFileSystem; using curve::fs::LocalFsFactory; using curve::fs::FileSystemType; +const uint32_t kOpRequestAlignSize = 4096; + const char kRaftLogRepTestLogDir[] = "./runlog/RaftLogRep"; static char* raftLogParam[5][16] = { diff --git a/test/integration/raft/raft_snapshot_test.cpp b/test/integration/raft/raft_snapshot_test.cpp index 23dbf6d44c..24309b362e 100644 --- a/test/integration/raft/raft_snapshot_test.cpp +++ b/test/integration/raft/raft_snapshot_test.cpp @@ -42,6 +42,8 @@ using curve::fs::FileSystemType; const char kRaftSnapshotTestLogDir[] = "./runlog/RaftSnapshot"; +const uint32_t kOpRequestAlignSize = 4096; + static char *raftVoteParam[4][16] = { { "chunkserver", diff --git a/test/integration/raft/raft_vote_test.cpp b/test/integration/raft/raft_vote_test.cpp index eff9f63a7a..252421364f 100644 --- a/test/integration/raft/raft_vote_test.cpp +++ b/test/integration/raft/raft_vote_test.cpp @@ -42,6 +42,8 @@ using curve::fs::FileSystemType; const char kRaftVoteTestLogDir[] = "./runlog/RaftVote"; +const uint32_t kOpRequestAlignSize = 4096; + static char* raftVoteParam[3][16] = { { "chunkserver",