curvebs: support 512 aligned IO

opencurve · Apr 3, 2022 · a7fe2e2 · a7fe2e2
1 parent c2c5341
commit a7fe2e2
Show file tree

Hide file tree

Showing 128 changed files with 2,042 additions and 1,355 deletions.
diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf
@@ -12,10 +12,13 @@ global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings

diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -13,6 +13,11 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/conf/mds.conf b/conf/mds.conf
@@ -195,6 +195,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config

diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@@ -12,6 +12,8 @@ global.external_subnet={{ chunkserver_external_subnet }}
 global.chunk_size={{ chunk_size }}
 # chunk 元数据页大小，一般4KB
 global.meta_page_size={{ chunkserver_meta_page_size }}
+# chunk block size，一般4KB
+global.block_size={{ chunkserver_block_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
 

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/include/chunkserver/chunkserver_common.h b/include/chunkserver/chunkserver_common.h
@@ -123,9 +123,6 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
 }
 #define ToGroupIdStr   ToGroupIdString
 
-// TODO(wudmeiao): 是否需要考虑可配置
-const uint32_t kOpRequestAlignSize = 4096;
-
 }  // namespace chunkserver
 }  // namespace curve
 

diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -139,6 +139,7 @@ typedef struct FileStatInfo {
     int             fileStatus;
     uint64_t        stripeUnit;
     uint64_t        stripeCount;
+    uint32_t        blocksize;
 } FileStatInfo_t;
 
 // 存储用户信息
@@ -519,6 +520,8 @@ class CurveClient {
      */
     virtual int64_t StatFile(const std::string& filename);
 
+    virtual int64_t StatFile(const std::string& filename,
+                             FileStatInfo* fileStat);
     /**
      * 异步读
      * @param fd 文件fd

diff --git a/nbd/src/ImageInstance.cpp b/nbd/src/ImageInstance.cpp
@@ -77,10 +77,14 @@ void ImageInstance::Flush(NebdClientAioContext* context) {
     nebd_lib_flush(fd_, context);
 }
 
-int64_t ImageInstance::GetImageSize() {
+int64_t ImageInstance::GetImageSize() const {
     return nebd_lib_filesize(fd_);
 }
 
+int64_t ImageInstance::GetBlockSize() const {
+    return nebd_lib_blocksize(fd_);
+}
+
 ImageInstance::~ImageInstance() {
     if (fd_ != -1) {
         Close();

diff --git a/nbd/src/ImageInstance.h b/nbd/src/ImageInstance.h
@@ -83,7 +83,14 @@ class ImageInstance {
      * @return 获取成功返回文件大小（正值）
      *         获取失败返回错误码（负值）
      */
-    virtual int64_t GetImageSize();
+    virtual int64_t GetImageSize() const;
+
+    /**
+     * @brief Get image's block size
+     * @return return block size when success, 
+     *         otherwise return negative error code
+     */
+    virtual int64_t GetBlockSize() const;
 
  private:
     // nebd返回的文件描述符

diff --git a/nbd/src/NBDController.cpp b/nbd/src/NBDController.cpp
@@ -42,12 +42,14 @@
 namespace curve {
 namespace nbd {
 
-int IOController::InitDevAttr(NBDConfig* config, uint64_t size,
+int IOController::InitDevAttr(NBDConfig* config,
+                              uint64_t size,
+                              uint32_t blocksize,
                               uint64_t flags) {
     int ret = -1;
 
     do {
-        ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE);
+        ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, blocksize);
         if (ret < 0) {
             break;
         }
@@ -136,8 +138,11 @@ int IOController::MapOnNbdDeviceByDevPath(int sockfd,
     return 0;
 }
 
-int IOController::SetUp(NBDConfig* config, int sockfd,
-                        uint64_t size, uint64_t flags) {
+int IOController::SetUp(NBDConfig* config,
+                        int sockfd,
+                        uint64_t size,
+                        uint32_t blocksize,
+                        uint64_t flags) {
     int ret = -1;
 
     if (config->devpath.empty()) {
@@ -150,7 +155,7 @@ int IOController::SetUp(NBDConfig* config, int sockfd,
         return -1;
     }
 
-    ret = InitDevAttr(config, size, flags);
+    ret = InitDevAttr(config, size, blocksize, flags);
     if (ret == 0) {
         ret = check_device_size(nbdIndex_, size);
     }
@@ -240,16 +245,19 @@ void NetLinkController::Uninit() {
     nlId_ = -1;
 }
 
-int NetLinkController::SetUp(NBDConfig* config, int sockfd,
-                             uint64_t size, uint64_t flags) {
+int NetLinkController::SetUp(NBDConfig* config,
+                             int sockfd,
+                             uint64_t size,
+                             uint32_t blocksize,
+                             uint64_t flags) {
     int ret = Init();
     if (ret < 0) {
         dout << "curve-nbd: Netlink interface not supported."
              << " Using ioctl interface." << std::endl;
         return ret;
     }
 
-    ret = ConnectInternal(config, sockfd, size, flags);
+    ret = ConnectInternal(config, sockfd, size, blocksize, flags);
     Uninit();
     if (ret < 0) {
         return ret;
@@ -259,7 +267,7 @@ int NetLinkController::SetUp(NBDConfig* config, int sockfd,
     if (index < 0) {
         return index;
     }
-    ret = check_block_size(index, CURVE_NBD_BLKSIZE);
+    ret = check_block_size(index, blocksize);
     if (ret < 0) {
         return ret;
     }
@@ -360,8 +368,11 @@ static int netlink_connect_cb(struct nl_msg *msg, void *arg) {
     return NL_OK;
 }
 
-int NetLinkController::ConnectInternal(NBDConfig* config, int sockfd,
-                                       uint64_t size, uint64_t flags) {
+int NetLinkController::ConnectInternal(NBDConfig* config,
+                                       int sockfd,
+                                       uint64_t size,
+                                       uint32_t blocksize,
+                                       uint64_t flags) {
     struct nlattr *sock_attr = nullptr;
     struct nlattr *sock_opt = nullptr;
     struct nl_msg *msg = nullptr;
@@ -393,7 +404,7 @@ int NetLinkController::ConnectInternal(NBDConfig* config, int sockfd,
         NLA_PUT_U64(msg, NBD_ATTR_TIMEOUT, config->timeout);
     }
     NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
-    NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, CURVE_NBD_BLKSIZE);
+    NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, blocksize);
     NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags);
 
     sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS);

diff --git a/nbd/src/NBDController.h b/nbd/src/NBDController.h
@@ -71,11 +71,15 @@ class NBDController {
      * @param config: 启动NBD设备相关的配置参数
      * @param sockfd: socketpair其中一端的fd，传给NBD设备用于跟NBDServer间的数据传输
      * @param size: 设置NBD设备的大小
+     * @param blocksize: device's block size
      * @param flags: 设置加载NBD设备的flags
      * @return: 成功返回0，失败返回负值
      */
-    virtual int SetUp(NBDConfig* config, int sockfd,
-                      uint64_t size, uint64_t flags) = 0;
+    virtual int SetUp(NBDConfig* config,
+                      int sockfd,
+                      uint64_t size,
+                      uint32_t blocksize,
+                      uint64_t flags) = 0;
     /**
      * @brief: 根据设备名来卸载已经映射的NBD设备
      * @param devpath: 设备路径，例如/dev/nbd0
@@ -141,13 +145,19 @@ class IOController : public NBDController {
     IOController() {}
     ~IOController() {}
 
-    int SetUp(NBDConfig* config, int sockfd,
-              uint64_t size, uint64_t flags) override;
+    int SetUp(NBDConfig* config,
+              int sockfd,
+              uint64_t size,
+              uint32_t blocksize,
+              uint64_t flags) override;
     int DisconnectByPath(const std::string& devpath) override;
     int Resize(uint64_t size) override;
 
  private:
-    int InitDevAttr(NBDConfig* config, uint64_t size, uint64_t flags);
+    int InitDevAttr(NBDConfig* config,
+                    uint64_t size,
+                    uint32_t blocksize,
+                    uint64_t flags);
     int MapOnUnusedNbdDevice(int sockfd, std::string* devpath);
     int MapOnNbdDeviceByDevPath(int sockfd, const std::string& devpath,
                                 bool logWhenError = true);
@@ -158,8 +168,11 @@ class NetLinkController : public NBDController {
     NetLinkController() : nlId_(-1), sock_(nullptr) {}
     ~NetLinkController() {}
 
-    int SetUp(NBDConfig* config, int sockfd,
-              uint64_t size, uint64_t flags) override;
+    int SetUp(NBDConfig* config,
+              int sockfd,
+              uint64_t size,
+              uint32_t blocksize,
+              uint64_t flags) override;
     int DisconnectByPath(const std::string& devpath) override;
     int Resize(uint64_t size) override;
     bool Support();
@@ -171,8 +184,11 @@ class NetLinkController : public NBDController {
  private:
     int Init();
     void Uninit();
-    int ConnectInternal(NBDConfig* config, int sockfd,
-                        uint64_t size, uint64_t flags);
+    int ConnectInternal(NBDConfig* config,
+                        int sockfd,
+                        uint64_t size,
+                        uint32_t blocksize,
+                        uint64_t flags);
     int DisconnectInternal(int index);
     int ResizeInternal(int nbdIndex, uint64_t size);
 

diff --git a/nbd/src/NBDTool.cpp b/nbd/src/NBDTool.cpp
@@ -89,6 +89,13 @@ int NBDTool::Connect(NBDConfig *cfg) {
         return -1;
     }
 
+    int64_t blockSize = imageInstance->GetBlockSize();
+    if (blockSize <= 0) {
+        dout << "curve-nbd: Get block size failed, image block size: "
+             << blockSize << std::endl;
+        return -1;
+    }
+
     // load nbd module
     ret = load_module(cfg);
     if (ret < 0) {
@@ -106,7 +113,7 @@ int NBDTool::Connect(NBDConfig *cfg) {
     if (cfg->readonly) {
         flags |= NBD_FLAG_READ_ONLY;
     }
-    ret = nbdCtrl->SetUp(cfg, socketPair_.First(), fileSize, flags);
+    ret = nbdCtrl->SetUp(cfg, socketPair_.First(), fileSize, blockSize, flags);
     if (ret < 0) {
         dout << "nbd controller setup failed, imgname = " << cfg->imgname
              << std::endl;

diff --git a/nbd/src/define.h b/nbd/src/define.h
@@ -55,7 +55,6 @@ namespace nbd {
 
 #define HELP_INFO 1
 #define VERSION_INFO 2
-#define CURVE_NBD_BLKSIZE 4096UL    // CURVE后端当前支持4096大小对齐的IO
 
 #define NBD_MAX_PATH "/sys/module/nbd/parameters/nbds_max"
 #define PROCESS_NAME "curve-nbd"

diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp
@@ -320,16 +320,14 @@ int check_dev_can_unmap(const NBDConfig *cfg) {
 
 int check_size_from_file(const std::string &path, uint64_t expected_size,
                          bool sizeInSector = false) {
-    std::ifstream ifs;
-    ifs.open(path.c_str(), std::ifstream::in);
+    std::ifstream ifs(path, std::ifstream::in);
     if (!ifs.is_open()) {
         dout << "curve-nbd: failed to open " << path << std::endl;
         return -EINVAL;
     }
 
     uint64_t size = 0;
     ifs >> size;
-    size *= CURVE_NBD_BLKSIZE;
 
     if (size == 0) {
         // Newer kernel versions will report real size only after nbd
@@ -351,14 +349,15 @@ int check_size_from_file(const std::string &path, uint64_t expected_size,
 }
 
 int check_block_size(int nbd_index, uint64_t expected_size) {
-    std::string path = "/sys/block/nbd" + std::to_string(nbd_index)
-                     + "/queue/hw_sector_size";
+    std::string path = "/sys/block/nbd" + std::to_string(nbd_index) +
+                       "/queue/logical_block_size";
     int ret = check_size_from_file(path, expected_size);
     if (ret < 0) {
         return ret;
     }
-    path = "/sys/block/nbd" + std::to_string(nbd_index)
-         + "/queue/minimum_io_size";
+
+    path = "/sys/block/nbd" + std::to_string(nbd_index) +
+           "/queue/physical_block_size";
     ret = check_size_from_file(path, expected_size);
     if (ret < 0) {
         return ret;

diff --git a/nbd/test/mock_nbd_controller.h b/nbd/test/mock_nbd_controller.h
@@ -38,7 +38,7 @@ class MockNBDController : public NBDController {
     ~MockNBDController() = default;
 
     MOCK_METHOD1(Resize, int(uint64_t));
-    MOCK_METHOD4(SetUp, int(NBDConfig*, int, uint64_t, uint64_t));
+    MOCK_METHOD5(SetUp, int(NBDConfig*, int, uint64_t, uint32_t, uint64_t));
     MOCK_METHOD1(DisconnectByPath, int(const std::string&));
 };
 

diff --git a/nebd/proto/client.proto b/nebd/proto/client.proto
@@ -98,6 +98,7 @@ message FileInfo {
    required uint64 size = 1;
    required uint64 objSize = 2;
    required uint64 objNums = 3;
+   optional uint32 blocksize = 4;
 }
 
 message GetInfoRequest {

diff --git a/nebd/src/part1/libnebd.cpp b/nebd/src/part1/libnebd.cpp
@@ -21,6 +21,7 @@
  */
 
 #include "nebd/src/part1/libnebd.h"
+
 #include "nebd/src/part1/libnebd_file.h"
 
 extern "C" {
@@ -109,6 +110,10 @@ int64_t nebd_lib_filesize(int fd) {
     return GetFileSize4Nebd(fd);
 }
 
+int64_t nebd_lib_blocksize(int fd) {
+    return GetBlockSize4Nebd(fd);
+}
+
 int nebd_lib_resize(int fd, int64_t size) {
     return Extend4Nebd(fd, size);
 }