opencurve · wu-hanqing · Feb 22, 2022 · Feb 23, 2022 · ilixiaocui · Jul 12, 2022
diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf
@@ -12,10 +12,13 @@ global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings

diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings

diff --git a/conf/client.conf b/conf/client.conf
@@ -167,10 +167,3 @@ discard.enable=true
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
diff --git a/conf/cs_client.conf b/conf/cs_client.conf
@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
diff --git a/conf/mds.conf b/conf/mds.conf
@@ -195,6 +195,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config

diff --git a/conf/py_client.conf b/conf/py_client.conf
@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
diff --git a/conf/snap_client.conf b/conf/snap_client.conf
@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml
@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@@ -12,10 +12,10 @@ global.external_subnet={{ chunkserver_external_subnet }}
 global.chunk_size={{ chunk_size }}
 # chunk 元数据页大小，一般4KB
 global.meta_page_size={{ chunkserver_meta_page_size }}
+# chunk block size，一般4KB
+global.block_size={{ chunkserver_block_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings
@@ -110,9 +110,9 @@ copyset.scan_rpc_timeout_ms={{ chunkserver_copyset_scan_rpc_timeout_ms }}
 copyset.scan_rpc_retry_times={{ chunkserver_copyset_scan_rpc_retry_times }}
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us={{ chunkserver_copyset_scan_rpc_retry_interval_us }}
-copyset.copyset_enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
-copyset.copyset_synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
-copyset.copyset_check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
+copyset.enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
+copyset.synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
+copyset.check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
 
 #
 # Clone settings

diff --git a/curve-ansible/roles/generate_config/templates/client.conf.j2 b/curve-ansible/roles/generate_config/templates/client.conf.j2
@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}
diff --git a/curve-ansible/roles/generate_config/templates/mds.conf.j2 b/curve-ansible/roles/generate_config/templates/mds.conf.j2
@@ -203,6 +203,8 @@ mds.curvefs.defaultSegmentSize={{ segment_size }}
 mds.curvefs.minFileLength={{ min_file_length }}
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength={{ max_file_length }}
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize={{ chunkserver_block_size }}
 
 #
 # chunkseverclient config

diff --git a/curve-ansible/roles/generate_config/templates/s3.conf.j2 b/curve-ansible/roles/generate_config/templates/s3.conf.j2
@@ -25,3 +25,4 @@ s3.throttle.iopsWriteLimit={{ s3_throttle_iopsWriteLimit }}
 s3.throttle.bpsTotalMB= {{ s3_throttle_bpsTotalLimit }}
 s3.throttle.bpsReadMB= {{ s3_throttle_bpsReadLimit }}
 s3.throttle.bpsWriteMB= {{ s3_throttle_bpsWriteLimit }}
+s3.useVirtualAddressing=False
diff --git a/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2 b/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2
@@ -167,7 +167,9 @@ do
   -filePoolDir=$dataDir/chunkserver$i/chunkfilepool \
   -filePoolMetaPath=$dataDir/chunkserver$i/chunkfilepool.meta \
   -fileSize={{ chunk_size }} \
-  -fileSystemPath=$dataDir/chunkserver$i/chunkfilepool  &
+  -fileSystemPath=$dataDir/chunkserver$i/chunkfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
 done
 wait
 }
@@ -185,7 +187,9 @@ function deploy_one_walfile_pool {
   -filePoolDir=$dataDir/chunkserver$1/walfilepool \
   -filePoolMetaPath=$dataDir/chunkserver$1/walfilepool.meta \
   -fileSize={{ chunkserver_walfilepool_segment_size }} \
-  -fileSystemPath=$dataDir/chunkserver$1/walfilepool  &
+  -fileSystemPath=$dataDir/chunkserver$1/walfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
 }
 
 
@@ -286,7 +290,9 @@ function deploy_one {
   -filePoolDir=$dirname/chunkfilepool \
   -filePoolMetaPath=$dirname/chunkfilepool.meta \
   -fileSize={{ chunk_size }} \
-  -fileSystemPath=$dirname/chunkfilepool  &
+  -fileSystemPath=$dirname/chunkfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
   wait
   # release disk reserved space
   sudo tune2fs -m 0 $diskname

diff --git a/curve-ansible/server.ini b/curve-ansible/server.ini
@@ -101,6 +101,8 @@ chunk_size=16777216
 chunkserver_walfilepool_segment_size=8388608
 retain_pool=False
 walfilepool_use_chunk_file_pool=True
+chunkserver_meta_page_size=4096
+chunkserver_block_size=4096
 
 [snapshotclone_nginx:vars]
 snapshot_nginx_package_version="0.0.6.1.1+7af4d6a4"

diff --git a/curvefs/src/client/fuse_client.h b/curvefs/src/client/fuse_client.h
@@ -41,7 +41,7 @@
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/common/fast_align.h"
+#include "src/common/fast_align.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/common/define.h"
@@ -69,7 +69,7 @@ using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 
-using curvefs::common::is_aligned;
+using curve::common::is_aligned;
 
 const uint32_t kMaxHostNameLength = 255u;
 

diff --git a/curvefs/src/common/fast_align.h b/curvefs/src/common/fast_align.h
diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0
@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1
@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2
@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #

diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -56,6 +56,7 @@ typedef struct FileStatInfo {
     int             fileStatus;
     uint64_t        stripeUnit;
     uint64_t        stripeCount;
+    uint32_t        blocksize;
 } FileStatInfo_t;
 
 // 存储用户信息
@@ -436,6 +437,8 @@ class CurveClient {
      */
     virtual int64_t StatFile(const std::string& filename);
 
+    virtual int64_t StatFile(const std::string& filename,
+                             FileStatInfo* fileStat);
     /**
      * 异步读
      * @param fd 文件fd

diff --git a/nbd/src/ImageInstance.cpp b/nbd/src/ImageInstance.cpp
@@ -77,10 +77,14 @@ void ImageInstance::Flush(NebdClientAioContext* context) {
     nebd_lib_flush(fd_, context);
 }
 
-int64_t ImageInstance::GetImageSize() {
+int64_t ImageInstance::GetImageSize() const {
     return nebd_lib_filesize(fd_);
 }
 
+int64_t ImageInstance::GetBlockSize() const {
+    return nebd_lib_blocksize(fd_);
+}
+
 ImageInstance::~ImageInstance() {
     if (fd_ != -1) {
         Close();

diff --git a/nbd/src/ImageInstance.h b/nbd/src/ImageInstance.h
@@ -83,7 +83,14 @@ class ImageInstance {
      * @return 获取成功返回文件大小（正值）
      *         获取失败返回错误码（负值）
      */
-    virtual int64_t GetImageSize();
+    virtual int64_t GetImageSize() const;
+
+    /**
+     * @brief Get image's block size
+     * @return return block size when success, 
+     *         otherwise return negative error code
+     */
+    virtual int64_t GetBlockSize() const;
 
  private:
     // nebd返回的文件描述符