curvebs: support 512 aligned IO

Signed-off-by: wuhanqing <[email protected]>
opencurve · Jun 23, 2022 · a5b44d7 · a5b44d7
1 parent 6bab185
commit a5b44d7
Show file tree

Hide file tree

Showing 137 changed files with 2,132 additions and 1,364 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -33,7 +33,7 @@ bazel_skylib_workspace()
 
 git_repository(
     name = "com_github_baidu_braft",
-    remote = "https://github.com/baidu/braft",
+    remote = "https://gitee.com/baidu/braft",
     commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
 )
 
@@ -52,7 +52,7 @@ http_archive(
     build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
     sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
     strip_prefix = "zlib-1.2.11",
-    urls = ["https://zlib.net/zlib-1.2.11.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/zlib-1.2.11.tar.gz"],
 )
 
 bind(
@@ -66,7 +66,7 @@ http_archive(
     patch_args = ["-p1"],
     patches = ["//:thirdparties/protobuf/protobuf.patch"],
     sha256 = "9510dd2afc29e7245e9e884336f848c8a6600a14ae726adb6befdb4f786f0be2",
-    urls = ["https://github.com/google/protobuf/archive/v3.6.1.3.zip"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/protobuf-3.6.1.3.zip"],
 )
 
 bind(
@@ -78,7 +78,7 @@ bind(
 new_git_repository(
     name = "com_google_googletest",
     build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
+    remote = "https://gitee.com/mirrors/googletest",
     tag = "release-1.8.0",
 )
 
@@ -91,7 +91,7 @@ bind(
 # brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog"
 git_repository(
     name = "com_github_google_glog",
-    remote = "https://github.com/google/glog",
+    remote = "https://gitee.com/mirrors/glog",
     commit = "4cc89c9e2b452db579397887c37f302fb28f6ca1",
     patch_args = ["-p1"],
     patches = ["//:thirdparties/glog/glog.patch"],
@@ -106,7 +106,7 @@ bind(
 http_archive(
     name = "com_github_gflags_gflags",
     strip_prefix = "gflags-2.2.2",
-    urls = ["https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/gflags-2.2.2.tar.gz"],
 )
 
 bind(
@@ -118,7 +118,7 @@ http_archive(
     name = "com_github_google_leveldb",
     build_file = "@com_github_apache_brpc//:leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
-    urls = ["https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
 )
 
 bind(
@@ -128,7 +128,7 @@ bind(
 
 git_repository(
     name = "com_github_apache_brpc",
-    remote = "https://github.com/apache/incubator-brpc",
+    remote = "https://gitee.com/baidu/BRPC",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
     patches = ["//:thirdparties/brpc/brpc.patch"],
     patch_args = ["-p1"],
@@ -158,7 +158,7 @@ bind(
 new_git_repository(
     name = "jsoncpp",
     build_file = "//:thirdparties/jsoncpp.BUILD",
-    remote = "https://github.com/open-source-parsers/jsoncpp.git",
+    remote = "https://gitee.com/mirrors/jsoncpp",
     tag = "1.8.4",
 )
 
@@ -175,31 +175,31 @@ new_local_repository(
 
 http_archive(
     name = "aws",
-    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/aws-sdk-cpp-1.7.340.tar.gz"],
     sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
     strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
 )
 
 http_archive(
     name = "aws_c_common",
-    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/aws-c-common-0.4.29.tar.gz"],
     sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
     strip_prefix = "aws-c-common-0.4.29",
     build_file = "//:thirdparties/aws/aws-c-common.BUILD",
 )
 
 http_archive(
     name = "aws_c_event_stream",
-    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/aws-c-event-stream-0.1.4.tar.gz"],
     sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
     strip_prefix = "aws-c-event-stream-0.1.4",
     build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
 )
 
 http_archive(
     name = "aws_checksums",
-    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/aws-checksums-0.1.5.tar.gz"],
     sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
@@ -216,7 +216,7 @@ http_archive(
 # abseil-cpp
 http_archive(
   name = "com_google_absl",
-  urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz"],
+  urls = ["https://curve-build.nos-eastchina1.126.net/abseil-cpp-20210324.2.tar.gz"],
   strip_prefix = "abseil-cpp-20210324.2",
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
@@ -226,7 +226,7 @@ http_archive(
     name = "platforms",
     sha256 = "b601beaf841244de5c5a50d2b2eddd34839788000fa1be4260ce6603ca0d8eb7",
     strip_prefix = "platforms-98939346da932eef0b54cf808622f5bb0928f00b",
-    urls = ["https://github.com/bazelbuild/platforms/archive/98939346da932eef0b54cf808622f5bb0928f00b.zip"],
+    urls = ["https://curve-build.nos-eastchina1.126.net/platforms-98939346da932eef0b54cf808622f5bb0928f00b.zip"],
 )
 
 # RocksDB

diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf
@@ -12,10 +12,13 @@ global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings

diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -13,6 +13,11 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/conf/mds.conf b/conf/mds.conf
@@ -195,6 +195,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config

diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@@ -12,6 +12,8 @@ global.external_subnet={{ chunkserver_external_subnet }}
 global.chunk_size={{ chunk_size }}
 # chunk 元数据页大小，一般4KB
 global.meta_page_size={{ chunkserver_meta_page_size }}
+# chunk block size，一般4KB
+global.block_size={{ chunkserver_block_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
 
@@ -108,9 +110,9 @@ copyset.scan_rpc_timeout_ms={{ chunkserver_copyset_scan_rpc_timeout_ms }}
 copyset.scan_rpc_retry_times={{ chunkserver_copyset_scan_rpc_retry_times }}
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us={{ chunkserver_copyset_scan_rpc_retry_interval_us }}
-copyset.copyset_enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
-copyset.copyset_synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
-copyset.copyset_check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
+copyset.enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
+copyset.synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
+copyset.check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
 
 #
 # Clone settings

diff --git a/curve-ansible/roles/generate_config/templates/mds.conf.j2 b/curve-ansible/roles/generate_config/templates/mds.conf.j2
@@ -203,6 +203,8 @@ mds.curvefs.defaultSegmentSize={{ segment_size }}
 mds.curvefs.minFileLength={{ min_file_length }}
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength={{ max_file_length }}
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize={{ chunkserver_block_size }}
 
 #
 # chunkseverclient config

diff --git a/curve-ansible/roles/generate_config/templates/s3.conf.j2 b/curve-ansible/roles/generate_config/templates/s3.conf.j2
@@ -25,3 +25,4 @@ s3.throttle.iopsWriteLimit={{ s3_throttle_iopsWriteLimit }}
 s3.throttle.bpsTotalMB= {{ s3_throttle_bpsTotalLimit }}
 s3.throttle.bpsReadMB= {{ s3_throttle_bpsReadLimit }}
 s3.throttle.bpsWriteMB= {{ s3_throttle_bpsWriteLimit }}
+s3.useVirtualAddressing=False
diff --git a/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2 b/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2
@@ -167,7 +167,9 @@ do
   -filePoolDir=$dataDir/chunkserver$i/chunkfilepool \
   -filePoolMetaPath=$dataDir/chunkserver$i/chunkfilepool.meta \
   -fileSize={{ chunk_size }} \
-  -fileSystemPath=$dataDir/chunkserver$i/chunkfilepool  &
+  -fileSystemPath=$dataDir/chunkserver$i/chunkfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
 done
 wait
 }
@@ -185,7 +187,9 @@ function deploy_one_walfile_pool {
   -filePoolDir=$dataDir/chunkserver$1/walfilepool \
   -filePoolMetaPath=$dataDir/chunkserver$1/walfilepool.meta \
   -fileSize={{ chunkserver_walfilepool_segment_size }} \
-  -fileSystemPath=$dataDir/chunkserver$1/walfilepool  &
+  -fileSystemPath=$dataDir/chunkserver$1/walfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
 }
 
 
@@ -286,7 +290,9 @@ function deploy_one {
   -filePoolDir=$dirname/chunkfilepool \
   -filePoolMetaPath=$dirname/chunkfilepool.meta \
   -fileSize={{ chunk_size }} \
-  -fileSystemPath=$dirname/chunkfilepool  &
+  -fileSystemPath=$dirname/chunkfilepool \
+  -metaPageSize={{ chunkserver_meta_page_size }} \
+  -blockSize={{ chunkserver_block_size }} &
   wait
   # release disk reserved space
   sudo tune2fs -m 0 $diskname

diff --git a/curve-ansible/server.ini b/curve-ansible/server.ini
@@ -101,6 +101,8 @@ chunk_size=16777216
 chunkserver_walfilepool_segment_size=8388608
 retain_pool=False
 walfilepool_use_chunk_file_pool=True
+chunkserver_meta_page_size=4096
+chunkserver_block_size=4096
 
 [snapshotclone_nginx:vars]
 snapshot_nginx_package_version="0.0.6.1.1+7af4d6a4"

diff --git a/curvefs_python/BUILD_bak b/curvefs_python/BUILD_bak
@@ -54,7 +54,6 @@ cc_library(
         "//proto:nameserver2_cc_proto",
         "//proto:common_cc_proto",
         "//proto:topology_cc_proto",
-        "//proto:scan_cc_proto",
         "//proto:chunkserver-cc-protos",
         "//src/client:curve_client"
     ],
@@ -80,7 +79,6 @@ cc_library(
         "-lnameserver2_proto",
         "-ltopology_proto",
         "-lcommon_proto",
-        "-lscan_proto",
         "-lchunkserver-protos",
         "-lprotobuf",
         "-lprotobuf_lite",

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2
@@ -26,6 +26,7 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
 #

diff --git a/include/chunkserver/chunkserver_common.h b/include/chunkserver/chunkserver_common.h
@@ -123,9 +123,6 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
 }
 #define ToGroupIdStr   ToGroupIdString
 
-// TODO(wudmeiao): 是否需要考虑可配置
-const uint32_t kOpRequestAlignSize = 4096;
-
 }  // namespace chunkserver
 }  // namespace curve
 

diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -56,6 +56,7 @@ typedef struct FileStatInfo {
     int             fileStatus;
     uint64_t        stripeUnit;
     uint64_t        stripeCount;
+    uint32_t        blocksize;
 } FileStatInfo_t;
 
 // 存储用户信息
@@ -436,6 +437,8 @@ class CurveClient {
      */
     virtual int64_t StatFile(const std::string& filename);
 
+    virtual int64_t StatFile(const std::string& filename,
+                             FileStatInfo* fileStat);
     /**
      * 异步读
      * @param fd 文件fd

diff --git a/nbd/src/ImageInstance.cpp b/nbd/src/ImageInstance.cpp
@@ -77,10 +77,14 @@ void ImageInstance::Flush(NebdClientAioContext* context) {
     nebd_lib_flush(fd_, context);
 }
 
-int64_t ImageInstance::GetImageSize() {
+int64_t ImageInstance::GetImageSize() const {
     return nebd_lib_filesize(fd_);
 }
 
+int64_t ImageInstance::GetBlockSize() const {
+    return nebd_lib_blocksize(fd_);
+}
+
 ImageInstance::~ImageInstance() {
     if (fd_ != -1) {
         Close();

diff --git a/nbd/src/ImageInstance.h b/nbd/src/ImageInstance.h
@@ -83,7 +83,14 @@ class ImageInstance {
      * @return 获取成功返回文件大小（正值）
      *         获取失败返回错误码（负值）
      */
-    virtual int64_t GetImageSize();
+    virtual int64_t GetImageSize() const;
+
+    /**
+     * @brief Get image's block size
+     * @return return block size when success, 
+     *         otherwise return negative error code
+     */
+    virtual int64_t GetBlockSize() const;
 
  private:
     // nebd返回的文件描述符