From 46042aa74a1caf1f1f96417a6fff0304e2118167 Mon Sep 17 00:00:00 2001 From: zhao liwei Date: Tue, 9 Feb 2021 11:13:54 +0800 Subject: [PATCH] fix: set burst_size of _write_token_bucket in fds_service to double max (#755) --- src/block_service/fds/fds_service.cpp | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/block_service/fds/fds_service.cpp b/src/block_service/fds/fds_service.cpp index e031c68262..9dc2b9d546 100644 --- a/src/block_service/fds/fds_service.cpp +++ b/src/block_service/fds/fds_service.cpp @@ -119,22 +119,15 @@ const std::string fds_service::FILE_MD5_KEY = "content-md5"; fds_service::fds_service() { - /// In normal scenario, the sst file size of level 0 is write_buffer_size * [0.75, 1.25] - /// And in BULK_LOAD scenario, it is 4 * write_buffer_size * [0.75, 1.25]. - /// In rdsn, we can't get the scenario, so if we take BULK_LOAD scenario into consideration, - /// we must set max_sst_file_size to 4 * write_buffer_size * [0.75, 1.25], which is too big. - /// So in this implementation, we don't take BULK_LOAD scenario into consideration. - uint64_t max_sst_file_size = std::max(FLAGS_rocksdb_target_file_size_base, - (uint64_t)1.25 * FLAGS_rocksdb_write_buffer_size); - /// For write operation, we can't send a file in batches. Because putContent interface of fds /// will overwrite what was sent before for the same file. So we must send a file as a whole. /// If file size > burst size, the file will be rejected by the token bucket. - /// Here we set burst_size = max_sst_file_size + 3MB, a litte greater than max_sst_file_size - uint32_t burst_size = - std::max(2.0 * (FLAGS_fds_write_limit_rate << 20), max_sst_file_size + 3e6); - _write_token_bucket.reset(new folly::TokenBucket(FLAGS_fds_write_limit_rate << 20, burst_size)); - burst_size = 2 * FLAGS_fds_read_limit_rate << 20; + /// Here we set burst_size to max value of double, in order to make file not rejected by the + /// token bucket + _write_token_bucket.reset(new folly::TokenBucket(FLAGS_fds_write_limit_rate << 20, + std::numeric_limits::max())); + + uint32_t burst_size = 2 * FLAGS_fds_read_limit_rate << 20; _read_token_bucket.reset(new folly::TokenBucket(FLAGS_fds_read_limit_rate << 20, burst_size)); } @@ -444,7 +437,6 @@ error_code fds_file_object::get_content_in_batches(uint64_t start, /*out*/ std::ostream &os, /*out*/ uint64_t &transfered_bytes) { - // the max batch size is 1MB const uint64_t BATCH_SIZE = std::min(FLAGS_fds_read_limit_rate, FLAGS_fds_read_batch_size) << 20; error_code err = ERR_OK;