From 0ace0c9c63202007bb40a5eee26cd79f8fb66130 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 31 Aug 2021 16:59:34 +0800 Subject: [PATCH 01/11] init limit --- src/nfs/nfs_client_impl.cpp | 2 +- src/nfs/nfs_server_impl.cpp | 7 +++++++ src/nfs/nfs_server_impl.h | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index c9cd17eca7..18d2180cfb 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -75,7 +75,7 @@ DSN_DEFINE_int32("nfs", "maximum retry count when copy failed"); DSN_DEFINE_int32("nfs", rpc_timeout_ms, - 10000, + 200000, "rpc timeout in milliseconds for nfs copy, " "0 means use default timeout of rpc engine"); diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index be04c2ecd3..ca4141c757 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -33,6 +33,7 @@ * xxxx-xx-xx, author, fix bug about xxx */ #include +#include #include #include #include @@ -42,6 +43,8 @@ namespace dsn { namespace service { +DSN_DEFINE_int32("nfs", max_send_rate_megabytes, 50, "max rate of send to remote node(MB/s)"); + DSN_DECLARE_int32(file_close_timer_interval_ms_on_server); DSN_DECLARE_int32(file_close_expire_time_ms); @@ -62,6 +65,8 @@ nfs_service_impl::nfs_service_impl() : ::dsn::serverlet("nfs") "recent_copy_fail_count", COUNTER_TYPE_VOLATILE_NUMBER, "nfs server copy fail count count in the recent period"); + + _send_token_bucket = std::make_unique(); } void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, @@ -131,6 +136,8 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, void nfs_service_impl::internal_read_callback(error_code err, size_t sz, callback_para &cp) { + _send_token_bucket->consumeWithBorrowAndWait( + sz, FLAGS_max_send_rate_megabytes << 20, 1.5 * (FLAGS_max_send_rate_megabytes << 20)); { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(cp.file_path); diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 9abd89b516..abb4c33380 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -122,6 +122,9 @@ class nfs_service_impl : public ::dsn::serverlet ::dsn::task_ptr _file_close_timer; + std::unique_ptr + _send_token_bucket; // rate limiter of copy from remote + perf_counter_wrapper _recent_copy_data_size; perf_counter_wrapper _recent_copy_fail_count; From a52a93e94a1d6cb5de5a1ee32b899a2e31d496ee Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 31 Aug 2021 17:07:56 +0800 Subject: [PATCH 02/11] init limit --- src/nfs/nfs_server_impl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index ca4141c757..c9198dacde 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -44,6 +44,7 @@ namespace dsn { namespace service { DSN_DEFINE_int32("nfs", max_send_rate_megabytes, 50, "max rate of send to remote node(MB/s)"); +DSN_TAG_VARIABLE(max_send_rate_megabytes, FT_MUTABLE); DSN_DECLARE_int32(file_close_timer_interval_ms_on_server); DSN_DECLARE_int32(file_close_expire_time_ms); From b730600d3e1a832fb3a35c55241970e22d37eb95 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 31 Aug 2021 19:28:55 +0800 Subject: [PATCH 03/11] init limit --- src/nfs/nfs_server_impl.cpp | 48 ++++++++++++++++++++++++++++++++++--- src/nfs/nfs_server_impl.h | 12 ++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index c9198dacde..f3424dd0fd 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -37,12 +37,16 @@ #include #include #include +#include +#include #include "nfs_server_impl.h" namespace dsn { namespace service { +static uint32_t current_send_copy_rate_megabytes = 0; + DSN_DEFINE_int32("nfs", max_send_rate_megabytes, 50, "max rate of send to remote node(MB/s)"); DSN_TAG_VARIABLE(max_send_rate_megabytes, FT_MUTABLE); @@ -67,7 +71,10 @@ nfs_service_impl::nfs_service_impl() : ::dsn::serverlet("nfs") COUNTER_TYPE_VOLATILE_NUMBER, "nfs server copy fail count count in the recent period"); - _send_token_bucket = std::make_unique(); + _send_token_bucket = std::make_unique(FLAGS_max_send_rate_megabytes << 20, + 1.5 * (FLAGS_max_send_rate_megabytes << 20)); + current_send_copy_rate_megabytes = FLAGS_max_send_rate_megabytes; + register_cli_commands(); } void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, @@ -137,8 +144,7 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, void nfs_service_impl::internal_read_callback(error_code err, size_t sz, callback_para &cp) { - _send_token_bucket->consumeWithBorrowAndWait( - sz, FLAGS_max_send_rate_megabytes << 20, 1.5 * (FLAGS_max_send_rate_megabytes << 20)); + _send_token_bucket->consumeWithBorrowAndWait(sz); { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(cp.file_path); @@ -249,5 +255,41 @@ void nfs_service_impl::close_file() // release out-of-date file handle it++; } } + +void nfs_service_impl::register_cli_commands() +{ + static std::once_flag flag; + std::call_once(flag, [&]() { + _nfs_max_send_rate_megabytes_cmd = dsn::command_manager::instance().register_command( + {"nfs.max_send_rate_megabytes"}, + "nfs.max_send_rate_megabytes [num | DEFAULT]", + "control the max rate(MB/s) to copy file from remote node", + [this](const std::vector &args) { + std::string result("OK"); + + if (args.empty()) { + return std::to_string(current_send_copy_rate_megabytes); + } + + if (args[0] == "DEFAULT") { + uint32_t max_send_rate_bytes = FLAGS_max_send_rate_megabytes << 20; + _send_token_bucket->reset(max_send_rate_bytes, 1.5 * max_send_rate_bytes); + current_send_copy_rate_megabytes = FLAGS_max_send_rate_megabytes; + return result; + } + + int32_t max_send_rate_megabytes = 0; + if (!dsn::buf2int32(args[0], max_send_rate_megabytes) || + max_send_rate_megabytes <= 0) { + return std::string("ERR: invalid arguments"); + } + + _send_token_bucket->reset(max_send_rate_megabytes, 1.5 * max_send_rate_megabytes); + current_send_copy_rate_megabytes = max_send_rate_megabytes; + return result; + }); + }); +} + } // namespace service } // namespace dsn diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index abb4c33380..da6aea2c06 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -30,6 +30,7 @@ #include #include #include +#include #include "nfs_code_definition.h" #include "nfs_types.h" @@ -37,6 +38,9 @@ namespace dsn { namespace service { + +using TokenBucket = folly::BasicTokenBucket; + class nfs_service_impl : public ::dsn::serverlet { public: @@ -50,10 +54,13 @@ class nfs_service_impl : public ::dsn::serverlet RPC_NFS_GET_FILE_SIZE, "get_file_size", &nfs_service_impl::on_get_file_size); } + void register_cli_commands(); + void close_service() { unregister_rpc_handler(RPC_NFS_COPY); unregister_rpc_handler(RPC_NFS_GET_FILE_SIZE); + UNREGISTER_VALID_HANDLER(_nfs_max_send_rate_megabytes_cmd); } protected: @@ -122,12 +129,13 @@ class nfs_service_impl : public ::dsn::serverlet ::dsn::task_ptr _file_close_timer; - std::unique_ptr - _send_token_bucket; // rate limiter of copy from remote + std::unique_ptr _send_token_bucket; // rate limiter of copy from remote perf_counter_wrapper _recent_copy_data_size; perf_counter_wrapper _recent_copy_fail_count; + dsn_handle_t _nfs_max_send_rate_megabytes_cmd; + dsn::task_tracker _tracker; }; } // namespace service From 7c4359ffa11ce373f09c7fdc83621a9c8d36f51b Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 31 Aug 2021 20:20:53 +0800 Subject: [PATCH 04/11] init limit --- src/nfs/nfs_server_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index f3424dd0fd..f15ea54ded 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -284,7 +284,7 @@ void nfs_service_impl::register_cli_commands() return std::string("ERR: invalid arguments"); } - _send_token_bucket->reset(max_send_rate_megabytes, 1.5 * max_send_rate_megabytes); + _send_token_bucket->reset(max_send_rate_megabytes << 20, 1.5 * (max_send_rate_megabytes <<20)); current_send_copy_rate_megabytes = max_send_rate_megabytes; return result; }); From 3699137461985fe9cd785c6c817ebc175a8fd363 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Wed, 1 Sep 2021 15:09:44 +0800 Subject: [PATCH 05/11] init limit --- src/nfs/nfs_server_impl.cpp | 30 ++++++++++-------------------- src/nfs/nfs_server_impl.h | 5 ++--- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index f15ea54ded..27169b0f69 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -45,9 +45,7 @@ namespace dsn { namespace service { -static uint32_t current_send_copy_rate_megabytes = 0; - -DSN_DEFINE_int32("nfs", max_send_rate_megabytes, 50, "max rate of send to remote node(MB/s)"); +DSN_DEFINE_int32("nfs", max_send_rate_megabytes, 100, "max rate of send to remote node(MB/s)"); DSN_TAG_VARIABLE(max_send_rate_megabytes, FT_MUTABLE); DSN_DECLARE_int32(file_close_timer_interval_ms_on_server); @@ -71,9 +69,7 @@ nfs_service_impl::nfs_service_impl() : ::dsn::serverlet("nfs") COUNTER_TYPE_VOLATILE_NUMBER, "nfs server copy fail count count in the recent period"); - _send_token_bucket = std::make_unique(FLAGS_max_send_rate_megabytes << 20, - 1.5 * (FLAGS_max_send_rate_megabytes << 20)); - current_send_copy_rate_megabytes = FLAGS_max_send_rate_megabytes; + _send_token_bucket = std::make_unique(); register_cli_commands(); } @@ -144,7 +140,8 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, void nfs_service_impl::internal_read_callback(error_code err, size_t sz, callback_para &cp) { - _send_token_bucket->consumeWithBorrowAndWait(sz); + _send_token_bucket->consumeWithBorrowAndWait( + sz, FLAGS_max_send_rate_megabytes << 20, 1.5 * (FLAGS_max_send_rate_megabytes << 20)); { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(cp.file_path); @@ -256,26 +253,20 @@ void nfs_service_impl::close_file() // release out-of-date file handle } } +// TODO(jiashuo1): just for compatibility, ready to delete it later void nfs_service_impl::register_cli_commands() { static std::once_flag flag; std::call_once(flag, [&]() { _nfs_max_send_rate_megabytes_cmd = dsn::command_manager::instance().register_command( {"nfs.max_send_rate_megabytes"}, - "nfs.max_send_rate_megabytes [num | DEFAULT]", - "control the max rate(MB/s) to copy file from remote node", - [this](const std::vector &args) { + "nfs.max_send_rate_megabytes [num]", + "control the max rate(MB/s) to send file to remote node", + [](const std::vector &args) { std::string result("OK"); if (args.empty()) { - return std::to_string(current_send_copy_rate_megabytes); - } - - if (args[0] == "DEFAULT") { - uint32_t max_send_rate_bytes = FLAGS_max_send_rate_megabytes << 20; - _send_token_bucket->reset(max_send_rate_bytes, 1.5 * max_send_rate_bytes); - current_send_copy_rate_megabytes = FLAGS_max_send_rate_megabytes; - return result; + return std::to_string(FLAGS_max_send_rate_megabytes); } int32_t max_send_rate_megabytes = 0; @@ -284,8 +275,7 @@ void nfs_service_impl::register_cli_commands() return std::string("ERR: invalid arguments"); } - _send_token_bucket->reset(max_send_rate_megabytes << 20, 1.5 * (max_send_rate_megabytes <<20)); - current_send_copy_rate_megabytes = max_send_rate_megabytes; + FLAGS_max_send_rate_megabytes = max_send_rate_megabytes; return result; }); }); diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index da6aea2c06..821787b1ba 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -39,8 +39,6 @@ namespace dsn { namespace service { -using TokenBucket = folly::BasicTokenBucket; - class nfs_service_impl : public ::dsn::serverlet { public: @@ -129,7 +127,8 @@ class nfs_service_impl : public ::dsn::serverlet ::dsn::task_ptr _file_close_timer; - std::unique_ptr _send_token_bucket; // rate limiter of copy from remote + std::unique_ptr + _send_token_bucket; // rate limiter of copy from remote perf_counter_wrapper _recent_copy_data_size; perf_counter_wrapper _recent_copy_fail_count; From 9394aafc15aeef1466d85d590d67a8171bf16085 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 2 Sep 2021 17:45:36 +0800 Subject: [PATCH 06/11] init limit --- include/dsn/http/http_server.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/dsn/http/http_server.h b/include/dsn/http/http_server.h index fcfb317713..ff5faf7ca6 100644 --- a/include/dsn/http/http_server.h +++ b/include/dsn/http/http_server.h @@ -28,8 +28,7 @@ namespace dsn { DSN_DECLARE_bool(enable_http_server); /// The rpc code for all the HTTP RPCs. -/// Since http is used only for system monitoring, it is restricted to lowest priority. -DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_LOW, THREAD_POOL_DEFAULT); +DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT); enum http_method { From f018976a472a725f74106f580c5f93c7066daf95 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Fri, 3 Sep 2021 17:42:53 +0800 Subject: [PATCH 07/11] init limit --- src/nfs/nfs_client_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index 18d2180cfb..67570685f6 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -75,7 +75,7 @@ DSN_DEFINE_int32("nfs", "maximum retry count when copy failed"); DSN_DEFINE_int32("nfs", rpc_timeout_ms, - 200000, + 100000, "rpc timeout in milliseconds for nfs copy, " "0 means use default timeout of rpc engine"); From fd6d979e596aa7b98e6e3c22fcf20c30c9d9e785 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Fri, 3 Sep 2021 18:04:41 +0800 Subject: [PATCH 08/11] init limit --- src/nfs/nfs_server_impl.cpp | 2 -- src/nfs/nfs_server_impl.h | 1 - 2 files changed, 3 deletions(-) diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index 27169b0f69..cd8d8f3cdb 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -33,11 +33,9 @@ * xxxx-xx-xx, author, fix bug about xxx */ #include -#include #include #include #include -#include #include #include "nfs_server_impl.h" diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 821787b1ba..bfdf0a13da 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -38,7 +38,6 @@ namespace dsn { namespace service { - class nfs_service_impl : public ::dsn::serverlet { public: From b9554a9b976e0844f892c440172b3162b0d515a7 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Fri, 3 Sep 2021 18:05:57 +0800 Subject: [PATCH 09/11] init limit --- src/nfs/nfs_server_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index bfdf0a13da..0680229957 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -127,7 +127,7 @@ class nfs_service_impl : public ::dsn::serverlet ::dsn::task_ptr _file_close_timer; std::unique_ptr - _send_token_bucket; // rate limiter of copy from remote + _send_token_bucket; // rate limiter of send to remote perf_counter_wrapper _recent_copy_data_size; perf_counter_wrapper _recent_copy_fail_count; From 3667be86a1040921fc4dea998a82a3d252f6079c Mon Sep 17 00:00:00 2001 From: jiashuo Date: Fri, 3 Sep 2021 18:07:18 +0800 Subject: [PATCH 10/11] init limit --- src/nfs/nfs_server_impl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 0680229957..49edf96760 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -126,8 +126,7 @@ class nfs_service_impl : public ::dsn::serverlet ::dsn::task_ptr _file_close_timer; - std::unique_ptr - _send_token_bucket; // rate limiter of send to remote + std::unique_ptr _send_token_bucket; // rate limiter of send to remote perf_counter_wrapper _recent_copy_data_size; perf_counter_wrapper _recent_copy_fail_count; From 87ef6d21c661bd89d110cf6ac6070052aad4e4e7 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Fri, 3 Sep 2021 18:26:00 +0800 Subject: [PATCH 11/11] init limit --- src/nfs/nfs_client_impl.cpp | 2 +- src/nfs/nfs_server_impl.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index 67570685f6..edcec4e92c 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -75,7 +75,7 @@ DSN_DEFINE_int32("nfs", "maximum retry count when copy failed"); DSN_DEFINE_int32("nfs", rpc_timeout_ms, - 100000, + 1e5, // 100s "rpc timeout in milliseconds for nfs copy, " "0 means use default timeout of rpc engine"); diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index cd8d8f3cdb..d444fcb653 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -36,7 +36,6 @@ #include #include #include -#include #include "nfs_server_impl.h"