From 7dc39c513da08133836fb02f40ca3fec081a3581 Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Mon, 9 Nov 2020 18:32:57 +0800 Subject: [PATCH 1/6] use ddl_client --- src/server/hotspot_partition_calculator.h | 5 ++- src/server/info_collector.cpp | 51 ++++++++++++---------- src/server/info_collector.h | 2 +- src/server/test/hotspot_partition_test.cpp | 2 +- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/server/hotspot_partition_calculator.h b/src/server/hotspot_partition_calculator.h index 2cbc62bcaf..680c8b74d6 100644 --- a/src/server/hotspot_partition_calculator.h +++ b/src/server/hotspot_partition_calculator.h @@ -37,7 +37,9 @@ typedef std::vector> hot_partition_coun class hotspot_partition_calculator { public: - hotspot_partition_calculator(const std::string &app_name, int partition_count) + hotspot_partition_calculator(const std::string &app_name, + int partition_count, + std::shared_ptr context) : _app_name(app_name), _hot_points(partition_count), _hotpartition_counter(partition_count) { init_perf_counter(partition_count); @@ -65,6 +67,7 @@ class hotspot_partition_calculator hot_partition_counters _hot_points; // saving historical data can improve accuracy stat_histories _partitions_stat_histories; + std::shared_ptr _shell_context; // _hotpartition_counter p[index_of_partitions][type_of_read(0)/write(1)_stat] // it's a counter to find partitions that often exceed the threshold diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index d3cbe74578..825ba02fb2 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -57,9 +57,9 @@ info_collector::info_collector() _cluster_name = dsn::replication::get_current_cluster_name(); - _shell_context.current_cluster_name = _cluster_name; - _shell_context.meta_list = meta_servers; - _shell_context.ddl_client.reset(new replication_ddl_client(meta_servers)); + _shell_context->current_cluster_name = _cluster_name; + _shell_context->meta_list = meta_servers; + _shell_context->ddl_client.reset(new replication_ddl_client(meta_servers)); _app_stat_interval_seconds = (uint32_t)dsn_config_get_value_uint64("pegasus.collector", "app_stat_interval_seconds", @@ -112,13 +112,13 @@ info_collector::~info_collector() void info_collector::start() { - _app_stat_timer_task = - ::dsn::tasking::enqueue_timer(LPC_PEGASUS_APP_STAT_TIMER, - &_tracker, - [this] { on_app_stat(); }, - std::chrono::seconds(_app_stat_interval_seconds), - 0, - std::chrono::minutes(1)); + _app_stat_timer_task = ::dsn::tasking::enqueue_timer( + LPC_PEGASUS_APP_STAT_TIMER, + &_tracker, + [this] { on_app_stat(); }, + std::chrono::seconds(_app_stat_interval_seconds), + 0, + std::chrono::minutes(1)); _capacity_unit_stat_timer_task = ::dsn::tasking::enqueue_timer( LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, @@ -143,7 +143,7 @@ void info_collector::on_app_stat() { ddebug("start to stat apps"); std::map> all_rows; - if (!get_app_partition_stat(&_shell_context, all_rows)) { + if (!get_app_partition_stat(_shell_context.get(), all_rows)) { derror("call get_app_stat() failed"); return; } @@ -241,17 +241,18 @@ void info_collector::on_capacity_unit_stat(int remaining_retry_count) { ddebug("start to stat capacity unit, remaining_retry_count = %d", remaining_retry_count); std::vector nodes_stat; - if (!get_capacity_unit_stat(&_shell_context, nodes_stat)) { + if (!get_capacity_unit_stat(_shell_context.get(), nodes_stat)) { if (remaining_retry_count > 0) { dwarn("get capacity unit stat failed, remaining_retry_count = %d, " "wait %u seconds to retry", remaining_retry_count, _capacity_unit_retry_wait_seconds); - ::dsn::tasking::enqueue(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, - &_tracker, - [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_capacity_unit_retry_wait_seconds)); + ::dsn::tasking::enqueue( + LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, + &_tracker, + [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_capacity_unit_retry_wait_seconds)); } else { derror("get capacity unit stat failed, remaining_retry_count = 0, no retry anymore"); } @@ -288,17 +289,18 @@ void info_collector::on_storage_size_stat(int remaining_retry_count) { ddebug("start to stat storage size, remaining_retry_count = %d", remaining_retry_count); app_storage_size_stat st_stat; - if (!get_storage_size_stat(&_shell_context, st_stat)) { + if (!get_storage_size_stat(_shell_context.get(), st_stat)) { if (remaining_retry_count > 0) { dwarn("get storage size stat failed, remaining_retry_count = %d, " "wait %u seconds to retry", remaining_retry_count, _storage_size_retry_wait_seconds); - ::dsn::tasking::enqueue(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, - &_tracker, - [=] { on_storage_size_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_storage_size_retry_wait_seconds)); + ::dsn::tasking::enqueue( + LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, + &_tracker, + [=] { on_storage_size_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_storage_size_retry_wait_seconds)); } else { derror("get storage size stat failed, remaining_retry_count = 0, no retry anymore"); } @@ -316,7 +318,8 @@ info_collector::get_hotspot_calculator(const std::string &app_name, const int pa if (iter != _hotspot_calculator_store.end()) { return iter->second; } - auto calculator = std::make_shared(app_name, partition_count); + auto calculator = + std::make_shared(app_name, partition_count, _shell_context); _hotspot_calculator_store[app_name_pcount] = calculator; return calculator; } diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 12bebdda0d..a7afff20c1 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -173,7 +173,7 @@ class info_collector dsn::task_tracker _tracker; ::dsn::rpc_address _meta_servers; std::string _cluster_name; - shell_context _shell_context; + std::shared_ptr _shell_context; uint32_t _app_stat_interval_seconds; ::dsn::task_ptr _app_stat_timer_task; ::dsn::utils::ex_lock_nr _app_stat_counter_lock; diff --git a/src/server/test/hotspot_partition_test.cpp b/src/server/test/hotspot_partition_test.cpp index f41923ccaa..645fa8abf1 100644 --- a/src/server/test/hotspot_partition_test.cpp +++ b/src/server/test/hotspot_partition_test.cpp @@ -29,7 +29,7 @@ DSN_DECLARE_int32(occurrence_threshold); class hotspot_partition_test : public pegasus_server_test_base { public: - hotspot_partition_test() : calculator("TEST", 8) + hotspot_partition_test() : calculator("TEST", 8, nullptr) { dsn::fail::setup(); dsn::fail::cfg("send_detect_hotkey_request", "return()"); From 2e838c8e30af73f5122a8db39a74c1012ae107ae Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Mon, 9 Nov 2020 21:34:28 +0800 Subject: [PATCH 2/6] update --- src/server/hotspot_partition_calculator.cpp | 62 +++++++++------------ src/server/hotspot_partition_calculator.h | 8 +-- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index aea5a335d8..7fa45b7da3 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -17,16 +17,13 @@ #include "hotspot_partition_calculator.h" -#include #include #include +#include #include -#include -#include #include -#include -#include #include +#include namespace pegasus { namespace server { @@ -180,44 +177,39 @@ void hotspot_partition_calculator::detect_hotkey_in_hotpartition(int data_type) const dsn::replication::detect_action::type action) { FAIL_POINT_INJECT_F("send_detect_hotkey_request", [](dsn::string_view) {}); - auto request = dsn::make_unique(); - request->type = hotkey_type; - request->action = action; + + dsn::replication::detect_hotkey_request req; + req.type = hotkey_type; + req.action = action; + dsn::replication::detect_hotkey_response resp; + ddebug_f("{} {} hotkey detection in {}.{}", (action == dsn::replication::detect_action::STOP) ? "Stop" : "Start", (hotkey_type == dsn::replication::hotkey_type::WRITE) ? "write" : "read", app_name, partition_index); - dsn::rpc_address meta_server; - meta_server.assign_group("meta-servers"); - std::vector meta_servers; - replica_helper::load_meta_servers(meta_servers); - for (const auto &address : meta_servers) { - meta_server.group_address()->add(address); + + int app_id; + int partition_count; + std::vector partitions; + _shell_context->ddl_client->list_app(app_name, app_id, partition_count, partitions); + auto target_address = partitions[partition_index].primary; + + auto error = _shell_context->ddl_client->detect_hotkey(target_address, req, resp); + + if (error != dsn::ERR_OK) { + derror_f("Hotkey detect rpc sending failed, in {}.{}, error_hint:{}", + app_name, + partition_index, + error.to_string()); } - auto cluster_name = dsn::replication::get_current_cluster_name(); - // TODO: (Tangyanzhao) refactor partition_resolver to replication_ddl_client - auto resolver = partition_resolver::get_resolver(cluster_name, meta_servers, app_name.c_str()); - dsn::task_tracker tracker; - detect_hotkey_rpc rpc( - std::move(request), RPC_DETECT_HOTKEY, std::chrono::seconds(10), partition_index); - rpc.call(resolver, - &tracker, - [app_name, partition_index](dsn::error_code error) { - if (error != dsn::ERR_OK) { - derror_f("Hotkey detect rpc sending failed, in {}.{}, error_hint:{}", - app_name, - partition_index, - error.to_string()); - } - }) - ->wait(); - if (rpc.response().err != dsn::ERR_OK) { - derror_f("Hotkey detect rpc sending failed, in {}.{}, error_hint:{} {}", + + if (resp.err != dsn::ERR_OK) { + derror_f("Hotkey detect rpc executing failed, in {}.{}, error_hint:{} {}", app_name, partition_index, - rpc.response().err, - rpc.response().err_hint); + resp.err, + resp.err_hint); } } diff --git a/src/server/hotspot_partition_calculator.h b/src/server/hotspot_partition_calculator.h index 680c8b74d6..ddbbb15026 100644 --- a/src/server/hotspot_partition_calculator.h +++ b/src/server/hotspot_partition_calculator.h @@ -48,10 +48,10 @@ class hotspot_partition_calculator void data_aggregate(const std::vector &partitions); // analyse the saved data to find hotspot partition void data_analyse(); - static void send_detect_hotkey_request(const std::string &app_name, - const uint64_t partition_index, - const dsn::replication::hotkey_type::type hotkey_type, - const dsn::replication::detect_action::type action); + void send_detect_hotkey_request(const std::string &app_name, + const uint64_t partition_index, + const dsn::replication::hotkey_type::type hotkey_type, + const dsn::replication::detect_action::type action); private: // empirical rule to calculate hot point of each partition From f332d61ca9168c504502c89185d6616d91a2888b Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Tue, 10 Nov 2020 10:20:47 +0800 Subject: [PATCH 3/6] format --- src/server/info_collector.cpp | 36 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 825ba02fb2..92b088a764 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -112,13 +112,13 @@ info_collector::~info_collector() void info_collector::start() { - _app_stat_timer_task = ::dsn::tasking::enqueue_timer( - LPC_PEGASUS_APP_STAT_TIMER, - &_tracker, - [this] { on_app_stat(); }, - std::chrono::seconds(_app_stat_interval_seconds), - 0, - std::chrono::minutes(1)); + _app_stat_timer_task = + ::dsn::tasking::enqueue_timer(LPC_PEGASUS_APP_STAT_TIMER, + &_tracker, + [this] { on_app_stat(); }, + std::chrono::seconds(_app_stat_interval_seconds), + 0, + std::chrono::minutes(1)); _capacity_unit_stat_timer_task = ::dsn::tasking::enqueue_timer( LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, @@ -247,12 +247,11 @@ void info_collector::on_capacity_unit_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _capacity_unit_retry_wait_seconds); - ::dsn::tasking::enqueue( - LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, - &_tracker, - [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_capacity_unit_retry_wait_seconds)); + ::dsn::tasking::enqueue(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, + &_tracker, + [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_capacity_unit_retry_wait_seconds)); } else { derror("get capacity unit stat failed, remaining_retry_count = 0, no retry anymore"); } @@ -295,12 +294,11 @@ void info_collector::on_storage_size_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _storage_size_retry_wait_seconds); - ::dsn::tasking::enqueue( - LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, - &_tracker, - [=] { on_storage_size_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_storage_size_retry_wait_seconds)); + ::dsn::tasking::enqueue(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, + &_tracker, + [=] { on_storage_size_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_storage_size_retry_wait_seconds)); } else { derror("get storage size stat failed, remaining_retry_count = 0, no retry anymore"); } From b35e47d09b85c6b39f3c1ec409b51f3ab1172d41 Mon Sep 17 00:00:00 2001 From: Smilencer <527646889@qq.com> Date: Wed, 11 Nov 2020 16:48:24 +0800 Subject: [PATCH 4/6] Update src/server/hotspot_partition_calculator.cpp Co-authored-by: Yingchun Lai <405403881@qq.com> --- src/server/hotspot_partition_calculator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 7fa45b7da3..08c169e9f4 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -196,7 +196,6 @@ void hotspot_partition_calculator::detect_hotkey_in_hotpartition(int data_type) auto target_address = partitions[partition_index].primary; auto error = _shell_context->ddl_client->detect_hotkey(target_address, req, resp); - if (error != dsn::ERR_OK) { derror_f("Hotkey detect rpc sending failed, in {}.{}, error_hint:{}", app_name, From 7f301ced269a03d2981ffd403904ef116a581de4 Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Wed, 11 Nov 2020 17:11:53 +0800 Subject: [PATCH 5/6] update && init ptr --- src/server/hotspot_partition_calculator.cpp | 22 ++++++------ src/server/info_collector.cpp | 37 +++++++++++---------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 7fa45b7da3..ae7dc0344d 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -178,25 +178,25 @@ void hotspot_partition_calculator::detect_hotkey_in_hotpartition(int data_type) { FAIL_POINT_INJECT_F("send_detect_hotkey_request", [](dsn::string_view) {}); - dsn::replication::detect_hotkey_request req; - req.type = hotkey_type; - req.action = action; - dsn::replication::detect_hotkey_response resp; - - ddebug_f("{} {} hotkey detection in {}.{}", - (action == dsn::replication::detect_action::STOP) ? "Stop" : "Start", - (hotkey_type == dsn::replication::hotkey_type::WRITE) ? "write" : "read", - app_name, - partition_index); - int app_id; int partition_count; std::vector partitions; _shell_context->ddl_client->list_app(app_name, app_id, partition_count, partitions); auto target_address = partitions[partition_index].primary; + dsn::replication::detect_hotkey_response resp; + dsn::replication::detect_hotkey_request req; + req.type = hotkey_type; + req.action = action; auto error = _shell_context->ddl_client->detect_hotkey(target_address, req, resp); + ddebug_f("{} {} hotkey detection in {}.{}, server address: {}", + (action == dsn::replication::detect_action::STOP) ? "Stop" : "Start", + (hotkey_type == dsn::replication::hotkey_type::WRITE) ? "write" : "read", + app_name, + partition_index, + target_address.to_string()); + if (error != dsn::ERR_OK) { derror_f("Hotkey detect rpc sending failed, in {}.{}, error_hint:{}", app_name, diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 92b088a764..6e5bc30995 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -57,6 +57,7 @@ info_collector::info_collector() _cluster_name = dsn::replication::get_current_cluster_name(); + _shell_context = std::make_shared(); _shell_context->current_cluster_name = _cluster_name; _shell_context->meta_list = meta_servers; _shell_context->ddl_client.reset(new replication_ddl_client(meta_servers)); @@ -112,13 +113,13 @@ info_collector::~info_collector() void info_collector::start() { - _app_stat_timer_task = - ::dsn::tasking::enqueue_timer(LPC_PEGASUS_APP_STAT_TIMER, - &_tracker, - [this] { on_app_stat(); }, - std::chrono::seconds(_app_stat_interval_seconds), - 0, - std::chrono::minutes(1)); + _app_stat_timer_task = ::dsn::tasking::enqueue_timer( + LPC_PEGASUS_APP_STAT_TIMER, + &_tracker, + [this] { on_app_stat(); }, + std::chrono::seconds(_app_stat_interval_seconds), + 0, + std::chrono::minutes(1)); _capacity_unit_stat_timer_task = ::dsn::tasking::enqueue_timer( LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, @@ -247,11 +248,12 @@ void info_collector::on_capacity_unit_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _capacity_unit_retry_wait_seconds); - ::dsn::tasking::enqueue(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, - &_tracker, - [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_capacity_unit_retry_wait_seconds)); + ::dsn::tasking::enqueue( + LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, + &_tracker, + [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_capacity_unit_retry_wait_seconds)); } else { derror("get capacity unit stat failed, remaining_retry_count = 0, no retry anymore"); } @@ -294,11 +296,12 @@ void info_collector::on_storage_size_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _storage_size_retry_wait_seconds); - ::dsn::tasking::enqueue(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, - &_tracker, - [=] { on_storage_size_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_storage_size_retry_wait_seconds)); + ::dsn::tasking::enqueue( + LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, + &_tracker, + [=] { on_storage_size_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_storage_size_retry_wait_seconds)); } else { derror("get storage size stat failed, remaining_retry_count = 0, no retry anymore"); } From 1570c38f0f81caaadd85451be59f57f79c740994 Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Wed, 11 Nov 2020 17:16:25 +0800 Subject: [PATCH 6/6] format --- src/server/hotspot_partition_calculator.cpp | 2 +- src/server/info_collector.cpp | 36 ++++++++++----------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index ae7dc0344d..cc0142bdf6 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -182,8 +182,8 @@ void hotspot_partition_calculator::detect_hotkey_in_hotpartition(int data_type) int partition_count; std::vector partitions; _shell_context->ddl_client->list_app(app_name, app_id, partition_count, partitions); - auto target_address = partitions[partition_index].primary; + auto target_address = partitions[partition_index].primary; dsn::replication::detect_hotkey_response resp; dsn::replication::detect_hotkey_request req; req.type = hotkey_type; diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 6e5bc30995..2efabaed3a 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -113,13 +113,13 @@ info_collector::~info_collector() void info_collector::start() { - _app_stat_timer_task = ::dsn::tasking::enqueue_timer( - LPC_PEGASUS_APP_STAT_TIMER, - &_tracker, - [this] { on_app_stat(); }, - std::chrono::seconds(_app_stat_interval_seconds), - 0, - std::chrono::minutes(1)); + _app_stat_timer_task = + ::dsn::tasking::enqueue_timer(LPC_PEGASUS_APP_STAT_TIMER, + &_tracker, + [this] { on_app_stat(); }, + std::chrono::seconds(_app_stat_interval_seconds), + 0, + std::chrono::minutes(1)); _capacity_unit_stat_timer_task = ::dsn::tasking::enqueue_timer( LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, @@ -248,12 +248,11 @@ void info_collector::on_capacity_unit_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _capacity_unit_retry_wait_seconds); - ::dsn::tasking::enqueue( - LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, - &_tracker, - [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_capacity_unit_retry_wait_seconds)); + ::dsn::tasking::enqueue(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, + &_tracker, + [=] { on_capacity_unit_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_capacity_unit_retry_wait_seconds)); } else { derror("get capacity unit stat failed, remaining_retry_count = 0, no retry anymore"); } @@ -296,12 +295,11 @@ void info_collector::on_storage_size_stat(int remaining_retry_count) "wait %u seconds to retry", remaining_retry_count, _storage_size_retry_wait_seconds); - ::dsn::tasking::enqueue( - LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, - &_tracker, - [=] { on_storage_size_stat(remaining_retry_count - 1); }, - 0, - std::chrono::seconds(_storage_size_retry_wait_seconds)); + ::dsn::tasking::enqueue(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, + &_tracker, + [=] { on_storage_size_stat(remaining_retry_count - 1); }, + 0, + std::chrono::seconds(_storage_size_retry_wait_seconds)); } else { derror("get storage size stat failed, remaining_retry_count = 0, no retry anymore"); }