From 59e221712109ecc71a39835d31e2708ffa1a3d04 Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Mon, 11 Jan 2021 11:28:10 +0800 Subject: [PATCH 1/4] mutable parameter --- rdsn | 2 +- src/server/hotkey_collector.cpp | 3 +++ src/server/hotspot_partition_calculator.cpp | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rdsn b/rdsn index 5fe7ff16a5..ec34633e46 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 5fe7ff16a5ecd7656f04a8689637d27865678e33 +Subproject commit ec34633e4695e881209973460469b2b069c0f52e diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index 26ff59dee6..d3320ad9b4 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -33,12 +33,14 @@ DSN_DEFINE_uint32( hot_bucket_variance_threshold, 7, "the variance threshold to detect hot bucket during coarse analysis of hotkey detection"); +DSN_TAG_VARIABLE(hot_bucket_variance_threshold, FT_MUTABLE); DSN_DEFINE_uint32( "pegasus.server", hot_key_variance_threshold, 5, "the variance threshold to detect hot key during fine analysis of hotkey detection"); +DSN_TAG_VARIABLE(hot_key_variance_threshold, FT_MUTABLE); DSN_DEFINE_uint32("pegasus.server", hotkey_buckets_num, @@ -63,6 +65,7 @@ DSN_DEFINE_uint32( max_seconds_to_detect_hotkey, 150, "the max time (in seconds) allowed to capture hotkey, will stop if hotkey's not found"); +DSN_TAG_VARIABLE(max_seconds_to_detect_hotkey, FT_MUTABLE); // 68–95–99.7 rule, same algorithm as hotspot_partition_calculator::stat_histories_analyse /*extern*/ bool diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 3b7086b695..f488c58ad9 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -41,17 +41,20 @@ DSN_DEFINE_bool("pegasus.collector", enable_detect_hotkey, false, "auto detect hot key in the hot paritition"); +DSN_TAG_VARIABLE(enable_detect_hotkey, FT_MUTABLE); DSN_DEFINE_int32("pegasus.collector", hot_partition_threshold, 3, "threshold of hotspot partition value, if app.stat.hotspots >= " "FLAGS_hotpartition_threshold, this partition is a hot partition"); +DSN_TAG_VARIABLE(hot_partition_threshold, FT_MUTABLE); DSN_DEFINE_int32("pegasus.collector", occurrence_threshold, 3, "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); +DSN_TAG_VARIABLE(occurrence_threshold, FT_MUTABLE); void hotspot_partition_calculator::data_aggregate(const std::vector &partition_stats) { From f5d1a3bdb328e9f2eb3756064bde2687de996bea Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Mon, 11 Jan 2021 14:49:17 +0800 Subject: [PATCH 2/4] update --- src/server/hotkey_collector.cpp | 6 ++++++ src/server/hotspot_partition_calculator.cpp | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index d3320ad9b4..fda14b9646 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -34,6 +34,8 @@ DSN_DEFINE_uint32( 7, "the variance threshold to detect hot bucket during coarse analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_bucket_variance_threshold, FT_MUTABLE); +DSN_DEFINE_validator(hot_bucket_variance_threshold, + [](int32_t threshold) -> bool { return threshold >= 0 }); DSN_DEFINE_uint32( "pegasus.server", @@ -41,6 +43,8 @@ DSN_DEFINE_uint32( 5, "the variance threshold to detect hot key during fine analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_key_variance_threshold, FT_MUTABLE); +DSN_DEFINE_validator(hot_key_variance_threshold, + [](int32_t threshold) -> bool { return threshold >= 0 }); DSN_DEFINE_uint32("pegasus.server", hotkey_buckets_num, @@ -66,6 +70,8 @@ DSN_DEFINE_uint32( 150, "the max time (in seconds) allowed to capture hotkey, will stop if hotkey's not found"); DSN_TAG_VARIABLE(max_seconds_to_detect_hotkey, FT_MUTABLE); +DSN_DEFINE_validator(max_seconds_to_detect_hotkey, + [](int32_t seconds) -> bool { return seconds >= 0 }); // 68–95–99.7 rule, same algorithm as hotspot_partition_calculator::stat_histories_analyse /*extern*/ bool diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index f488c58ad9..f422808e18 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -49,12 +49,15 @@ DSN_DEFINE_int32("pegasus.collector", "threshold of hotspot partition value, if app.stat.hotspots >= " "FLAGS_hotpartition_threshold, this partition is a hot partition"); DSN_TAG_VARIABLE(hot_partition_threshold, FT_MUTABLE); +DSN_DEFINE_validator(hot_partition_threshold, + [](int32_t threshold) -> bool { return threshold >= 0 }); DSN_DEFINE_int32("pegasus.collector", occurrence_threshold, 3, "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); DSN_TAG_VARIABLE(occurrence_threshold, FT_MUTABLE); +DSN_DEFINE_validator(occurrence_threshold, [](int32_t threshold) -> bool { return threshold >= 0 }); void hotspot_partition_calculator::data_aggregate(const std::vector &partition_stats) { @@ -152,9 +155,11 @@ void hotspot_partition_calculator::data_analyse() void hotspot_partition_calculator::detect_hotkey_in_hotpartition(int data_type) { + auto now_hot_partition_threshold = FLAGS_hot_partition_threshold; + auto now_occurrence_threshold = FLAGS_occurrence_threshold; for (int index = 0; index < _hot_points.size(); index++) { - if (_hot_points[index][data_type].get()->get_value() >= FLAGS_hot_partition_threshold) { - if (++_hotpartition_counter[index][data_type] >= FLAGS_occurrence_threshold) { + if (_hot_points[index][data_type].get()->get_value() >= now_hot_partition_threshold) { + if (++_hotpartition_counter[index][data_type] >= now_occurrence_threshold) { derror_f("Find a {} hot partition {}.{}", (data_type == partition_qps_type::READ_HOTSPOT_DATA ? "read" : "write"), _app_name, From 56b475d73e5ec9b67ea6f7e139ca853fc47c8b3b Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Mon, 11 Jan 2021 15:03:46 +0800 Subject: [PATCH 3/4] update --- src/server/hotkey_collector.cpp | 6 +++--- src/server/hotspot_partition_calculator.cpp | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index fda14b9646..b06e2345be 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -35,7 +35,7 @@ DSN_DEFINE_uint32( "the variance threshold to detect hot bucket during coarse analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_bucket_variance_threshold, FT_MUTABLE); DSN_DEFINE_validator(hot_bucket_variance_threshold, - [](int32_t threshold) -> bool { return threshold >= 0 }); + [](int32_t threshold) -> bool { return threshold >= 0; }); DSN_DEFINE_uint32( "pegasus.server", @@ -44,7 +44,7 @@ DSN_DEFINE_uint32( "the variance threshold to detect hot key during fine analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_key_variance_threshold, FT_MUTABLE); DSN_DEFINE_validator(hot_key_variance_threshold, - [](int32_t threshold) -> bool { return threshold >= 0 }); + [](int32_t threshold) -> bool { return threshold >= 0; }); DSN_DEFINE_uint32("pegasus.server", hotkey_buckets_num, @@ -71,7 +71,7 @@ DSN_DEFINE_uint32( "the max time (in seconds) allowed to capture hotkey, will stop if hotkey's not found"); DSN_TAG_VARIABLE(max_seconds_to_detect_hotkey, FT_MUTABLE); DSN_DEFINE_validator(max_seconds_to_detect_hotkey, - [](int32_t seconds) -> bool { return seconds >= 0 }); + [](int32_t seconds) -> bool { return seconds >= 0; }); // 68–95–99.7 rule, same algorithm as hotspot_partition_calculator::stat_histories_analyse /*extern*/ bool diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index f422808e18..0a148f3952 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -50,14 +50,15 @@ DSN_DEFINE_int32("pegasus.collector", "FLAGS_hotpartition_threshold, this partition is a hot partition"); DSN_TAG_VARIABLE(hot_partition_threshold, FT_MUTABLE); DSN_DEFINE_validator(hot_partition_threshold, - [](int32_t threshold) -> bool { return threshold >= 0 }); + [](int32_t threshold) -> bool { return threshold >= 0; }); DSN_DEFINE_int32("pegasus.collector", occurrence_threshold, 3, "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); DSN_TAG_VARIABLE(occurrence_threshold, FT_MUTABLE); -DSN_DEFINE_validator(occurrence_threshold, [](int32_t threshold) -> bool { return threshold >= 0 }); +DSN_DEFINE_validator(occurrence_threshold, + [](int32_t threshold) -> bool { return threshold >= 0; }); void hotspot_partition_calculator::data_aggregate(const std::vector &partition_stats) { From 0d5aa52d87331131f7432200bda7158d1d4a733f Mon Sep 17 00:00:00 2001 From: tangyanzhao Date: Tue, 12 Jan 2021 12:10:14 +0800 Subject: [PATCH 4/4] delete validate --- src/server/hotkey_collector.cpp | 8 +------- src/server/hotspot_partition_calculator.cpp | 22 +++++++++------------ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index b06e2345be..d59214deef 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -34,8 +34,6 @@ DSN_DEFINE_uint32( 7, "the variance threshold to detect hot bucket during coarse analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_bucket_variance_threshold, FT_MUTABLE); -DSN_DEFINE_validator(hot_bucket_variance_threshold, - [](int32_t threshold) -> bool { return threshold >= 0; }); DSN_DEFINE_uint32( "pegasus.server", @@ -43,15 +41,13 @@ DSN_DEFINE_uint32( 5, "the variance threshold to detect hot key during fine analysis of hotkey detection"); DSN_TAG_VARIABLE(hot_key_variance_threshold, FT_MUTABLE); -DSN_DEFINE_validator(hot_key_variance_threshold, - [](int32_t threshold) -> bool { return threshold >= 0; }); DSN_DEFINE_uint32("pegasus.server", hotkey_buckets_num, 37, "the number of data capture hash buckets"); -DSN_DEFINE_validator(hotkey_buckets_num, [](int32_t bucket_num) -> bool { +DSN_DEFINE_validator(hotkey_buckets_num, [](uint32_t bucket_num) -> bool { if (bucket_num < 3) { return false; } @@ -70,8 +66,6 @@ DSN_DEFINE_uint32( 150, "the max time (in seconds) allowed to capture hotkey, will stop if hotkey's not found"); DSN_TAG_VARIABLE(max_seconds_to_detect_hotkey, FT_MUTABLE); -DSN_DEFINE_validator(max_seconds_to_detect_hotkey, - [](int32_t seconds) -> bool { return seconds >= 0; }); // 68–95–99.7 rule, same algorithm as hotspot_partition_calculator::stat_histories_analyse /*extern*/ bool diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 0a148f3952..68ad3dbc88 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -43,22 +43,18 @@ DSN_DEFINE_bool("pegasus.collector", "auto detect hot key in the hot paritition"); DSN_TAG_VARIABLE(enable_detect_hotkey, FT_MUTABLE); -DSN_DEFINE_int32("pegasus.collector", - hot_partition_threshold, - 3, - "threshold of hotspot partition value, if app.stat.hotspots >= " - "FLAGS_hotpartition_threshold, this partition is a hot partition"); +DSN_DEFINE_uint32("pegasus.collector", + hot_partition_threshold, + 3, + "threshold of hotspot partition value, if app.stat.hotspots >= " + "FLAGS_hotpartition_threshold, this partition is a hot partition"); DSN_TAG_VARIABLE(hot_partition_threshold, FT_MUTABLE); -DSN_DEFINE_validator(hot_partition_threshold, - [](int32_t threshold) -> bool { return threshold >= 0; }); -DSN_DEFINE_int32("pegasus.collector", - occurrence_threshold, - 3, - "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); +DSN_DEFINE_uint32("pegasus.collector", + occurrence_threshold, + 3, + "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); DSN_TAG_VARIABLE(occurrence_threshold, FT_MUTABLE); -DSN_DEFINE_validator(occurrence_threshold, - [](int32_t threshold) -> bool { return threshold >= 0; }); void hotspot_partition_calculator::data_aggregate(const std::vector &partition_stats) {