diff --git a/rdsn b/rdsn index 7ccb58ed9c..4be1128919 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 7ccb58ed9c6319f1a8ccb5fe28276a8a1014379a +Subproject commit 4be112891938ccea16bee2805a24861b2557726e diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index a4f4d76fcd..42be060281 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -149,16 +149,16 @@ void info_collector::on_app_stat() return; } - table_stats all_stats("_all_"); + row_data all_stats("_all_"); for (const auto &app_rows : all_rows) { // get statistics data for app - table_stats app_stats(app_rows.first); + row_data app_stats(app_rows.first); for (auto partition_row : app_rows.second) { app_stats.aggregate(partition_row); } - get_app_counters(app_stats.app_name)->set(app_stats); + get_app_counters(app_stats.row_name)->set(app_stats); // get row data statistics for all of the apps - all_stats.merge(app_stats); + all_stats.aggregate(app_stats); // hotspot_partition_calculator is used for detecting hotspots auto hotspot_partition_calculator = @@ -166,7 +166,7 @@ void info_collector::on_app_stat() hotspot_partition_calculator->data_aggregate(app_rows.second); hotspot_partition_calculator->data_analyse(); } - get_app_counters(all_stats.app_name)->set(all_stats); + get_app_counters(all_stats.row_name)->set(all_stats); ddebug_f("stat apps succeed, app_count = {}, total_read_qps = {}, total_write_qps = {}", all_rows.size(), diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 65312a0957..5f09aa4792 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -33,7 +33,6 @@ #include #include "../shell/commands.h" -#include "table_stats.h" namespace pegasus { namespace server { @@ -51,65 +50,58 @@ class info_collector return std::abs(total) < 1e-6 ? 0 : hit / total * 1e6; } - void set(const table_stats &row_stats) + void set(const row_data &row_stats) { - get_qps->set(row_stats.total_get_qps); - multi_get_qps->set(row_stats.total_multi_get_qps); - put_qps->set(row_stats.total_put_qps); - multi_put_qps->set(row_stats.total_multi_put_qps); - remove_qps->set(row_stats.total_remove_qps); - multi_remove_qps->set(row_stats.total_multi_remove_qps); - incr_qps->set(row_stats.total_incr_qps); - check_and_set_qps->set(row_stats.total_check_and_set_qps); - check_and_mutate_qps->set(row_stats.total_check_and_mutate_qps); - scan_qps->set(row_stats.total_scan_qps); - duplicate_qps->set(row_stats.total_duplicate_qps); - dup_shipped_ops->set(row_stats.total_dup_shipped_ops); - dup_failed_shipping_ops->set(row_stats.total_dup_failed_shipping_ops); - recent_read_cu->set(row_stats.total_recent_read_cu); - recent_write_cu->set(row_stats.total_recent_write_cu); - recent_expire_count->set(row_stats.total_recent_expire_count); - recent_filter_count->set(row_stats.total_recent_filter_count); - recent_abnormal_count->set(row_stats.total_recent_abnormal_count); - recent_write_throttling_delay_count->set( - row_stats.total_recent_write_throttling_delay_count); + get_qps->set(row_stats.get_qps); + multi_get_qps->set(row_stats.multi_get_qps); + put_qps->set(row_stats.put_qps); + multi_put_qps->set(row_stats.multi_put_qps); + remove_qps->set(row_stats.remove_qps); + multi_remove_qps->set(row_stats.multi_remove_qps); + incr_qps->set(row_stats.incr_qps); + check_and_set_qps->set(row_stats.check_and_set_qps); + check_and_mutate_qps->set(row_stats.check_and_mutate_qps); + scan_qps->set(row_stats.scan_qps); + duplicate_qps->set(row_stats.duplicate_qps); + dup_shipped_ops->set(row_stats.dup_shipped_ops); + dup_failed_shipping_ops->set(row_stats.dup_failed_shipping_ops); + recent_read_cu->set(row_stats.recent_read_cu); + recent_write_cu->set(row_stats.recent_write_cu); + recent_expire_count->set(row_stats.recent_expire_count); + recent_filter_count->set(row_stats.recent_filter_count); + recent_abnormal_count->set(row_stats.recent_abnormal_count); + recent_write_throttling_delay_count->set(row_stats.recent_write_throttling_delay_count); recent_write_throttling_reject_count->set( - row_stats.total_recent_write_throttling_reject_count); - recent_read_throttling_delay_count->set( - row_stats.total_recent_read_throttling_delay_count); - recent_read_throttling_reject_count->set( - row_stats.total_recent_read_throttling_reject_count); - storage_mb->set(row_stats.total_storage_mb); - storage_count->set(row_stats.total_storage_count); - rdb_block_cache_hit_rate->set( - convert_to_1M_ratio(row_stats.total_rdb_block_cache_hit_count, - row_stats.total_rdb_block_cache_total_count)); + row_stats.recent_write_throttling_reject_count); + recent_read_throttling_delay_count->set(row_stats.recent_read_throttling_delay_count); + recent_read_throttling_reject_count->set(row_stats.recent_read_throttling_reject_count); + storage_mb->set(row_stats.storage_mb); + storage_count->set(row_stats.storage_count); + rdb_block_cache_hit_rate->set(convert_to_1M_ratio( + row_stats.rdb_block_cache_hit_count, row_stats.rdb_block_cache_total_count)); rdb_index_and_filter_blocks_mem_usage->set( - row_stats.total_rdb_index_and_filter_blocks_mem_usage); - rdb_memtable_mem_usage->set(row_stats.total_rdb_memtable_mem_usage); - rdb_estimate_num_keys->set(row_stats.total_rdb_estimate_num_keys); - rdb_bf_seek_negatives_rate->set(convert_to_1M_ratio( - row_stats.total_rdb_bf_seek_negatives, row_stats.total_rdb_bf_seek_total)); - rdb_bf_point_negatives_rate->set( - convert_to_1M_ratio(row_stats.total_rdb_bf_point_negatives, - row_stats.total_rdb_bf_point_negatives + - row_stats.total_rdb_bf_point_positive_total)); - rdb_bf_point_false_positive_rate->set( - convert_to_1M_ratio(row_stats.total_rdb_bf_point_positive_total - - row_stats.total_rdb_bf_point_positive_true, - (row_stats.total_rdb_bf_point_positive_total - - row_stats.total_rdb_bf_point_positive_true) + - row_stats.total_rdb_bf_point_negatives)); + row_stats.rdb_index_and_filter_blocks_mem_usage); + rdb_memtable_mem_usage->set(row_stats.rdb_memtable_mem_usage); + rdb_estimate_num_keys->set(row_stats.rdb_estimate_num_keys); + rdb_bf_seek_negatives_rate->set( + convert_to_1M_ratio(row_stats.rdb_bf_seek_negatives, row_stats.rdb_bf_seek_total)); + rdb_bf_point_negatives_rate->set(convert_to_1M_ratio( + row_stats.rdb_bf_point_negatives, + row_stats.rdb_bf_point_negatives + row_stats.rdb_bf_point_positive_total)); + rdb_bf_point_false_positive_rate->set(convert_to_1M_ratio( + row_stats.rdb_bf_point_positive_total - row_stats.rdb_bf_point_positive_true, + (row_stats.rdb_bf_point_positive_total - row_stats.rdb_bf_point_positive_true) + + row_stats.rdb_bf_point_negatives)); read_qps->set(row_stats.get_total_read_qps()); write_qps->set(row_stats.get_total_write_qps()); - backup_request_qps->set(row_stats.total_backup_request_qps); - get_bytes->set(row_stats.total_get_bytes); - multi_get_bytes->set(row_stats.total_multi_get_bytes); - scan_bytes->set(row_stats.total_scan_bytes); - put_bytes->set(row_stats.total_put_bytes); - multi_put_bytes->set(row_stats.total_multi_put_bytes); - check_and_set_bytes->set(row_stats.total_check_and_set_bytes); - check_and_mutate_bytes->set(row_stats.total_check_and_mutate_bytes); + backup_request_qps->set(row_stats.backup_request_qps); + get_bytes->set(row_stats.get_bytes); + multi_get_bytes->set(row_stats.multi_get_bytes); + scan_bytes->set(row_stats.scan_bytes); + put_bytes->set(row_stats.put_bytes); + multi_put_bytes->set(row_stats.multi_put_bytes); + check_and_set_bytes->set(row_stats.check_and_set_bytes); + check_and_mutate_bytes->set(row_stats.check_and_mutate_bytes); read_bytes->set(row_stats.get_total_read_bytes()); write_bytes->set(row_stats.get_total_write_bytes()); } diff --git a/src/server/table_stats.h b/src/server/table_stats.h deleted file mode 100644 index c3bafcdc06..0000000000 --- a/src/server/table_stats.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -struct table_stats -{ - table_stats(const std::string &app_name) : app_name(app_name) {} - - double get_total_read_qps() const - { - return total_get_qps + total_multi_get_qps + total_scan_qps; - } - - double get_total_write_qps() const - { - return total_put_qps + total_multi_put_qps + total_remove_qps + total_multi_remove_qps + - total_incr_qps + total_check_and_set_qps + total_check_and_mutate_qps + - total_duplicate_qps; - } - - double get_total_read_bytes() const - { - return total_get_bytes + total_multi_get_bytes + total_scan_bytes; - } - - double get_total_write_bytes() const - { - return total_put_bytes + total_multi_put_bytes + total_check_and_set_bytes + - total_check_and_mutate_bytes; - } - - void aggregate(const row_data &row) - { - total_get_qps += row.get_qps; - total_multi_get_qps += row.multi_get_qps; - total_put_qps += row.put_qps; - total_multi_put_qps += row.multi_put_qps; - total_remove_qps += row.remove_qps; - total_multi_remove_qps += row.multi_remove_qps; - total_incr_qps += row.incr_qps; - total_check_and_set_qps += row.check_and_set_qps; - total_check_and_mutate_qps += row.check_and_mutate_qps; - total_scan_qps += row.scan_qps; - total_duplicate_qps += row.duplicate_qps; - total_dup_shipped_ops += row.dup_shipped_ops; - total_dup_failed_shipping_ops += row.dup_failed_shipping_ops; - total_recent_read_cu += row.recent_read_cu; - total_recent_write_cu += row.recent_write_cu; - total_recent_expire_count += row.recent_expire_count; - total_recent_filter_count += row.recent_filter_count; - total_recent_abnormal_count += row.recent_abnormal_count; - total_recent_write_throttling_delay_count += row.recent_write_throttling_delay_count; - total_recent_write_throttling_reject_count += row.recent_write_throttling_reject_count; - total_recent_read_throttling_delay_count += row.recent_read_throttling_delay_count; - total_recent_read_throttling_reject_count += row.recent_read_throttling_reject_count; - total_storage_mb += row.storage_mb; - total_storage_count += row.storage_count; - total_rdb_block_cache_hit_count += row.rdb_block_cache_hit_count; - total_rdb_block_cache_total_count += row.rdb_block_cache_total_count; - total_rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage; - total_rdb_memtable_mem_usage += row.rdb_memtable_mem_usage; - total_rdb_estimate_num_keys += row.rdb_estimate_num_keys; - total_rdb_bf_seek_negatives += row.rdb_bf_seek_negatives; - total_rdb_bf_seek_total += row.rdb_bf_seek_total; - total_rdb_bf_point_positive_true += row.rdb_bf_point_positive_true; - total_rdb_bf_point_positive_total += row.rdb_bf_point_positive_total; - total_rdb_bf_point_negatives += row.rdb_bf_point_negatives; - total_backup_request_qps += row.backup_request_qps; - total_get_bytes += row.get_bytes; - total_multi_get_bytes += row.multi_get_bytes; - total_scan_bytes += row.scan_bytes; - total_put_bytes += row.put_bytes; - total_multi_put_bytes += row.multi_put_bytes; - total_check_and_set_bytes += row.check_and_set_bytes; - total_check_and_mutate_bytes += row.check_and_mutate_bytes; - } - - void merge(const table_stats &row_stats) - { - total_get_qps += row_stats.total_get_qps; - total_multi_get_qps += row_stats.total_multi_get_qps; - total_put_qps += row_stats.total_put_qps; - total_multi_put_qps += row_stats.total_multi_put_qps; - total_remove_qps += row_stats.total_remove_qps; - total_multi_remove_qps += row_stats.total_multi_remove_qps; - total_incr_qps += row_stats.total_incr_qps; - total_check_and_set_qps += row_stats.total_check_and_set_qps; - total_check_and_mutate_qps += row_stats.total_check_and_mutate_qps; - total_scan_qps += row_stats.total_scan_qps; - total_duplicate_qps += row_stats.total_duplicate_qps; - total_dup_shipped_ops += row_stats.total_dup_shipped_ops; - total_dup_failed_shipping_ops += row_stats.total_dup_failed_shipping_ops; - total_recent_read_cu += row_stats.total_recent_read_cu; - total_recent_write_cu += row_stats.total_recent_write_cu; - total_recent_expire_count += row_stats.total_recent_expire_count; - total_recent_filter_count += row_stats.total_recent_filter_count; - total_recent_abnormal_count += row_stats.total_recent_abnormal_count; - total_recent_write_throttling_delay_count += - row_stats.total_recent_write_throttling_delay_count; - total_recent_write_throttling_reject_count += - row_stats.total_recent_write_throttling_reject_count; - total_recent_read_throttling_delay_count += - row_stats.total_recent_read_throttling_delay_count; - total_recent_read_throttling_reject_count += - row_stats.total_recent_read_throttling_reject_count; - total_storage_mb += row_stats.total_storage_mb; - total_storage_count += row_stats.total_storage_count; - total_rdb_block_cache_hit_count += row_stats.total_rdb_block_cache_hit_count; - total_rdb_block_cache_total_count += row_stats.total_rdb_block_cache_total_count; - total_rdb_index_and_filter_blocks_mem_usage += - row_stats.total_rdb_index_and_filter_blocks_mem_usage; - total_rdb_memtable_mem_usage += row_stats.total_rdb_memtable_mem_usage; - total_rdb_estimate_num_keys += row_stats.total_rdb_estimate_num_keys; - total_rdb_bf_seek_negatives += row_stats.total_rdb_bf_seek_negatives; - total_rdb_bf_seek_total += row_stats.total_rdb_bf_seek_total; - total_rdb_bf_point_positive_true += row_stats.total_rdb_bf_point_positive_true; - total_rdb_bf_point_positive_total += row_stats.total_rdb_bf_point_positive_total; - total_rdb_bf_point_negatives += row_stats.total_rdb_bf_point_negatives; - total_backup_request_qps += row_stats.total_backup_request_qps; - total_get_bytes += row_stats.total_get_bytes; - total_multi_get_bytes += row_stats.total_multi_get_bytes; - total_scan_bytes += row_stats.total_scan_bytes; - total_put_bytes += row_stats.total_put_bytes; - total_multi_put_bytes += row_stats.total_multi_put_bytes; - total_check_and_set_bytes += row_stats.total_check_and_set_bytes; - total_check_and_mutate_bytes += row_stats.total_check_and_mutate_bytes; - } - - std::string app_name; - double total_get_qps = 0; - double total_multi_get_qps = 0; - double total_put_qps = 0; - double total_multi_put_qps = 0; - double total_remove_qps = 0; - double total_multi_remove_qps = 0; - double total_incr_qps = 0; - double total_check_and_set_qps = 0; - double total_check_and_mutate_qps = 0; - double total_scan_qps = 0; - double total_duplicate_qps = 0; - double total_dup_shipped_ops = 0; - double total_dup_failed_shipping_ops = 0; - double total_recent_read_cu = 0; - double total_recent_write_cu = 0; - double total_recent_expire_count = 0; - double total_recent_filter_count = 0; - double total_recent_abnormal_count = 0; - double total_recent_write_throttling_delay_count = 0; - double total_recent_write_throttling_reject_count = 0; - double total_recent_read_throttling_delay_count = 0; - double total_recent_read_throttling_reject_count = 0; - double total_storage_mb = 0; - double total_storage_count = 0; - double total_rdb_block_cache_hit_count = 0; - double total_rdb_block_cache_total_count = 0; - double total_rdb_index_and_filter_blocks_mem_usage = 0; - double total_rdb_memtable_mem_usage = 0; - double total_rdb_estimate_num_keys = 0; - double total_rdb_bf_seek_negatives = 0; - double total_rdb_bf_seek_total = 0; - double total_rdb_bf_point_positive_true = 0; - double total_rdb_bf_point_positive_total = 0; - double total_rdb_bf_point_negatives = 0; - double total_backup_request_qps = 0; - double total_get_bytes = 0; - double total_multi_get_bytes = 0; - double total_scan_bytes = 0; - double total_put_bytes = 0; - double total_multi_put_bytes = 0; - double total_check_and_set_bytes = 0; - double total_check_and_mutate_bytes = 0; - double max_total_qps = 0; - double min_total_qps = INT_MAX; - double max_total_cu = 0; - double min_total_cu = INT_MAX; - std::string max_qps_partition_id; - std::string max_cu_partition_id; -}; diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 0b9f9590a7..4d468959e2 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -555,14 +555,8 @@ inline bool parse_app_perf_counter_name(const std::string &name, struct row_data { - double get_total_qps() const - { - return get_qps + multi_get_qps + scan_qps + put_qps + multi_put_qps + remove_qps + - multi_remove_qps + incr_qps + check_and_set_qps + check_and_mutate_qps + - duplicate_qps; - } - - double get_total_cu() const { return recent_read_cu + recent_write_cu; } + row_data() = default; + explicit row_data(const std::string &row_name) : row_name(row_name) {} double get_total_read_qps() const { return get_qps + multi_get_qps + scan_qps; } @@ -572,6 +566,59 @@ struct row_data check_and_mutate_qps; } + double get_total_read_bytes() const { return get_bytes + multi_get_bytes + scan_bytes; } + + double get_total_write_bytes() const + { + return put_bytes + multi_put_bytes + check_and_set_bytes + check_and_mutate_bytes; + } + + void aggregate(const row_data &row) + { + get_qps += row.get_qps; + multi_get_qps += row.multi_get_qps; + put_qps += row.put_qps; + multi_put_qps += row.multi_put_qps; + remove_qps += row.remove_qps; + multi_remove_qps += row.multi_remove_qps; + incr_qps += row.incr_qps; + check_and_set_qps += row.check_and_set_qps; + check_and_mutate_qps += row.check_and_mutate_qps; + scan_qps += row.scan_qps; + duplicate_qps += row.duplicate_qps; + dup_shipped_ops += row.dup_shipped_ops; + dup_failed_shipping_ops += row.dup_failed_shipping_ops; + recent_read_cu += row.recent_read_cu; + recent_write_cu += row.recent_write_cu; + recent_expire_count += row.recent_expire_count; + recent_filter_count += row.recent_filter_count; + recent_abnormal_count += row.recent_abnormal_count; + recent_write_throttling_delay_count += row.recent_write_throttling_delay_count; + recent_write_throttling_reject_count += row.recent_write_throttling_reject_count; + recent_read_throttling_delay_count += row.recent_read_throttling_delay_count; + recent_read_throttling_reject_count += row.recent_read_throttling_reject_count; + storage_mb += row.storage_mb; + storage_count += row.storage_count; + rdb_block_cache_hit_count += row.rdb_block_cache_hit_count; + rdb_block_cache_total_count += row.rdb_block_cache_total_count; + rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage; + rdb_memtable_mem_usage += row.rdb_memtable_mem_usage; + rdb_estimate_num_keys += row.rdb_estimate_num_keys; + rdb_bf_seek_negatives += row.rdb_bf_seek_negatives; + rdb_bf_seek_total += row.rdb_bf_seek_total; + rdb_bf_point_positive_true += row.rdb_bf_point_positive_true; + rdb_bf_point_positive_total += row.rdb_bf_point_positive_total; + rdb_bf_point_negatives += row.rdb_bf_point_negatives; + backup_request_qps += row.backup_request_qps; + get_bytes += row.get_bytes; + multi_get_bytes += row.multi_get_bytes; + scan_bytes += row.scan_bytes; + put_bytes += row.put_bytes; + multi_put_bytes += row.multi_put_bytes; + check_and_set_bytes += row.check_and_set_bytes; + check_and_mutate_bytes += row.check_and_mutate_bytes; + } + std::string row_name; int32_t app_id = 0; int32_t partition_count = 0;