diff --git a/src/shell/commands.h b/src/shell/commands.h index 3a7319c3b2..dc7efd68ad 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -47,6 +47,14 @@ struct list_nodes_helper int64_t mem_idx_bytes; int64_t disk_available_total_ratio; int64_t disk_available_min_ratio; + double get_qps; + double put_qps; + double multi_get_qps; + double multi_put_qps; + double get_p99; + double put_p99; + double multi_get_p99; + double multi_put_p99; list_nodes_helper(const std::string &n, const std::string &s) : node_name(n), node_status(s), @@ -57,7 +65,15 @@ struct list_nodes_helper mem_tbl_bytes(0), mem_idx_bytes(0), disk_available_total_ratio(0), - disk_available_min_ratio(0) + disk_available_min_ratio(0), + get_qps(0.0), + put_qps(0.0), + multi_get_qps(0.0), + multi_put_qps(0.0), + get_p99(0.0), + put_p99(0.0), + multi_get_p99(0.0), + multi_put_p99(0.0) { } }; diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index dace39d528..c6898b227e 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -48,6 +48,7 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) static struct option long_options[] = {{"detailed", no_argument, 0, 'd'}, {"resolve_ip", no_argument, 0, 'r'}, {"resource_usage", no_argument, 0, 'u'}, + {"qps", no_argument, 0, 'q'}, {"json", no_argument, 0, 'j'}, {"status", required_argument, 0, 's'}, {"output", required_argument, 0, 'o'}, @@ -58,12 +59,13 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) bool detailed = false; bool resolve_ip = false; bool resource_usage = false; + bool show_qps = false; bool json = false; optind = 0; while (true) { int option_index = 0; int c; - c = getopt_long(args.argc, args.argv, "drujs:o:", long_options, &option_index); + c = getopt_long(args.argc, args.argv, "druqjs:o:", long_options, &option_index); if (c == -1) break; switch (c) { @@ -76,6 +78,9 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) case 'u': resource_usage = true; break; + case 'q': + show_qps = true; + break; case 'j': json = true; break; @@ -227,6 +232,71 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) } } + if (show_qps) { + std::vector nodes; + if (!fill_nodes(sc, "replica-server", nodes)) { + std::cout << "get replica server node list failed" << std::endl; + return true; + } + + // TODO(heyuchen): add cu statistics + ::dsn::command command; + command.cmd = "perf-counters-by-postfix"; + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_GET.qps"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_PUT.qps"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_MULTI_GET.qps"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.qps"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_GET.latency.server"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_PUT.latency.server"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_MULTI_GET.latency.server"); + command.arguments.push_back("zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.latency.server"); + std::vector> results; + call_remote_command(sc, nodes, command, results); + + for (int i = 0; i < nodes.size(); ++i) { + dsn::rpc_address node_addr = nodes[i].address; + auto tmp_it = tmp_map.find(node_addr); + if (tmp_it == tmp_map.end()) + continue; + if (!results[i].first) { + std::cout << "query perf counter info from node " << node_addr.to_string() + << " failed" << std::endl; + return true; + } + dsn::perf_counter_info info; + dsn::blob bb(results[i].second.data(), 0, results[i].second.size()); + if (!dsn::json::json_forwarder::decode(bb, info)) { + std::cout << "decode perf counter info from node " << node_addr.to_string() + << " failed, result = " << results[i].second << std::endl; + return true; + } + if (info.result != "OK") { + std::cout << "query perf counter info from node " << node_addr.to_string() + << " returns error, error = " << info.result << std::endl; + return true; + } + list_nodes_helper &h = tmp_it->second; + for (dsn::perf_counter_metric &m : info.counters) { + if (m.name.find("RPC_RRDB_RRDB_GET.qps") != std::string::npos) + h.get_qps = m.value; + else if (m.name.find("RPC_RRDB_RRDB_PUT.qps") != std::string::npos) + h.put_qps = m.value; + else if (m.name.find("RPC_RRDB_RRDB_MULTI_GET.qps") != std::string::npos) + h.multi_get_qps = m.value; + else if (m.name.find("RPC_RRDB_RRDB_MULTI_PUT.qps") != std::string::npos) + h.put_qps = m.value; + else if (m.name.find("RPC_RRDB_RRDB_GET.latency.server") != std::string::npos) + h.get_p99 = m.value; + else if (m.name.find("RPC_RRDB_RRDB_PUT.latency.server") != std::string::npos) + h.put_p99 = m.value; + else if (m.name.find("RPC_RRDB_RRDB_MULTI_GET.latency.server") != std::string::npos) + h.multi_get_p99 = m.value; + else if (m.name.find("RPC_RRDB_RRDB_MULTI_PUT.latency.server") != std::string::npos) + h.multi_put_p99 = m.value; + } + } + } + // print configuration_list_nodes_response std::streambuf *buf; std::ofstream of; @@ -255,6 +325,16 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) tp.add_column("disk_avl_total_ratio", tp_alignment::kRight); tp.add_column("disk_avl_min_ratio", tp_alignment::kRight); } + if (show_qps) { + tp.add_column("get_qps", tp_alignment::kRight); + tp.add_column("get_p99(ms)", tp_alignment::kRight); + tp.add_column("mget_qps", tp_alignment::kRight); + tp.add_column("mget_p99(ms)", tp_alignment::kRight); + tp.add_column("put_qps", tp_alignment::kRight); + tp.add_column("put_p99(ms)", tp_alignment::kRight); + tp.add_column("mput_qps", tp_alignment::kRight); + tp.add_column("mput_p99(ms)", tp_alignment::kRight); + } for (auto &kv : tmp_map) { tp.add_row(kv.second.node_name); tp.append_data(kv.second.node_status); @@ -271,6 +351,16 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) tp.append_data(kv.second.disk_available_total_ratio); tp.append_data(kv.second.disk_available_min_ratio); } + if (show_qps) { + tp.append_data(kv.second.get_qps); + tp.append_data(kv.second.get_p99 / 1000000); + tp.append_data(kv.second.multi_get_qps); + tp.append_data(kv.second.multi_get_p99 / 1000000); + tp.append_data(kv.second.put_qps); + tp.append_data(kv.second.put_p99 / 1000000); + tp.append_data(kv.second.multi_put_qps); + tp.append_data(kv.second.multi_put_p99 / 1000000); + } } mtp.add(std::move(tp)); diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 46f0b9cd96..6ae2a78059 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -60,7 +60,7 @@ static command_executor commands[] = { "nodes", "get the node status for this cluster", "[-d|--detailed] [-j|--json] [-r|--resolve_ip] [-u|--resource_usage]" - "[-o|--output file_name] [-s|--status all|alive|unalive]", + "[-o|--output file_name] [-s|--status all|alive|unalive] [-q|--qps]", ls_nodes, }, {