From 16de4c9764bc92e38cbb678f8231c30f34b578e6 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 6 Sep 2022 13:21:42 -0400 Subject: [PATCH] Remove `pbench-run-benchmark` This allows us to remove our `ansible` or `ansible-core` dependency. --- agent/Makefile | 14 - .../bench-scripts/pbench-run-benchmark-sample | 248 ----- agent/bench-scripts/pbench-run-benchmark.pl | 431 --------- .../postprocess/fio-postprocess-cdm | 138 --- .../tests/pbench-run-benchmark/test-41.sts | 1 - .../tests/pbench-run-benchmark/test-41.txt | 14 - .../tests/pbench-run-benchmark/test-42.opts | 1 - .../tests/pbench-run-benchmark/test-42.txt | 13 - .../tests/pbench-run-benchmark/test-43.opts | 1 - .../tests/pbench-run-benchmark/test-43.sts | 1 - .../tests/pbench-run-benchmark/test-43.txt | 13 - .../tests/pbench-run-benchmark/test-44.opts | 1 - .../tests/pbench-run-benchmark/test-44.txt | 89 -- .../tests/pbench-run-benchmark/test-45.opts | 1 - .../tests/pbench-run-benchmark/test-45.txt | 69 -- .../tests/pbench-run-benchmark/test-46.opts | 1 - .../tests/pbench-run-benchmark/test-46.sts | 1 - .../tests/pbench-run-benchmark/test-46.txt | 14 - .../tests/pbench-run-benchmark/test-48.opts | 1 - .../tests/pbench-run-benchmark/test-48.sts | 1 - .../tests/pbench-run-benchmark/test-48.txt | 14 - agent/bench-scripts/unittests | 28 +- agent/config/benchmark/fio.json | 124 --- agent/config/benchmark/fio.json.example | 176 ---- agent/config/benchmark/pbench.json | 27 - agent/lib/PbenchAnsible.pm | 255 ------ agent/lib/PbenchCDM.pm | 401 --------- agent/lib/PbenchES.pm | 849 ------------------ agent/rpm/pbench-agent.spec.j2 | 8 - .../postprocess/iostat-postprocess | 117 +-- .../postprocess/iostat-postprocess-cdm | 1 - .../postprocess/mpstat-postprocess | 103 +-- .../postprocess/mpstat-postprocess-cdm | 1 - .../postprocess/pidstat-postprocess | 134 +-- .../postprocess/pidstat-postprocess-cdm | 1 - .../tool-scripts/postprocess/sar-postprocess | 581 +++--------- .../postprocess/sar-postprocess-cdm | 1 - agent/util-scripts/cdm-get-iterations | 91 -- .../util-scripts/pbench-get-iteration-metrics | 67 -- agent/util-scripts/pbench-get-metric-data | 73 -- agent/util-scripts/pbench-get-primary-metric | 37 - agent/util-scripts/pbench-import-cdm | 77 -- agent/util-scripts/unittests | 2 - 43 files changed, 228 insertions(+), 3993 deletions(-) delete mode 100755 agent/bench-scripts/pbench-run-benchmark-sample delete mode 100755 agent/bench-scripts/pbench-run-benchmark.pl delete mode 100755 agent/bench-scripts/postprocess/fio-postprocess-cdm delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-41.sts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-41.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-42.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-42.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-43.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-43.sts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-43.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-44.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-44.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-45.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-45.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-46.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-46.sts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-46.txt delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-48.opts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-48.sts delete mode 100644 agent/bench-scripts/tests/pbench-run-benchmark/test-48.txt delete mode 100644 agent/config/benchmark/fio.json delete mode 100644 agent/config/benchmark/fio.json.example delete mode 100644 agent/config/benchmark/pbench.json delete mode 100644 agent/lib/PbenchAnsible.pm delete mode 100644 agent/lib/PbenchCDM.pm delete mode 100644 agent/lib/PbenchES.pm delete mode 120000 agent/tool-scripts/postprocess/iostat-postprocess-cdm delete mode 120000 agent/tool-scripts/postprocess/mpstat-postprocess-cdm delete mode 120000 agent/tool-scripts/postprocess/pidstat-postprocess-cdm delete mode 120000 agent/tool-scripts/postprocess/sar-postprocess-cdm delete mode 100755 agent/util-scripts/cdm-get-iterations delete mode 100755 agent/util-scripts/pbench-get-iteration-metrics delete mode 100755 agent/util-scripts/pbench-get-metric-data delete mode 100755 agent/util-scripts/pbench-get-primary-metric delete mode 100755 agent/util-scripts/pbench-import-cdm diff --git a/agent/Makefile b/agent/Makefile index 5d8eb6f388..707e0a7fb0 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -21,24 +21,18 @@ INSTALLOPTS = --directory # all the scripts that's fit to install util-scripts = \ - cdm-get-iterations \ get-internal-tool \ pbench-add-metalog-option \ pbench-avg-stddev \ pbench-copy-results \ pbench-copy-result-tb \ pbench-display-sysinfo-options \ - pbench-get-iteration-metrics \ - pbench-get-metric-data \ - pbench-get-primary-metric \ - pbench-import-cdm \ pbench-kill-tools \ pbench-log-timestamp \ pbench-make-result-tb \ pbench-move-results \ pbench-output-monitor \ pbench-postprocess-tools \ - pbench-postprocess-tools-cdm \ pbench-register-tool \ pbench-register-tool-set \ pbench-send-tools \ @@ -75,9 +69,6 @@ bench-scripts = \ pbench-fio.md \ pbench-gen-iterations \ pbench-linpack \ - pbench-run-benchmark \ - pbench-run-benchmark.pl \ - pbench-run-benchmark-sample \ pbench-specjbb2005 \ pbench-uperf \ pbench-uperf.md \ @@ -88,7 +79,6 @@ bench-postprocess = \ BenchPostprocess.pm \ compare-bench-results \ fio-postprocess \ - fio-postprocess-cdm \ fio-postprocess-viz.py \ fio-prepare-jobfile \ generate-benchmark-summary \ @@ -195,20 +185,17 @@ tool-postprocess = \ docker-postprocess \ haproxy-ocp-postprocess \ iostat-postprocess \ - iostat-postprocess-cdm \ jmap-postprocess \ jstack-postprocess \ kvm-spinlock-postprocess \ kvmstat-postprocess \ kvmtrace-stop-postprocess \ mpstat-postprocess \ - mpstat-postprocess-cdm \ mpstat-stop-postprocess \ numastat-postprocess \ openvswitch-postprocess \ perf-stop-postprocess \ pidstat-postprocess \ - pidstat-postprocess-cdm \ proc-interrupts-postprocess \ proc-sched_debug-postprocess \ proc-vmstat-postprocess \ @@ -216,7 +203,6 @@ tool-postprocess = \ qemu-migrate-postprocess \ rabbit-postprocess \ sar-postprocess \ - sar-postprocess-cdm \ sysfs-postprocess \ virsh-migrate-postprocess \ vmstat-postprocess diff --git a/agent/bench-scripts/pbench-run-benchmark-sample b/agent/bench-scripts/pbench-run-benchmark-sample deleted file mode 100755 index 7dbc4d28b0..0000000000 --- a/agent/bench-scripts/pbench-run-benchmark-sample +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/perl -# -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl -# -# Author: Andrew Theurer -# -# This script: -# - is normally called from pbench-run-benchmark -# - executes a single sample of a benchmark -# - runs any postprocessing to get all data in CDM - -use strict; -use warnings; -use lib $ENV{'pbench_lib_dir'}; -use File::Temp; -use Data::Dumper; -use PbenchBase qw(get_json_file put_json_file get_benchmark_names get_clients - get_params get_pbench_bench_config_dir load_benchmark_config - get_pbench_install_dir ); -use PbenchAnsible qw(ssh_hosts ping_hosts copy_files_to_hosts copy_files_from_hosts - remove_files_from_hosts remove_dir_from_hosts create_dir_hosts - sync_dir_from_hosts verify_success yum_install_hosts); -use PbenchCDM qw(create_bench_iter_sample_doc); - -my $pbench_bench_config_dir = get_pbench_bench_config_dir; - -# main program starts here - -if (scalar @ARGV < 5) { - print "You must supply at least the /path/to/iteration-doc.json, results directory, " . - "base benchmark directory, and the tool group name (in that order):\n"; - exit; -} -my $iter_doc_filename = shift(@ARGV); -my $sample_dir = shift(@ARGV); # dir for *this* iteration-sample execution -my $base_dir = shift(@ARGV); # dir for all of the pbench run -my $tool_group = shift(@ARGV); -my $last_sample = shift(@ARGV); -my $pp_mode = shift(@ARGV); -my $pp_only = shift(@ARGV); -if (! -d $base_dir) { - die "The base bench directory, $base_dir must already exist"; -} -if (! $pp_only) { - mkdir($sample_dir); -} -my $json_ref = get_json_file($iter_doc_filename); -my %iteration = %$json_ref; # this is the CDM doc for this benchmark-iteration -my %sample = create_bench_iter_sample_doc(\%iteration); -my %specs = load_benchmark_config($pbench_bench_config_dir, $iteration{'run'}{'bench'}{'name'}); - -my $output_wrapper_name = "pbench-output-monitor"; -my $local_output_wrapper_path = get_pbench_install_dir . "/util-scripts"; -my $remote_output_wrapper_path = "/tmp"; - -my $sample_num = 0; -if ($sample_dir =~ /.+([0-9])+$/) { - $sample_num = $1; -} -$sample{'sample'}{'num'} = $sample_num; - -# build a hash of clients and servers to be used in almost all the steps bewlow -my %hosts; -for my $type ("client", "server") { - if (exists $iteration{'run'}{'bench'}{$type . 's'} and $iteration{'run'}{'bench'}{$type . 's'}) { - my @hosts = split(/ /, $iteration{'run'}{'bench'}{$type . 's'}); - $hosts{$type} = \@hosts; - } -} - -# all benchmarks must use --clients with at least 1 host -if (! exists $hosts{"client"}) { - print "There are no clients, exiting\n"; - exit 1; -} - -if (! $pp_only) { - # pre-benchmark-execution: - # - verify access to clients and servers by attempting to create the directories - for my $type (keys %hosts) { - if (scalar @{ $hosts{$type} }) { - if ($specs{$type} and ${specs}{$type}{"copy-to"}) { - my $result; - my $rc; - my @files = @{ ${specs}{$type}{"copy-to"} }; - # the end of the dir name gets "-client" or "-server" just in case this host is - # also a controller or both client and server - $result = create_dir_hosts(\@{ $hosts{$type} }, $sample_dir . "-" . $type, - $base_dir); - } - } - } - # - verify required software on clients and server (todo) - for my $type (keys %hosts) { - if (scalar @{ $hosts{$type} }) { - if ($specs{$type} and ${specs}{$type}{"packages"}) { - my $result; - my $rc; - my @packages = @{ ${specs}{$type}{"packages"} }; - $result = yum_install_hosts(\@{ $hosts{$type} }, \@packages, - $base_dir); - } - } - } - # - run pre-script in controller - if (exists $specs{"controller"}{"pre-script"} and $specs{"controller"}{"pre-script"} ne "") { - # pre-script gets same params as benchmark. - my $preprocess_cmd = $specs{"controller"}{"pre-script"} . " " . $iteration{'iteration'}{'params'}; - my $preprocess_output = `$preprocess_cmd 2>&1`; - my $rc = $? >> 8; - if (open(my $fh, ">" . $sample_dir . "/preprocess-output.txt")) { - printf { $fh } "%s\n\n", $preprocess_output; - printf { $fh } "rc=%d\n", $rc; - close($fh); - } - if ($rc != 0) { - printf "Controller pre-script failed, exiting [rc=$rc]\n"; - exit 1; - } - } - # - copy files to clients and servers - for my $type (keys %hosts) { - if (scalar @{ $hosts{$type} }) { - if ($specs{$type} and ${specs}{$type}{"copy-to"}) { - my $result; - my $rc; - my @files = @{ ${specs}{$type}{"copy-to"} }; - # the end of the dir name gets "-client" or "-server" just in case this host is - # also a controller or both client and server - $result = copy_files_to_hosts(\@{ $hosts{$type} }, \@files, $sample_dir . "-" . - $type, $base_dir); - } - if ($specs{$type} and ${specs}{$type}{"bin"}) { - # since there is an executable that is going to be used lets copy over the output - # wrapper in advance - my $result; - my @files = ( $local_output_wrapper_path ."/" . $output_wrapper_name ); - $result = copy_files_to_hosts(\@{ $hosts{$type} }, \@files, $remote_output_wrapper_path, $base_dir); - } - } - } - # - execute pre-benchmark scripts on clients and servers (todo) - - # start the tools: - print "starting tools\n"; - system("pbench-start-tools --group=" . $tool_group . " --dir=" . $sample_dir) == 0 - or die "Failed to start tools: $?"; - - # benchmark-execution: - print "starting benchmark\n"; - for my $type (reverse sort keys %hosts) { # reverse so "server" is first - if ($hosts{$type} and scalar @{ $hosts{$type} }) { - my $cmd; - if ($specs{$type}{"bin"}) { - $cmd = $remote_output_wrapper_path . "/" . $output_wrapper_name. " " . - $specs{$type}{"bin"} . " " . $iteration{'iteration'}{'params'}; - } else { - printf "%s->bin does not exist, exiting\n", $type; - exit 1; - } - # Apply a regex from the benchmark config file to the command - # This is used to remove things like "--clients=" because the - # native benchmark does not understand this parameter - if ($specs{$type} and $specs{$type}{"param_regex"}) { - for my $r (@{ $specs{$type}{"param_regex"} }) { - # to apply the 's/x/y/' regex from the file, some eval trickery is necessary - # todo: first test the $r regex separately for [perl syntax] errors with eval - # before doing below - $cmd = eval "\$_='$cmd'; $r; return scalar \$_"; - } - } - my $result = ssh_hosts(\@{ $hosts{$type} }, $cmd, $sample_dir . "-" . $type, $base_dir); - } - } - - # stop the tools: - print "stopping tools\n"; - system("pbench-stop-tools --group=" . $tool_group . " --dir=" . $sample_dir) == 0 - or die "Failed to stop tools: $?"; - - # send the tools: - print "sending tools\n"; - system("pbench-send-tools --group=" . $tool_group . " --dir=" . $sample_dir) == 0 - or die "Failed to send tools: $?"; - - # post-process the tools: - print "post-processing tools\n"; - system("pbench-postprocess-tools --group=" . $tool_group . " --dir=" . $sample_dir) == 0 - or die "Failed to post-process tools: $?"; - - # post-benchmark-execution: - # - copy files back to controller - # FIXME: with tool meister, all data is moved locally by the tool-data-sink - print "copying benchmark data\n"; - for my $type (keys %hosts) { - if ($hosts{$type} and scalar @{ $hosts{$type} }) { - mkdir($sample_dir . "/" . $type . "s"); - my $sync_result = sync_dir_from_hosts(\@{ $hosts{$type} }, - $sample_dir . "-" . $type . "/", #copy-from - $sample_dir . "/" . $type . "s", #copy-to - $base_dir); - # adding the "/" ensures the base dir - # name is not copied - my $remove_result = remove_dir_from_hosts(\@{ $hosts{$type} }, - $sample_dir . "-" . $type, - $base_dir); - } - } - # - post-process on the client/server, if applicable (todo) -} # if (! $pp_only) - -# - post-process the result data on the controller -if (exists $specs{"controller"}{"post-script"} and $specs{"controller"}{"post-script"} ne "") { - print "post-processing\n"; - # A couple of things about the benchmark post-procesing script: - # (1) It is responsible for completing and writing the sample document and creating all period - # and metric documents. It requires a partially completed sample document, that is done below, - # to the $sample_dir (./iterationX/sampleY). - # (2) Because tool data goes into metric documents, and metric documents generally should be - # inherting data from a period document, and those period documents get created by the - # benchmark post-processing script, it is up to that benchmark post-processing script to call - # pbench-postprocess-tools-cdm and passing the period doc as one of the paramters. - put_json_file(\%sample, $sample_dir . "/sample" . ".json"); - my $postprocess_cmd = ". /opt/pbench-agent/base; cd \"" . $sample_dir . "\"; " . - $specs{"controller"}{"post-script"} . " " . $sample_dir . " " . - $base_dir . " " . $tool_group . " " .$last_sample . - " " . $pp_mode . " 2>&1"; - my $pp_cmd_file = $sample_dir . "/postprocess.cmd"; - open(my $fh_cmd, ">" . $pp_cmd_file); - printf { $fh_cmd } "%s\n", $postprocess_cmd; - close($fh_cmd); - my $postprocess_output = `. $pp_cmd_file 2>&1`; - my $rc = $? >> 8; - if (open(my $fh_output, ">" . $sample_dir . "/postprocess-output.txt")) { - printf { $fh_output } "%s\n\n", $postprocess_output; - printf { $fh_output } "rc=%d\n", $rc; - close($fh_output); - } - if ($rc != 0) { - printf "Controller post-script failed, exiting [rc=$rc]\n"; - exit 1; - } -} else { - # If there is no post-processor for the benchmark (there really should be!) then this - # script will write the saple CDM doc. This document is not really complete. - put_json_file(\%sample, $base_dir . "/es/bench/sample-" . $sample{'sample'}{'id'} . ".json"); - print "warning: there is no post-script for this benchmark\n"; -} diff --git a/agent/bench-scripts/pbench-run-benchmark.pl b/agent/bench-scripts/pbench-run-benchmark.pl deleted file mode 100755 index e2f257b2a2..0000000000 --- a/agent/bench-scripts/pbench-run-benchmark.pl +++ /dev/null @@ -1,431 +0,0 @@ -#!/usr/bin/perl -# -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl -# -# Author: Andrew Theurer -# -# This is a wrapper script that will run a benchmark for the user by doing the following: -# - validating the benchmark exists -# - validating the benchmark parameters and pbench parameters (via pbench-gen-iterations) -# - constructing a list of benchmark-iterations (via pbench-gen-iterations) -# - executing those iterations, with N sample-executions per iteration (via pbench-run-iteration) -# - run any post-processing for those executions -# - bundle all the data in a JSON document - -use strict; -use warnings; -use File::Basename; -my $pbench_install_dir = $ENV{'pbench_install_dir'}; -use lib $ENV{'pbench_lib_dir'}; -use File::Path qw(remove_tree); -use PbenchCDM qw(create_run_doc create_config_osrelease_doc create_config_cpuinfo_doc - create_config_netdevs_doc create_config_ethtool_doc create_config_base_doc - get_uuid create_bench_iter_doc create_config_doc); -use PbenchBase qw(get_json_file put_json_file get_benchmark_names get_pbench_run_dir - get_pbench_install_dir get_pbench_config_dir get_pbench_bench_config_dir - get_benchmark_results_dir get_params remove_params get_hostname - begin_run end_run interrupt_run metadata_log_record_iteration - get_pbench_datetime); -use PbenchAnsible qw(ssh_hosts ping_hosts copy_files_to_hosts copy_files_from_hosts - remove_files_from_hosts remove_dir_from_hosts create_dir_hosts - sync_dir_from_hosts verify_success); - -my %defaults = ( - "num-samples" => 1, - "tool-group" => "default", - "sysinfo" => "default", - "postprocess-mode" => "html" # other mode is "cdm" -); - -my $pp_only = 0; -my $base_bench_dir = ""; - -print "WARNING: pbench-run-benchmark is deprecated and will be removed in the next release\n"; - -my @benchmarks = get_benchmark_names(get_pbench_bench_config_dir); -# The only required [positional] argument is the benchmark name; verify it now -if (scalar @ARGV == 0) { - print "You must supply at least one of the following supported benchmark names:\n"; - printf "\t%s\n", join(" ", @benchmarks); - exit 1; -} -my $benchmark = shift(@ARGV); -if ($benchmark eq "list") { - printf "%s\n", join(" ", @benchmarks); - exit; -} -my %benchmarks_hash = map { $_ => 1 } @benchmarks; -if (! exists($benchmarks_hash{$benchmark})) { - print "Unsupported benchmark " . $benchmark . "; please supply one of the following supported benchmark names:\n"; - printf "\t%s\n", join(" ", @benchmarks); - exit 1; -} - -# The rest of the parameters are --arg=val, most of which we just pass to other scripts, -my %params = get_params(@ARGV); - -# determine the common parameters from a group of iterations -sub find_common_parameters { - my @iterations = @_; - - my %common_parameters; - my $counter = 1; - for my $iteration_params (@iterations) { - $iteration_params =~ s/^\s+(.+)\s+$/$1/; - - my @split_params = split(/\s+/, $iteration_params); - - # count the number of occurences of each parameter key=value - # pair - for my $param_piece (@split_params) { - if (! exists($common_parameters{$param_piece})) { - $common_parameters{$param_piece} = 1; - } else { - $common_parameters{$param_piece}++; - } - } - - $counter++; - } - - # decrement by one since we are post decrementing in the loop - $counter--; - - for my $key (keys %common_parameters) { - # if the number of iterations is the same as the count for the - # key=value pair then the key=value pair is common across all - # iterations -- so remove key=value pairs where the count is - # not the same - if ($common_parameters{$key} != $counter) { - delete $common_parameters{$key}; - } - } - - return(%common_parameters); -} - -# Prepare for a post-process only mode if detected -if (exists $params{"postprocess-only"} and $params{"postprocess-only"} eq "y") { - $pp_only = 1; - printf "found postprocess-only\n"; - delete $params{"postprocess-only"}; - if (exists $params{"postprocess-dir"}) { - $base_bench_dir = $params{"postprocess-dir"}; - delete $params{"postprocess-dir"}; - } else { - # Assume already in the base benchmark directory - $base_bench_dir = `/bin/pwd`; - chomp $base_bench_dir; - } - printf "base_bench_dir: %s\n", $base_bench_dir; -} - -# Every benchmark must have at least 1 client, even if the client is the same host as the controller -if (! exists $params{'clients'}) { - print "You must specify at least 1 client with --clients\n"; - exit 1; -} - -# Warn if a few optional but highly recommended params are missing (can also be defined with env vars) -for my $param (qw(user-name user-email user-desc user-tags)) { - my $env_var = uc $param; - $env_var =~ s/-/_/g; - if (! exists $params{$param}) { # if --arg_name was used, $ARG_NAME will not be used - if (! exists $ENV{$env_var}) { - printf "\n***** it is highly recommended you do one of the following:\n" . - "- export this variable before calling this script %s\n" . - "- use this parameter when calling this script: --%s\n\n", $env_var, $param; - $params{$param} = ""; - } else { - $params{$param} = $ENV{$env_var}; - } - } -} - -# Apply the Pbench defaults (not the benchmark defaults) -for my $def (keys %defaults) { - if (! exists $params{$def}) { - $params{$def} = $defaults{$def}; - } -} - -my $date = get_pbench_datetime(); - -# Prepare all the dirs for the run -if ($pp_only) { - if (! -e $base_bench_dir) { - die "Expected the base benchmark directory, [$base_bench_dir], to already exist, since this is a post-process only mode"; - } -} else { - $base_bench_dir = get_benchmark_results_dir($benchmark, $params{"user-tags"}, $date); - # Spaces in directories, just don't do it - $base_bench_dir =~ s/\s+/_/g; - mkdir("$base_bench_dir"); - # Ensure base dir is exported in environment (required by TM) - $ENV{"benchmark_run_dir"} = $base_bench_dir; - - # Document the params used for this invocation so one can re-run, and then they can add - # "--postprocess-only=y --base-bench_dir=$base_bench_dir" if they wish to not run but - # only postprocess. - my $cmdfile = $base_bench_dir . "/pbench-run-benchmark.cmd"; - printf "cmdfile: %s\n", $cmdfile; - open(my $cmd_fh, ">" . $cmdfile) || - die "Could not create $cmdfile"; - printf $cmd_fh "pbench-run-benchmark %s %s\n", $benchmark, join(" ", @ARGV); -} -my $es_dir = $base_bench_dir . "/es"; -if ($pp_only) { - # Do not re-use any existing CDM docs - remove_tree($es_dir); -} -mkdir($es_dir); -for my $es_subdir (qw(run bench config metrics)) { - mkdir($es_dir . "/" . $es_subdir); -} - -my $tool_group = $params{"tool-group"}; -my $iteration_id = 0; -my $run_id = get_uuid; -my %last_run_doc; -print "Generating all benchmark iterations\n"; -# We don't want these params passed to gen-iterations because they are not benchmark-native options -remove_params(\@ARGV, qw(postprocess-only user-desc user-tags user-name user-email pre-sample-cmd postprocess-mode postprocess-dir)); -# First call pbench-gen-iterations and only resolve any --defaults=, so that the run -# doc has the full set of params -my $get_defs_cmd = "pbench-gen-iterations " . $benchmark . " --defaults-only " . join(" ", @ARGV); -# Escape any quotes so they don't get lost before calling pbench-gen-iterations -$get_defs_cmd =~ s/\"/\\\"/g; -print "resolving default first with $get_defs_cmd\n"; -my @param_sets = `$get_defs_cmd 2>&1`; -my $rc = $? >> 8; -if ($rc != 0) { - printf "%s\nCalling pbench-gen-iterations failed, exiting [rc=%d]\n", join(" ", @param_sets), $rc; - exit 1; -} -my @iteration_names; -my %param_sets_common_params = find_common_parameters(@param_sets); - -my $iterations_and_params_fh; -my $iterations_fh; - -if (! $pp_only) { - mkdir($base_bench_dir); - open($iterations_and_params_fh, ">" . $base_bench_dir . "/iteration-list.txt"); - open($iterations_fh, ">" . $base_bench_dir . "/.iterations"); - - sub signal_handler { - interrupt_run($tool_group); - } - - $SIG{INT} = "signal_handler"; - $SIG{QUIT} = "signal_handler"; - $SIG{TERM} = "signal_handler"; - - # start tool meister ensuring benchmark, config, and date ENVs are presented so - # that they are recorded by the Tool Data Sink in the metadata.log file. - begin_run($base_bench_dir, $benchmark, $params{"user-tags"}, $date, $params{"sysinfo"}, $tool_group); -} - -# There can be multiple parts of a run if the user generated multiple parameter-sets -# (used a "--" in their cmdline). Each part of the run has it's own run document, -# but all of the run documents share the same run ID. -my $run_part = 0; -while (scalar @param_sets > 0) { - my $nr_pp_jobs = 0; - my $num_samples; - my $param_set = shift(@param_sets); - chomp $param_set; - if ($param_set =~ /^#/) { - printf "%s\n", $param_set; - next; - } - # Process --samples= here, since it can be different for each parameter-set, - # and we need this below for running pbench-run-benchmark-sample N times - if ($param_set =~ s/--samples=(\S+)\s*//) { - $num_samples = $1; - } else { - $num_samples = $defaults{'num-samples'}; - } - my $get_iters_cmd = "pbench-gen-iterations " . $benchmark . " " . $param_set; - # Escape quotes again as this will get passed to pbench-gen-iterations again. - # We don't escape the quotes when writing to the run document (with $param_set) - # because the conversion to JSON will do it for us. - $get_iters_cmd =~ s/\"/\\\"/g; - my @iterations = `$get_iters_cmd 2>&1`; - $rc = $? >> 8; - if ($rc != 0) { - printf "%s\nCalling pbench-gen-iterations failed, exiting [rc=%d]\n", $get_iters_cmd, $rc; - exit 1; - } - my %run_doc = create_run_doc($benchmark, $param_set, $params{"clients"}, $params{"servers"}, - $params{"user-desc"}, $params{"user-tags"}, $params{"user-name"}, $params{"user-email"}, - "pbench", ""); #todo: include tool names - $run_doc{'run'}{'id'} = $run_id; # use the same run ID for all run docs - # Now run the iterations for this parameter-set - if ($pp_only) { - # Prepare a bash script to process all iterations & samples at once - # This provides for a *much* fatser post-processing on server-side - open(BULK_FH, ">bulk-sample.sh"); - print BULK_FH "#!/bin/bash\n"; - } - - my %iterations_common_params = find_common_parameters(@iterations); - - my @iterations_labels; - - for my $iteration_params (@iterations) { - $iteration_params =~ s/^\s+(.+)\s+$/$1/; - - my @split_params = split(/\s+/, $iteration_params); - - # create a label for the iteration, which becomes part of the - # iteration name. the label should be comprised of the - # parameters that are not common across all iterations - my $iteration_label = ""; - for my $param_piece (@split_params) { - # if the param_piece does not exist in either of the - # common parameters hashes then it should become part of - # the iteration's label - if (! exists($iterations_common_params{$param_piece}) || - ! exists($param_sets_common_params{$param_piece})) { - $iteration_label .= $param_piece . " "; - } - } - - # strip off the -- from each parameter, remove beginning and - # ending spaces, and convert spaces to underscores - $iteration_label =~ s/\s*--/ /g; - $iteration_label =~ s/\s+$//; - $iteration_label =~ s/^\s+//; - $iteration_label =~ s/\s/_/g; - $iteration_label =~ s/\//:/g; - - # if the logic for determining a label for the iteration has - # yielded an empty string just use the benchmark name -- this - # is likely because there is a single iteration so all - # parameters are common - if (length($iteration_label) == 0) { - $iteration_label = $benchmark; - } - - push(@iterations_labels, $iteration_label); - } - - for (my $index=0; $index<@iterations; $index++) { - my $iteration_params = $iterations[$index]; - my $iteration_label = $iterations_labels[$index]; - - chomp $iteration_params; - if ($iteration_params =~ /^#/) { - printf "%s\n", $param_set; - next; - } - my $iteration_name = $iteration_id . "__" . $iteration_label; - push(@iteration_names, $iteration_name); - my %iter_doc = create_bench_iter_doc(\%run_doc, $iteration_params,); - if (! $pp_only) { - put_json_file(\%iter_doc, $es_dir . "/bench/iteration-" . $iter_doc{'iteration'}{'id'} . ".json"); - metadata_log_record_iteration($base_bench_dir, $iteration_id, $iteration_params, $iteration_name); - printf $iterations_fh "%d\n", $iteration_id; - printf $iterations_and_params_fh "%d %s\n", $iteration_id, $iteration_params; - } - printf "\n\n\niteration_ID: %d\niteration_params: %s\n", $iteration_id, $iteration_params; - my $iteration_dir = $base_bench_dir . "/" . $iteration_name; - my @sample_dirs; - if ($pp_only) { - opendir(my $iter_dh, $iteration_dir) || die "Can't opendir $iteration_dir: $!"; - @sample_dirs = grep { /^sample\d+/ } readdir($iter_dh); - } else { - mkdir($iteration_dir); - for (my $sample_id=0; $sample_id<$num_samples; $sample_id++) { - $sample_dirs[$sample_id] = "sample" . $sample_id; - } - } - while (scalar @sample_dirs > 0) { - if (exists $params{'pre-sample-cmd'}) { - my $pre_sample_cmd_output = `$params{'pre-sample-cmd'} 2>&1`; - my $exit_code = $? >> 8; - print "pre-sample-cmd output:\n$pre_sample_cmd_output"; - if ($exit_code != 0) { - printf "Stopping because of pre-sample-cmd exit code: %d\n", $exit_code; - exit 1; - } - } - my $sample_dir = shift(@sample_dirs); - print "$sample_dir\n"; - my $iteration_sample_dir = $iteration_dir . "/" . $sample_dir; - if (! $pp_only) { - mkdir($iteration_sample_dir); - } - my $last_sample = "0"; - if (scalar @sample_dirs == 0) { - $last_sample = "1"; - } - my $benchmark_cmd = "pbench-run-benchmark-sample " . $es_dir . "/bench/iteration-" . - $iter_doc{'iteration'}{'id'} . ".json " . $iteration_sample_dir . - " " . $base_bench_dir . " " . $tool_group . - " " . $last_sample . " " . $params{"postprocess-mode"} . - " " . $pp_only; - if ($pp_only) { - if ($params{'postprocess-mode'} eq 'html') { - # the last sample must be run by itself after all - # other samples are processed; other samples can - # be run in parallel - if ($last_sample eq "0") { - print BULK_FH "$benchmark_cmd &\n"; - } elsif ($last_sample eq "1") { - print BULK_FH "echo Waiting for $nr_pp_jobs post-processing jobs to finish\n"; - print BULK_FH "wait\n"; - print BULK_FH "echo Post-processing last sample\n"; - print BULK_FH "$benchmark_cmd\n"; - } - } else { - print BULK_FH "$benchmark_cmd &\n"; - } - $nr_pp_jobs++; - } else { - open(CMD_FH, ">" . $iteration_sample_dir . "/benchmark-sample.cmd"); - printf CMD_FH "%s\n", $benchmark_cmd; - close(CMD_FH); - chmod(0755, $iteration_sample_dir . "/benchmark-sample.cmd"); - system($iteration_sample_dir . "/benchmark-sample.cmd"); - my $exit_code = $?; - if ($exit_code != 0) { - printf "Stopping because of iteration-sample exit code: %d\n", $exit_code; - exit 1; - } - } - } - $iteration_id++; - } - if ($pp_only) { - if ($params{'postprocess-mode'} eq 'cdm') { - print BULK_FH "echo Waiting for $nr_pp_jobs post-processing jobs to finish\n"; - print BULK_FH "wait\n"; - } - print BULK_FH "echo Sample processing complete!\n"; - close(BULK_FH); - system(". ./bulk-sample.sh"); - } - $run_doc{'run'}{'end'} = int time * 1000; # time in milliseconds - put_json_file(\%run_doc, $es_dir . "/run/run" . $run_part . "-" . $run_doc{'run'}{'id'} . ".json"); - %last_run_doc = %run_doc; # We need to keep at least 1 run doc to create the config docs later - $run_part++; -} - -if (! $pp_only) { - close $iterations_fh; - close $iterations_and_params_fh; - end_run($params{"sysinfo"}, $tool_group); -} - -if ($params{'postprocess-mode'} eq 'html') { - # Generate the result.html (to go away once CDM/Elastic UI is available) - print "Running generate-benchmark-summary..."; - system(". " . $pbench_install_dir . "/base; " . $pbench_install_dir . - "/bench-scripts/postprocess/generate-benchmark-summary " . $benchmark . - " " . $benchmark . " " . $base_bench_dir); - print "finished\n"; -} - -printf "Run complete\n\n"; diff --git a/agent/bench-scripts/postprocess/fio-postprocess-cdm b/agent/bench-scripts/postprocess/fio-postprocess-cdm deleted file mode 100755 index 74b988f5f8..0000000000 --- a/agent/bench-scripts/postprocess/fio-postprocess-cdm +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/perl -## -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl - -use strict; -use warnings; -use File::Basename; -use lib $ENV{'pbench_lib_dir'}; -use JSON; -use Data::Dumper; -use PbenchCDM qw(log_cdm_metric_sample gen_cdm_metric_data create_bench_iter_sample_period_doc); -use PbenchBase qw(get_json_file put_json_file remove_element); - -printf "\@ARGV: %s\n", join(" ", @ARGV); - -if (scalar @ARGV < 4) { - print "You must supply 4 arguments to fio-postprocess-cdm:\n"; - print "(1) The run directory\n"; - print "(2) The sample directory\n"; - print "(3) The tool group\n"; - print "(4) 0 or 1 indicating this is the last sample\n"; - exit 1; -} - -my $script_name = basename($0); -my $rundir = shift(@ARGV); -my $base_bench_dir = shift(@ARGV); -my $tool_group = shift(@ARGV); -my $last_sample = shift(@ARGV); -my $pp_mode = shift(@ARGV); - -if ($pp_mode eq 'cdm') { - my $json_ref = get_json_file("sample.json"); - my %sample = %$json_ref; # This is the CDM doc for this benchmark-iteration-sample - my $sample_id = $sample{'sample'}{'id'}; # Needs to be included in all metric-sample docs - my $run_id = $sample{'run'}{'id'}; # Needs to be included in all metric-sample docs - - # Create [at least one] benchmark period: a window of time during this benchmark, this one marked - # as "measurement", to indicate this period as the one the the UI/query tool computes a summary - # metric - my %period = create_bench_iter_sample_period_doc(\%sample, "measurement"); - # The primary metric needs to match exactly one metric-type that gets created from the benchmark data - $sample{'iteration'}{'primary_metric'} = "iops"; - $sample{'iteration'}{'primary_period'} = "measurement"; - put_json_file(\%sample, $base_bench_dir . "/es/bench/sample-" . $sample{'sample'}{'id'} . ".json"); - - # Under ./clients/, process the fio result files - opendir(my $clients_dh, "clients") || die "$script_name: could not open directory clients: $!\n"; - my @client_dirs = grep(!/^\./, (sort readdir($clients_dh))); - closedir $clients_dh; - my $earliest_timestamp; - my $latest_timestamp; - my %fio_cdm; - for my $host (@client_dirs) { - # Until the CDM/Elastic UI/Web interface is ready, we need to - # call the legacy fio post-processing script to generate a - # Pbench result.json for each sample. In order to do that, we - # need a single "fio-result.json", so we symilnk that from the - # first client to the main directory, where that script expects - # it to be - mkdir("../../es/metrics"); - mkdir("../../es/metrics/" . $host); - print "client: $host\n"; - opendir(my $client_dh, "clients/" . $host) || - die "$script_name: could not open client directory: $!\n"; - my @log_files = grep(/fio_([a-z]+)\.(\d+)\.log$/, (sort readdir($client_dh))); - closedir $client_dh; - for my $log_file (@log_files) { - if ($log_file =~ /fio_([a-z]+)\.(\d+)\.log$/) { - my $metric_type = $1; - my $job_id = $2; - open(my $log_fh, "clients/$host/$log_file") - || die "$script_name: could not open file clients/$host/$log_file $!\n"; - while (<$log_fh>) { - my $line = "$_"; - chomp($line); - if ( $line =~ /(\d+),\s+(\d+),\s+(\d+),\s+(\d+)/ ) { - my $timestamp_ms = $1; - my $value = $2; - my $rwtype = $3; - if ($rwtype == 0) { - $rwtype = "Read"; - } elsif ($rwtype == 1) { - $rwtype = "Write"; - } elsif ($rwtype == 2) { - $rwtype = "Trim"; - } else { - $rwtype = "Unknown"; - } - my %md = ( 'host' => $host, 'job' => $job_id, 'action' => $rwtype ); - log_cdm_metric_sample('fio', 'throughput', $metric_type, '%host%-%job%-%action%', - \%md, \%{ $fio_cdm{$host} }, $timestamp_ms, $value); - $earliest_timestamp = $timestamp_ms if (not defined $earliest_timestamp - or $timestamp_ms < $earliest_timestamp); - $latest_timestamp = $timestamp_ms if (not defined $latest_timestamp - or $timestamp_ms > $latest_timestamp); - } - } - close($log_fh); - } - } - # Metric documents are not generated yet because we - # don't have all the info for the period document - } - $period{'period'}{'begin'} = $earliest_timestamp; - $period{'period'}{'end'} = $latest_timestamp; - my $period_file = $base_bench_dir . "/es/bench/period-" . $period{'period'}{'id'} . ".json"; - put_json_file(\%period, $period_file); - # Now that the period document is complete, we can create the metric documents - for my $host (keys %fio_cdm) { - gen_cdm_metric_data(\%{ $fio_cdm{$host} }, $period_file, $base_bench_dir . "/es", $host, "fio"); - } - - # Now post-process the tools for CDM - my $tools_cdm_pp_cmd = "pbench-postprocess-tools-cdm " . $period_file . " " . - $base_bench_dir . "/es " . $rundir; - printf "tools_cdm_pp cmd\n%s\n\n", $tools_cdm_pp_cmd; - my $tools_cdm_pp_output = `$tools_cdm_pp_cmd`; - printf "tools_cdm_pp output\n%s\n\n", $tools_cdm_pp_output; -} else { # $pp_mode = html - opendir(my $clients_dh, "clients") || die "$script_name: could not open directory clients: $!\n"; - my @client_dirs = grep(!/^\./, (sort readdir($clients_dh))); - closedir $clients_dh; - for my $host (@client_dirs) { - # We need a single "fio-result.json", so we symilnk that from the - # first client to the main directory, where the legacy pp script expects - # it to be - if (not -e "fio-result.txt") { - symlink("clients" . "/" . $host . "/" . "fio-result.json", "fio-result.txt"); - last; - } - } - # Call the legacy post-process script, so we can build the static html reports later - system($ENV{'pbench_install_dir'} . "/bench-scripts/postprocess/fio-postprocess `/bin/pwd` fio default"); - if (defined $last_sample and $last_sample eq "1") { - system($ENV{'pbench_install_dir'} . "/bench-scripts/postprocess/process-iteration-samples `/bin/pwd`/.. iops 100 0 1 n n"); - } -} diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-41.sts b/agent/bench-scripts/tests/pbench-run-benchmark/test-41.sts deleted file mode 100644 index d00491fd7e..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-41.sts +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-41.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-41.txt deleted file mode 100644 index 968d7b29f3..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-41.txt +++ /dev/null @@ -1,14 +0,0 @@ -+++ Running test-41 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -You must supply at least one of the following supported benchmark names: - fio ---- Finished test-41 pbench-run-benchmark (status=1) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-42.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-42.opts deleted file mode 100644 index b92b8b7026..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-42.opts +++ /dev/null @@ -1 +0,0 @@ -list diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-42.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-42.txt deleted file mode 100644 index 92b017add7..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-42.txt +++ /dev/null @@ -1,13 +0,0 @@ -+++ Running test-42 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -fio ---- Finished test-42 pbench-run-benchmark (status=0) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-43.opts deleted file mode 100644 index 62151047cd..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.opts +++ /dev/null @@ -1 +0,0 @@ -fio diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.sts b/agent/bench-scripts/tests/pbench-run-benchmark/test-43.sts deleted file mode 100644 index d00491fd7e..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.sts +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-43.txt deleted file mode 100644 index 6992a84759..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-43.txt +++ /dev/null @@ -1,13 +0,0 @@ -+++ Running test-43 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -You must specify at least 1 client with --clients ---- Finished test-43 pbench-run-benchmark (status=1) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-44.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-44.opts deleted file mode 100644 index 8ca45fbffb..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-44.opts +++ /dev/null @@ -1 +0,0 @@ -fio --clients=testhost.example.com diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-44.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-44.txt deleted file mode 100644 index bae67a3d6f..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-44.txt +++ /dev/null @@ -1,89 +0,0 @@ -+++ Running test-44 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release - -***** it is highly recommended you do one of the following: -- export this variable before calling this script USER_NAME -- use this parameter when calling this script: --user-name - - -***** it is highly recommended you do one of the following: -- export this variable before calling this script USER_EMAIL -- use this parameter when calling this script: --user-email - - -***** it is highly recommended you do one of the following: -- export this variable before calling this script USER_DESC -- use this parameter when calling this script: --user-desc - - -***** it is highly recommended you do one of the following: -- export this variable before calling this script USER_TAGS -- use this parameter when calling this script: --user-tags - -cmdfile: /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/pbench-run-benchmark.cmd -Generating all benchmark iterations -resolving default first with pbench-gen-iterations fio --defaults-only --clients=testhost.example.com - - - -iteration_ID: 0 -iteration_params: --log_avg_msec=1000 --log_hist_msec=10000 --log_unix_epoch=1 --output=fio-result.json --output-format=json --write_bw_log=fio --write_hist_log=fio --write_iops_log=fio --write_lat_log=fio --clients=testhost.example.com -sample0 -Running generate-benchmark-summary...finished -Run complete - ---- Finished test-44 pbench-run-benchmark (status=0) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00 -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/.iterations -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/0__fio -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/0__fio/sample0 -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/bench -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/config -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/metrics -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/run -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/run/run0-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/iteration-list.txt -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/metadata.log -/var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/pbench-run-benchmark.cmd -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state -+++ test-execution.log file contents -/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/generate-benchmark-summary fio fio /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00 -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-run-benchmark-sample /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/0__fio/sample0 /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00 default 1 html 0 -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-tool-meister-start --sysinfo=default default -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-tool-meister-stop --sysinfo=default default ---- test-execution.log file contents -+++ fio__1900.01.01T00.00.00/metadata.log file contents -[pbench] - -[iterations/0__fio] -iteration_number = 0 -iteration_name = 0__fio -log_avg_msec = 1000 -log_hist_msec = 10000 -log_unix_epoch = 1 -output = fio-result.json -output-format = json -write_bw_log = fio -write_hist_log = fio -write_iops_log = fio -write_lat_log = fio -clients = testhost.example.com - ---- fio__1900.01.01T00.00.00/metadata.log file contents -+++ fio__1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd file contents -pbench-run-benchmark-sample /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00/0__fio/sample0 /var/tmp/pbench-test-bench/pbench-agent/fio__1900.01.01T00.00.00 default 1 html 0 ---- fio__1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd file contents -+++ fio__1900.01.01T00.00.00/pbench-run-benchmark.cmd file contents -pbench-run-benchmark fio --clients=testhost.example.com ---- fio__1900.01.01T00.00.00/pbench-run-benchmark.cmd file contents diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-45.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-45.opts deleted file mode 100644 index e04df0a967..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-45.opts +++ /dev/null @@ -1 +0,0 @@ -fio --user-tags=test-45-tags --user-email=test-45@example.com --user-name=test-45-user --user-desc=test-45-desc --clients=testhost.example.com diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-45.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-45.txt deleted file mode 100644 index 0ad42a3d01..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-45.txt +++ /dev/null @@ -1,69 +0,0 @@ -+++ Running test-45 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -cmdfile: /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/pbench-run-benchmark.cmd -Generating all benchmark iterations -resolving default first with pbench-gen-iterations fio --defaults-only --clients=testhost.example.com - - - -iteration_ID: 0 -iteration_params: --log_avg_msec=1000 --log_hist_msec=10000 --log_unix_epoch=1 --output=fio-result.json --output-format=json --write_bw_log=fio --write_hist_log=fio --write_iops_log=fio --write_lat_log=fio --clients=testhost.example.com -sample0 -Running generate-benchmark-summary...finished -Run complete - ---- Finished test-45 pbench-run-benchmark (status=0) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00 -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/.iterations -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/0__fio -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0 -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/bench -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/config -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/metrics -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/run -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/run/run0-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/iteration-list.txt -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/metadata.log -/var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/pbench-run-benchmark.cmd -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state -+++ test-execution.log file contents -/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/generate-benchmark-summary fio fio /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00 -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-run-benchmark-sample /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0 /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00 default 1 html 0 -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-tool-meister-start --sysinfo=default default -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-tool-meister-stop --sysinfo=default default ---- test-execution.log file contents -+++ fio_test-45-tags_1900.01.01T00.00.00/metadata.log file contents -[pbench] - -[iterations/0__fio] -iteration_number = 0 -iteration_name = 0__fio -log_avg_msec = 1000 -log_hist_msec = 10000 -log_unix_epoch = 1 -output = fio-result.json -output-format = json -write_bw_log = fio -write_hist_log = fio -write_iops_log = fio -write_lat_log = fio -clients = testhost.example.com - ---- fio_test-45-tags_1900.01.01T00.00.00/metadata.log file contents -+++ fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd file contents -pbench-run-benchmark-sample /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/es/bench/iteration-AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE.json /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0 /var/tmp/pbench-test-bench/pbench-agent/fio_test-45-tags_1900.01.01T00.00.00 default 1 html 0 ---- fio_test-45-tags_1900.01.01T00.00.00/0__fio/sample0/benchmark-sample.cmd file contents -+++ fio_test-45-tags_1900.01.01T00.00.00/pbench-run-benchmark.cmd file contents -pbench-run-benchmark fio --user-tags=test-45-tags --user-email=test-45@example.com --user-name=test-45-user --user-desc=test-45-desc --clients=testhost.example.com ---- fio_test-45-tags_1900.01.01T00.00.00/pbench-run-benchmark.cmd file contents diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-46.opts deleted file mode 100644 index 8bf7e708cf..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.opts +++ /dev/null @@ -1 +0,0 @@ -linpack --user-tags=test-46-tags --user-email=test-46@example.com --user-name=test-46-user --user-desc=test-46-desc --clients=testhost.example.com diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.sts b/agent/bench-scripts/tests/pbench-run-benchmark/test-46.sts deleted file mode 100644 index d00491fd7e..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.sts +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-46.txt deleted file mode 100644 index b2b7122161..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-46.txt +++ /dev/null @@ -1,14 +0,0 @@ -+++ Running test-46 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -Unsupported benchmark linpack; please supply one of the following supported benchmark names: - fio ---- Finished test-46 pbench-run-benchmark (status=1) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.opts b/agent/bench-scripts/tests/pbench-run-benchmark/test-48.opts deleted file mode 100644 index 1ebca08572..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.opts +++ /dev/null @@ -1 +0,0 @@ -uperf --user-tags=test-45-tags --user-email=test-45@example.com --user-name=test-45-user --user-desc=test-45-desc --clients=testhost.example.com diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.sts b/agent/bench-scripts/tests/pbench-run-benchmark/test-48.sts deleted file mode 100644 index d00491fd7e..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.sts +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.txt b/agent/bench-scripts/tests/pbench-run-benchmark/test-48.txt deleted file mode 100644 index 0ad2c230b5..0000000000 --- a/agent/bench-scripts/tests/pbench-run-benchmark/test-48.txt +++ /dev/null @@ -1,14 +0,0 @@ -+++ Running test-48 pbench-run-benchmark -WARNING: pbench-run-benchmark is deprecated and will be removed in the next release -Unsupported benchmark uperf; please supply one of the following supported benchmark names: - fio ---- Finished test-48 pbench-run-benchmark (status=1) -+++ pbench tree state -/var/tmp/pbench-test-bench/pbench-agent -/var/tmp/pbench-test-bench/pbench-agent/tmp -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/__trigger__ -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/mpstat -/var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar ---- pbench tree state diff --git a/agent/bench-scripts/unittests b/agent/bench-scripts/unittests index 541cec52f6..404259494e 100755 --- a/agent/bench-scripts/unittests +++ b/agent/bench-scripts/unittests @@ -27,14 +27,12 @@ export _testdir=${_testroot}/pbench-agent # Copy bench-scripts execution environment to _testroot export _testopt=${_testroot}/opt/pbench-agent res=0 -mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/driver ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib ${_testopt}/config/benchmark +mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/driver ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib ${_testopt}/config let res=res+${?} cat ${_tdir}/../base.ut.pre ${_tdir}/../base ${_tdir}/../base.ut.post > ${_testopt}/base let res=res+${?} cp -rH ${_tdir}/../lib/* ${_testopt}/lib/ let res=res+${?} -cp -rH ${_tdir}/../config/benchmark/* ${_testopt}/config/benchmark/ -let res=res+${?} for script in ${_tdir}/pbench-* ; do cp $script ${_testopt}/bench-scripts/ let res=res+${?} @@ -111,11 +109,7 @@ function _run { ${_real_cat} ${env_file} >> ${_testroot}/run.cmd printf -- "\n" >> ${_testroot}/run.cmd fi - if [[ "${tscrpt}" == "pbench-run-benchmark" ]]; then - local config="" - else - local config=" --config=${tname}" - fi + local config=" --config=${tname}" printf -- "%s%s %s\n" "${tscrpt}" "${config}" "${opts}" >> ${_testroot}/run.cmd chmod +x ${_testroot}/run.cmd ${_testroot}/run.cmd >> ${_testout} 2>&1 @@ -334,24 +328,6 @@ while read tst benchmark; do # Determine what the test will use for a "benchmark_run_dir" if [[ ${benchmark} == "pbench-user-benchmark" ]]; then _rundir="${benchmark}_${tst}_${_date_suffix}" - elif [[ ${benchmark} == "pbench-run-benchmark" ]]; then - # The first argument in the "opts" file determines the name - # that will be used for the benchmark run directory. - if [[ -f ${_tdir}/tests/${_tst}.opts ]]; then - bm_name="$(head -n 1 ${_tdir}/tests/${_tst}.opts | awk '{print $1}')" - if [[ "${bm_name}" != "list" ]]; then - bm_config="$(head -n 1 ${_tdir}/tests/${_tst}.opts | awk '{print $2}')" - if [[ "${bm_config%=*}" == "--user-tags" ]]; then - _rundir="${bm_name}_${bm_config#*=}_${_date_suffix}" - else - _rundir="${bm_name}__${_date_suffix}" - fi - else - _rundir="" - fi - else - _rundir="" - fi elif [[ ${benchmark%%-*} == "pbench" ]]; then _rundir="${bm_postfix}_${tst}_${_date_suffix}" else diff --git a/agent/config/benchmark/fio.json b/agent/config/benchmark/fio.json deleted file mode 100644 index 2666b8a77b..0000000000 --- a/agent/config/benchmark/fio.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "controller" : { - "pre-script" : "/opt/pbench-agent/bench-scripts/postprocess/fio-prepare-jobfile", - "post-script" : "/opt/pbench-agent/bench-scripts/postprocess/fio-postprocess-cdm", - "parameters" : { - "defaults": { - "basic" : [ "--rw=read,randread", "--filename=/tmp/fio-tst", "--filesize=256M", - "--runtime=30s", "--bs=16k", "--time_based=1" - ], - "ceph-osp" : [ "--rw=read,randread,write,randwrite", "--bs=1m", "--ioengine=sync", - "--numjobs=1", "--runtime=30s", "--filesize=512M" - ], - "mandatory" : [ "--write_hist_log=fio", "--log_hist_msec=10000", - "--write_bw_log=fio", "--write_iops_log=fio", "--write_lat_log=fio", - "--log_avg_msec=1000", "--log_unix_epoch=1", "--output-format=json", - "--output=fio-result.json" - ] - }, - "specs" : { - "size_KMG" : { - "description" : "bytes in k/K (1024), m/M (1024^2) or g/G (1024^3): 4k 16M 1g", - "arguments" : [ "bs", "filesize", "io_size", "mem" ], - "value_regex" : "[0-9]+[kbmgKBMG]", - "value_transforms" : [ "s/([0-9]+)[gG]/($1*1024).\"M\"/e", - "s/([0-9]+)[mM]/($1*1024).\"K\"/e" - ] - }, - "size_KMG_range" : { - "description" : "a range of size_KMG: 4k-8k 16k-2m 24m-1G", - "arguments" : [ "bsrange", "bssplit" ], - "value_regex" : "[0-9]+[kbmgKBMG]\\-[0-9]+[kbmgKBMG]", - "value_transforms" : [ "s/([0-9]+)[gG]/($1*1024).\"M\"/eg", - "s/([0-9]+)[mM]/($1*1024).\"K\"/eg" - ] - }, - "generic_string" : { - "description" : "all types of strings", - "arguments" : [ - "output-format", "output", "buffer_pattern", "cgroup", "clocksource", - "continue_on_error", "cpus_allowed_policy", "cpus_allowed", "directory", - "exec_postrun", "exec_prerun", "filename", "ignore_error", "ioscheduler", - "jobfile" - ], - "value_regex" : ".+" - }, - "rw_types" : { - "description" : "all possible testtypes", - "arguments" : [ "rw" ], - "value_regex" : "^(|rand)(read|write|trim)$|^readwrite$|^randrw$|^trimwrite$" - }, - "ioengine_types" : { - "description" : "all possible ioengine types", - "arguments" : [ "ioengine" ], - "value_regex" : "^(|p|pv|v)sync$|^pvsync2$|^posixaio$|^mmap$|^(|net)splice$|^sg$|^null$|^net$|^cpuio$|^rdma$|^e4defrag$|^falloc$|^filecreate$|^external$|^libaio$" - }, - "log_types" : { - "description" : "all possible log types", - "arguments" : [ "write_bw_log", "write_hist_log", "write_iolog", "write_iops_log", - "write_lat_log" - ], - "value_regex" : "^fio$" - }, - "io_submit_modes" : { - "description" : "How IO submissions and completions are done", - "arguments" : [ "io_submit_mode" ], - "value_regex" : "^inline$|^offload$" - }, - "time_smh" : { - "description" : "time in seconds, minutes, or hours: 10s 2m 1h", - "arguments" : [ "runtime", "steadystate_duration", "steadystate_ramp_time", - "steadystate" - ], - "value_regex" : "^[0-9]+[smh]$" - }, - "boolean" : { - "description" : "1 for true and 0 for false", - "arguments" : [ "allow_file_create", "allow_mounted_write", "allrandrepeat", - "atomic", "block_error_percentiles", "buffered", "cgroup_nodelete", - "clat_percentiles", "create_fsync", "create_only", "create_on_open", - "create_serialize", "direct", "disable_bw_measurement", "disable_clat", - "disable_lat", "disable_slat", "disk_util", "do_verify", "end_fsync", - "error_dump", "experimental_verify", "file_append", "fill_device", - "fsync_on_close", "gtod_reduce", "invalidate", "lat_percentiles", "log_max_value", - "log_offset", "log_store_compressed", "log_unix_epoch", "overwrite", "per_job_logs", - "pre_read", "randrepeat", "rate_ignore_thinktime", "replay_no_stall", - "scramble_buffers", "serialize_overlap", "stats", "sync", "trim_verify_zero", - "unified_rw_reporting", "unique_filename", "unlink_each_loop", "unlink", - "verify_dump", "erify_fatal", "verify_state_load", "verify_state_save", - "time_based" - ], - "value_regex" : "[0,1]" - }, - "positive_integer" : { - "description" : "a whole number greater than 0", - "arguments" : [ "ba", "buffer_compress_chunk", "buffer_compress_percentage", - "bwavgtime", "cgroup_weight", "cpumask", "dedupe_percentage", "fdatasync", "flow_id", - "flow_sleep", "flow", "flow_watermark", "fsync", "gia", "gtod_cpu", "hugepage-size", - "iodepth_batch_complete_max", "iodepth_batch_complete_min", "iodepth_batch", - "iodepth_low", "iodepth", "iopsavgtime", "kb_base", "log_avg_msec", "log_compression", - "log_hist_coarseness", "log_hist_msec", "loops", "nice", "nrfiles", "numjobs", - "offset_align", "openfiles", "percentage_random", "prioclass", "prio", "rate_cycle", - "rate_iops_min", "rate_iops", "rate_min", "rate", "replay_align", "replay_scale", - "replay_time_scale", "rwmixread", "rwmixwrite", "significant_figures", - "thinktime_blocks", "thinktime_spin", "thinktime", "trim_backlog_batch", - "trim_percentage", "uid", "unit_base", "verify_async", "verify_backlog_batch", - "verify_interval", "verify_offset", "write_barrier" - ], - "value_regex" : "[1-9][0-9]*" - } - } - } - }, - "client" : { - "packages" : [ "pbench-fio-3.19-1" ], - "pre-script" : " ", - "post-script" : " ", - "copy-to" : [ "/var/lib/pbench-agent/tmp/fio.job" ], - "bin" : "fio", - "param_regex" : [ "s/--clients=.+//", - "s/\\s--jobfile=(\\S+)//", - "s/(.*)/$1fio.job/" - ] - } -} diff --git a/agent/config/benchmark/fio.json.example b/agent/config/benchmark/fio.json.example deleted file mode 100644 index 47b53a53b9..0000000000 --- a/agent/config/benchmark/fio.json.example +++ /dev/null @@ -1,176 +0,0 @@ -// Benchmark configuration data for fio -// -// There are potentially 3 sections for a benchmark: controller, client, and server -// Every benchmark must have a cotroller and client section, and some benchmarks -// need a server section. -{ - // This section defines how the controller works: benchmark options, packages to install, - "controller" : { - // This script is run before any client/server execution. This may be used to - // prepare certain files on the controller for the benchmark. - "pre-script" : "/opt/pbench-agent/bench-scripts/postprocess/fio-prepare-jobfile", - // This script is run after data is copied from clients [and servers if applicable]. - // This script must translate the result data into metrics consumable by ES. - "post-script" : "/opt/pbench-agent/bench-scripts/postprocess/fio-postprocess-cdm", - // The following deals specially with command-line parameters of this benchmark. - // A parameter is a benchmark option, typically written as "--arg-=value" format. This - // section specifies various defaults (there can be different defaults for different - // use-cases), mandatory parameters, of which the pbench execution and post-processing - // expects, and finally a "specs" section which validates the user's parameters for - // correctness. - "parameters" : { - "defaults": { - // When calling pbench-run-benchmark, if one uses "--defaults=", a matching - // name will cause their respective parameters listed here to be used. - // If not "--defaults= is used, the entry "default" will be used - "basic" : [ "--rw=read,randread", "--filename=/tmp/fio-tst", "--filesize=256M", - "--runtime=30s", "--bs=16k" - ], - // This one is completely made-up and may bear no relevance to actual ceph use-case - "ceph-osp" : [ "--rw=read,randread,write,randwrite", "--bs=1m", "--ioengine=sync", - "--numjobs=1", "--runtime=30s", "--filesize=512M" - ], - // The "mandatory" entry will alwyas be included and cannot be overrided by the user. - // These are considered mandatory because the post-processing relies on the data - // these options produce. - "mandatory" : [ "--write_hist_log=fio", "--log_hist_msec=10000", - "--write_bw_log=fio", "--write_iops_log=fio", "--write_lat_log=fio", - "--log_avg_msec=1000", "--log_unix_epoch=1", "--output-format=json", - "--output=fio-result.json" - ] - }, - // The following infomation is to aid pbench in parsing benchmark-specific paramters. - // Paramaters are grouped together by similar *value* types and tranforms. - "specs" : { - // For example, a group of parameters named "size_KMG" describes values which are sizes, - // like 4k or 16M. This is a common value type used in fio for several parameters. - // Since the input value can use different units, the value is converted (transformed) - // to a common unit. - "size_KMG" : { - // A description of the types of values in "--arg=value" - "description" : "bytes in k/K (1024), m/M (1024^2) or g/G (1024^3): 4k 16M 1g", - // All of the valid strings for "arg" in "--arg=value" - "arguments" : [ "bs", "filesize", "io_size", "mem" ], - // This is the perl regex tested against the value in "--arg=value" which must pass - "value_regex" : "[0-9]+[kbmgKBMG]", - // Once value_regex passes, it can be optionally be tranformed with perl s// regex - "value_transforms" : [ "s/([0-9]+)[gG]/($1*1024).\"M\"/e", - "s/([0-9]+)[mM]/($1*1024).\"K\"/e" - ] - }, - "size_KMG_range" : { - "description" : "a range of size_KMG: 4k-8k 16k-2m 24m-1G", - "arguments" : [ "bsrange", "bssplit" ], - "value_regex" : "[0-9]+[kbmgKBMG]\\-[0-9]+[kbmgKBMG]", - "value_transforms" : [ "s/([0-9]+)[gG]/($1*1024).\"M\"/eg", - "s/([0-9]+)[mM]/($1*1024).\"K\"/eg" - ] - }, - "generic_string" : { - "description" : "all types of strings", - "arguments" : [ - "output-format", "output", "buffer_pattern", "cgroup", "clocksource", - "continue_on_error", "cpus_allowed_policy", "cpus_allowed", "directory", - "exec_postrun", "exec_prerun", "filename", "ignore_error", "ioscheduler", - "jobfile" - ], - "value_regex" : ".+" - // no "value_transform" here because there's no reason to alter this value - }, - "rw_types" : { - "description" : "all possible testtypes", - "arguments" : [ "rw" ], - "value_regex" : "^(|rand)(read|write|trim)$|^readwrite$|^randrw$|^trimwrite$" - }, - "ioengine_types" : { - "description" : "all possible ioengine types", - "arguments" : [ "ioengine" ], - "value_regex" : "^(|p|pv|v)sync$|^pvsync2$|^posixaio$|^mmap$|^(|net)splice$|^sg$|^null$|^net$|^cpuio$|^rdma$|^e4defrag$|^falloc$|^filecreate$|^external$|^libaio$" - }, - "log_types" : { - "description" : "all possible log types", - "arguments" : [ "write_bw_log", "write_hist_log", "write_iolog", "write_iops_log", - "write_lat_log" - ], - "value_regex" : "^fio$" - }, - "io_submit_modes" : { - "description" : "How IO submissions and completions are done", - "arguments" : [ "io_submit_mode" ], - "value_regex" : "^inline$|^offload$" - }, - "time_smh" : { - "description" : "time in seconds, minutes, or hours: 10s 2m 1h", - "arguments" : [ "runtime", "steadystate_duration", "steadystate_ramp_time", - "steadystate" - ], - "value_regex" : "^[0-9]+[smh]$" - // todo: use value_transform to convert all values to seconds (for consistency in search for ES) - }, - "boolean" : { - "description" : "1 for true and 0 for false", - "arguments" : [ "allow_file_create", "allow_mounted_write", "allrandrepeat", - "atomic", "block_error_percentiles", "buffered", "cgroup_nodelete", - "clat_percentiles", "create_fsync", "create_only", "create_on_open", - "create_serialize", "direct", "disable_bw_measurement", "disable_clat", - "disable_lat", "disable_slat", "disk_util", "do_verify", "end_fsync", - "error_dump", "experimental_verify", "file_append", "fill_device", - "fsync_on_close", "gtod_reduce", "invalidate", "lat_percentiles", "log_max_value", - "log_offset", "log_store_compressed", "log_unix_epoch", "overwrite", "per_job_logs", - "pre_read", "randrepeat", "rate_ignore_thinktime", "replay_no_stall", - "scramble_buffers", "serialize_overlap", "stats", "sync", "trim_verify_zero", - "unified_rw_reporting", "unique_filename", "unlink_each_loop", "unlink", - "verify_dump", "erify_fatal", "verify_state_load", "verify_state_save", - "time_based" - ], - "value_regex" : "[0,1]" - }, - "positive_integer" : { - "description" : "a whole number greater than 0", - "arguments" : [ "ba", "buffer_compress_chunk", "buffer_compress_percentage", - "bwavgtime", "cgroup_weight", "cpumask", "dedupe_percentage", "fdatasync", "flow_id", - "flow_sleep", "flow", "flow_watermark", "fsync", "gia", "gtod_cpu", "hugepage-size", - "iodepth_batch_complete_max", "iodepth_batch_complete_min", "iodepth_batch", - "iodepth_low", "iodepth", "iopsavgtime", "kb_base", "log_avg_msec", "log_compression", - "log_hist_coarseness", "log_hist_msec", "loops", "nice", "nrfiles", "numjobs", - "offset_align", "openfiles", "percentage_random", "prioclass", "prio", "rate_cycle", - "rate_iops_min", "rate_iops", "rate_min", "rate", "replay_align", "replay_scale", - "replay_time_scale", "rwmixread", "rwmixwrite", "significant_figures", - "thinktime_blocks", "thinktime_spin", "thinktime", "trim_backlog_batch", - "trim_percentage", "uid", "unit_base", "verify_async", "verify_backlog_batch", - "verify_interval", "verify_offset", "write_barrier" - ], - "value_regex" : "[1-9][0-9]*" - } - } // end of specs - } // end of parameters - }, // end of controller - // If the benchmark uses --clients parameter (all of them should), then this "client" section - // must be filled out. - "client" : { - // Any package listed here must be present, or an install is attempted - "packages" : [ "pbench-fio-3.3-1" ], - // A local path or URL to a script to run on client before test but after copy-to files, - // The benchmark parameters are passed to this, and this script must exit "0" to continue. - // If a URL is provided, the client will attempt to retrive it via curl. - "pre-script" : " ", - // Similar ro the pre-script, this will execute after the test. - "post-script" : " ", - // A list of files to copy to the client (under $pbench_bench) before the test - "copy-to" : [ "/var/lib/pbench-agent/tmp/fio.job" ], //todo: allow vars - // The benchmark binary to execute on the client - "bin" : "fio", - // When the benchmark binary is executed on the client, all of the parameters - // included in the benchmark-iteration are passed along. In order to make changes - // to those parameters, Apply a regex to the paramaters before executing. - // ##"s/\\s--jobfile=(\\S+)(.*)/$2$1/" // Strip --jobfile and put filename at end - "param_regex" : [ "s/--clients=.+//", // Remove entire --clients=hostanmes param - "s/\\s--jobfile=(\\S+)//", // Strip --jobfile - "s/(.*)/$1fio.job/" // put job filename at end - ] - } - // There can be a "server" section, just like a "client" section, but fio - // does not use this. Other benchmarks like uperf will use it. The server - // section follows the same format and function as the "clients" section, but - // applies to the hostnames provides with --servers= parameter. -} diff --git a/agent/config/benchmark/pbench.json b/agent/config/benchmark/pbench.json deleted file mode 100644 index 86728a01eb..0000000000 --- a/agent/config/benchmark/pbench.json +++ /dev/null @@ -1,27 +0,0 @@ -{ -"controller" : { - "parameters" : { - "specs" : { - "positive_integers" : { - "description" : "whole number >0, no + sign", - "arguments" : [ "samples" ], - "value_regex" : "^[0-9]+$" }, - "positive_floating_point" : { - "description" : "floating point number >0, no + sign", - "arguments" : [ "min_stddev_pct" ], - "value_regex" : "(^[0-9]+)\\.*[0-9]*$)|(^\\.[0-9]+)$" }, - "boolean" : { - "description" : "1 for true and 0 for false", - "arguments" : [ "postprocess-only" ], - "value_regex" : "^[01]$" }, - "comma-separated list, no spaces" : { - "description" : "list of words", - "arguments" : [ "clients" , "servers" ], - "value_regex" : "^(\\S+)(,\\S+)*$" } - }, - "defaults" : { - "fio" : "--maxstddevpct=3 --samples=3 --clients=localhost", - "uperf" : "--maxstddevpct=10 --samples=5 --clients=localhost --servers=localhost" } - } -} -} diff --git a/agent/lib/PbenchAnsible.pm b/agent/lib/PbenchAnsible.pm deleted file mode 100644 index a6227dcfa0..0000000000 --- a/agent/lib/PbenchAnsible.pm +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/perl -# -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 8 -*- -# Author: Andrew Theurer - -package PbenchAnsible; -use strict; -use warnings; -use File::Basename; -my $pbench_lib_path; -my $script_path; -my $script_name; -BEGIN { - $script_path = dirname($0); - $script_name = basename($0); - $pbench_lib_path = $script_path . "/postprocess"; -} -use lib "$pbench_lib_path"; -use Cwd 'abs_path'; -use Exporter qw(import); -use List::Util qw(max); -use Data::Dumper; -use JSON; -use PbenchBase qw(get_hostname); -our @EXPORT_OK = qw(ssh_hosts ping_hosts copy_files_to_hosts copy_files_from_hosts remove_files_from_hosts remove_dir_from_hosts create_dir_hosts sync_dir_from_hosts verify_success yum_install_hosts); - -my $script = "PbenchAnsible.pm"; -my $sub; -my $ansible_bin = "ansible"; -my $ansible_playbook_bin = "ansible-playbook"; -my $inventory_opt = " --inventory /var/lib/pbench-agent/ansible-hosts"; -my $ansible_base_cmdline = $ansible_bin; -my $ansible_playbook_cmdline = $ansible_playbook_bin; - -sub verify_success { - my $text_data = shift; - $text_data =~ s/^[^\{]*//; # remove any junk before actual json - my $data_ref = from_json($text_data); - my %stats = %{ $$data_ref{"stats"}}; - for my $host (keys %stats) { - if ((exists $stats{$host}{"failures"} and $stats{$host}{"failures"} > 0) or - (exists $stats{$host}{"unreachable"} and $stats{$host}{"unreachable"} > 0)) { - print "host $host failed"; - return 0; - } - } - return 1; -} -sub get_ansible_logdir { # create a directory to store ansible files - my $basedir = shift; - my $action = shift; - my $logdir = $basedir . "/ansible-log/"; - mkdir($logdir); - $logdir = $logdir . time . "-" . $action; - mkdir($logdir); - return $logdir; -} -sub log_ansible { # log the comand line and its output - my $logdir = shift; - my $cmd = shift; - my $output = shift; - mkdir($logdir); - my $fh; - open($fh, ">" . $logdir . "/command.txt") or die "Could not open $logdir/command.txt"; - print $fh $cmd; - close $fh; - open($fh, ">" . $logdir . "/output.json") or die "Could not open $logdir/output.json"; - print $fh $output; - close $fh; -} -sub build_inventory { # create an inventory file with hosts - my $hosts_ref = shift; - my $logdir = shift; - my $file = $logdir . "/hosts"; - my $fh; - open($fh, ">", $file) or die "Could not create the inventory file $file"; - for my $h (@$hosts_ref) { - print $fh "$h\n"; - } - close $fh; - return $file; -} -sub build_playbook { # create the playbok file - my $playbook_ref = shift; - my $logdir = shift; - my $fh; - my $file = $logdir . "/playbook.json"; - open($fh, ">", $file) or die "Could not create the playbook file $file"; - printf $fh "%s", to_json( $playbook_ref, { ascii => 1, pretty => 1, canonical => 1 } ); - close $fh; - return $file; -} -sub run_playbook { # execute a playbook - my $playbook_ref = shift; - my $inv_file = shift; - my $logdir = shift; - my $extra_vars = shift; - my $extra_vars_opt = ""; - if ($extra_vars) { - $extra_vars_opt = " --extra-vars " . $extra_vars . " "; - } - my $playbook_file = build_playbook($playbook_ref, $logdir); - my $full_cmd = "ANSIBLE_CONFIG=" . $script_path . "/../config/ansible.cfg " . - $ansible_playbook_cmdline . $extra_vars_opt . " -i " . $inv_file . " " . $playbook_file; - my $output = `$full_cmd`; - log_ansible($logdir, $full_cmd, $output); - if ($? == 0 and verify_success($output)) { - return $output; - } else { - print "Execution of this Ansible playbook failed\n"; - printf "playbook file: %s\n", $playbook_file; - exit 1; - } -} -sub ping_hosts { # check for connectivity with ping - my $hosts_ref = shift; - my $basedir = shift; # we create a new dir under this and log all Ansible files and output - my $logdir = get_ansible_logdir($basedir, "ping_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my $full_cmd = "ANSIBLE_CONFIG=" . $script_path . "/../config/ansible.cfg " . - $ansible_base_cmdline . " -i " . $inv_file . " all -m ping"; - my $output = `$full_cmd`; - log_ansible($logdir, $full_cmd, $output); - if (verify_success($output)) { - return $output; - } else { - print "Execution of this Ansible playbook failed\n"; - printf "Ansible log dir: %s\n", $logdir; - exit 1; - } -} -sub yum_install_hosts { # verify/install these packages on hosts - my $hosts_ref = shift; # array-reference to hosts - my $packages = shift; # array-reference of packages - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "yum_install_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - my @packages = @$packages; - my %yum = ( "name" => \@packages, "state" => "present" ); - my %task = ( "name" => "install a list of packages", "yum" => \%yum ); - push(@tasks, \%task); - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play);; - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub create_dir_hosts { # creates a directory on remote hosts - my $hosts_ref = shift; # array-reference to host list to copy from - my $dir = shift; # the directory to create - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "create_dir_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - my %task = ( "name" => "create dir on hosts", "file" => "path=" . $dir . " state=directory" ); - push(@tasks, \%task); - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play);; - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub ssh_hosts { # run a command on remote hosts - my $hosts_ref = shift; # array-reference to host list - my $cmd = shift; # command to run - my $chdir = shift; # directory to run command - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "ssh_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - my %task = ( name => "run cmd on hosts", command => $cmd . " chdir=" . $chdir); - push(@tasks, \%task); - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play);; - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub copy_files_to_hosts { # copies local files to hosts with a new, common destination path - my $hosts_ref = shift; # array-reference to host list - my $src_files_ref = shift; # array-refernce to file list - my $dst_path = shift; # a single destination path - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "copy_files_to_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - for my $src_file (@$src_files_ref) { - my %task = ( name => "copy files to hosts", copy => "mode=preserve src=" . $src_file . " dest=" . $dst_path . "/" . basename($src_file) ); - push(@tasks, \%task); - } - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play); - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub copy_files_from_hosts { # copies files from remote hosts to a local path which includes $hostbname directory - my $hosts_ref = shift; # array-reference to host list to copy from - my $src_files_ref = shift; # array-refernce to file list to fetch - my $src_path = shift; # a single src path where all files in list can be found - my $dst_path = shift; - my $basedir = shift; - if (!$dst_path) { - $dst_path="/tmp/"; - } - my $logdir = get_ansible_logdir($basedir, "copy_files_from_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - for my $src_file (@$src_files_ref) { - my %task = ( "name" => "copy files from hosts", "fetch" => "flat=yes " . "src=" . - $src_path . "/" . $src_file . " dest=" . $dst_path . - "/{{ inventory_hostname }}/" . $src_file); - push(@tasks, \%task); - } - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play);; - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub sync_dir_from_hosts { # copies files from remote hosts to a local path which includes $hostbname directory - my $hosts_ref = shift; # array-reference to host list to copy from - my $src_dir = shift; # the dir to sync from on the hosts - my $dst_dir = shift; # a single dst dir where all the remote host dirs will be sync'd to, first with a dir=hostname - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "sync_dir_from_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - my %task = ( "name" => "sync dirs from hosts", "synchronize" => "mode=pull src=" . $src_dir . " dest=" . $dst_dir . - "/{{ inventory_hostname }}/"); - push(@tasks, \%task); - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play); - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub remove_files_from_hosts { # copies files from remote hosts to a local path which includes $hostbname directory - my $hosts_ref = shift; # array-reference to host list to copy from - my $src_files_ref = shift; # array-refernce to file list to fetch - my $src_path = shift; # a single src path where all files in list can be found - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "remove_files_from_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - for my $src_file (@$src_files_ref) { - my %task = ( "name" => "remove files from hosts", "file" => "path=" . $src_path . "/" . $src_file . " state=absent" ); - push(@tasks, \%task); - } - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play); - return run_playbook(\@playbook, $inv_file, $logdir); -} -sub remove_dir_from_hosts { # copies files from remote hosts to a local path which includes $hostbname directory - my $hosts_ref = shift; # array-reference to host list to copy from - my $dir = shift; # the directory to delete - my $basedir = shift; - my $logdir = get_ansible_logdir($basedir, "remove_dir_from_hosts"); - my $inv_file = build_inventory($hosts_ref, $logdir); - my @tasks; - my %task = ( "name" => "remove dir from hosts", "file" => "path=" . $dir . " state=absent" ); - push(@tasks, \%task); - my %play = ( hosts => "all", tasks => \@tasks );; - my @playbook = (\%play);; - return run_playbook(\@playbook, $inv_file, $logdir); -} -1; diff --git a/agent/lib/PbenchCDM.pm b/agent/lib/PbenchCDM.pm deleted file mode 100644 index 1a7608e62d..0000000000 --- a/agent/lib/PbenchCDM.pm +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/perl -# -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl -# Author: Andrew Theurer - -package PbenchCDM; -use strict; -use warnings; -use File::Basename; -use Cwd 'abs_path'; -use Exporter qw(import); -use List::Util qw(max); -use JSON; -use Data::Dumper; -use Data::UUID; -use PbenchAnsible qw(ssh_hosts ping_hosts copy_files_to_hosts copy_files_from_hosts - remove_files_from_hosts remove_dir_from_hosts create_dir_hosts - sync_dir_from_hosts verify_success); -use PbenchBase qw(get_hostname get_pbench_datetime get_json_file); - -our @EXPORT_OK = qw(create_run_doc create_config_osrelease_doc create_config_cpuinfo_doc - create_config_netdevs_doc create_config_ethtool_doc create_config_base_doc - get_uuid create_bench_iter_sample_doc create_metric_desc_doc - create_metric_data_doc create_bench_iter_sample_period_doc - create_bench_iter_doc create_config_doc get_cdm_ver get_cdm_rel - log_cdm_metric_sample gen_cdm_metric_data); - -my $script = "PbenchCDM.pm"; -my $condense_samples = 1; -my $uuid = Data::UUID->new; - -sub get_cdm_ver { - return 4; -} - -sub get_cdm_rel { - return "dev"; # can also be "prod" -} - -sub get_uuid { - if (exists $ENV{"_PBENCH_UNIT_TESTS"}) { - return "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE"; - } else { - return $uuid->create_str(); - } -} - -sub get_user_name { # Looks for USER_NAME in %ENV - if (exists $ENV{"USER_NAME"}) { - return $ENV{"USER_NAME"} - } -} - -sub get_user_email { # Looks for USER_NAME in %ENV - if (exists $ENV{"USER_EMAIL"}) { - return $ENV{"USER_EMAIL"}; - } -} - -# Create the fields every doc must have -sub populate_base_fields { - my $doc_ref = shift; - $$doc_ref{'cdm'}{'ver'} = int get_cdm_ver; -} - -sub copy_doc_fields { - my $copy_from_ref = shift; # What document we need to copy from - my $copy_to_ref = shift; # What new document we're copying to - %$copy_to_ref = %$copy_from_ref; - undef $$copy_to_ref{'cdm'}; -} - -# Create a document with all of the run information -sub create_run_doc { - my %doc; - populate_base_fields(\%doc); - $doc{'run'}{'bench'}{'name'} = shift; # Name of the actual benchmark used, like fio ir uperf - $doc{'run'}{'bench'}{'params'} = shift; # Full list of parameters when calling the benchmark - $doc{'run'}{'bench'}{'clients'} = shift; # Client hosts involved in the benchmark - $doc{'run'}{'bench'}{'clients'} =~ s/,/ /g if (defined($doc{'run'}{'bench'}{'clients'})); - $doc{'run'}{'bench'}{'servers'} = shift; # Server hosts involved in the benchmark - $doc{'run'}{'bench'}{'servers'} =~ s/,/ /g if (defined($doc{'run'}{'bench'}{'servers'})); - $doc{'run'}{'user'}{'desc'} = shift; # User provided test description - $doc{'run'}{'user'}{'tags'} = shift; # User provided tags like, "beta,project-X" - $doc{'run'}{'user'}{'tags'} =~ s/,/ /g if (defined($doc{'run'}{'user'}{'tags'})); - $doc{'run'}{'user'}{'name'} = shift; # User's real name - $doc{'run'}{'user'}{'email'} = shift; # User's email address - $doc{'run'}{'harness_name'} = shift; # Harness name like pbench, browbeat, cbt - $doc{'run'}{'tool_names'} = shift; # List tool names like sar, mpstat - $doc{'run'}{'tool_names'} =~ s/,/ /g if (defined($doc{'run'}{'tool_names'})); - $doc{'run'}{'host'} = get_hostname; # Hostname of this controller system - $doc{'run'}{'ignore'} = JSON::false; # Set to true later if run should be ingnored in queries - $doc{'run'}{'begin'} = int time * 1000; - $doc{'run'}{'id'} = get_uuid; - $doc{'cdm'}{'doctype'} = 'run'; - return %doc; -} - -# Create a document describing a configuration source -sub create_config_doc { - my $copy_from_ref = shift; # first arg is a reference to a doc - # (like the run doc) we copy info from - my $config_ref = shift; # second arg is hash reference to any other - # keys/values to include in this doc - my %doc; - copy_doc_fields($copy_from_ref, \%doc); # get some essential fields from another doc (like run) - populate_base_fields(\%doc); - $doc{'cdm'}{'doctype'} = "config_" . $$config_ref{'module'}; - $doc{'config'}{'id'} = get_uuid; - for my $key (keys %$config_ref) { - if ($key =~ /^host$|^module$|^source_type$|^scribe_uuid$/) { - $doc{'config'}{$key} = $$config_ref{$key}; - } else { - $doc{'config'}{$$config_ref{'module'}}{$key} = $$config_ref{$key}; - } - } - return %doc; -} - -# Create a document describing the benchmark iteraton -# The benchmark iteration represents the benchmark configuration (parameters, hosts, etc). -# Typically, within a run, each iteration represents a unique set of parameters, where -# at least one --arg or value is different from another parameter. -sub create_bench_iter_doc { - my %doc; - copy_doc_fields(shift, \%doc); # Get some essential fields from iter-sample, our first arg - populate_base_fields(\%doc); - $doc{'iteration'}{'params'} = shift; # Second arg is benchmark parameters for this iter - $doc{'iteration'}{'id'} = get_uuid; - $doc{'cdm'}{'doctype'} = 'iteration'; - return %doc; -} - -# Create a document describing the benchmark iteraton sample -# A sample is a single execution of a benchmark iteration -# Multiple samples have the exact same benchmark parameters -sub create_bench_iter_sample_doc { - my %doc; - copy_doc_fields(shift, \%doc); # Get some essential fields from iter doc, our first arg - populate_base_fields(\%doc); - $doc{'sample'}{'num'} = shift; # Second arg is sample number - $doc{'sample'}{'id'} = get_uuid; - $doc{'cdm'}{'doctype'} = 'sample'; - return %doc; -} - -# Create a document describing the benchmark iteraton sample period -# A period is a length of time in the benchmark execution representing -# a certain action, like "warmup", "measurement", or "cool-down". -# All benchmarks must have at least 1 period, a "measurement" period -sub create_bench_iter_sample_period_doc { - my %doc; - copy_doc_fields(shift, \%doc); # Get some essential fields from iter-sample doc, our first arg - populate_base_fields(\%doc); - $doc{'period'}{'name'} = shift; # Second arg is period name - $doc{'period'}{'prev_id'} = shift; # Third arg is link to prev period in this sample, if any - $doc{'period'}{'id'} = get_uuid; - $doc{'cdm'}{'doctype'} = 'period'; - return %doc; -} - -# Create a document describing either a benchmark or tool metric (result) -sub create_metric_desc_doc { - my %doc; - my $period_doc_ref = shift; - $doc{'run'}{'id'} = $$period_doc_ref{'run'}{'id'}; - $doc{'iteration'}{'id'} = $$period_doc_ref{'iteration'}{'id'}; - $doc{'period'}{'id'} = $$period_doc_ref{'period'}{'id'}; - $doc{'sample'}{'id'} = $$period_doc_ref{'sample'}{'id'}; - populate_base_fields(\%doc); - $doc{'cdm'}{'doctype'} = 'metric_desc'; - # These are the required fields for any metric. There are potentially - # more fields, but not all metrics use all the same options fields. However, the ones - # below must all be used, and so creating a new doc requires that these fields be - # defined. - $doc{'metric_desc'}{'class'} = shift; # "throughput" (work over time, like Gbps or interrupts/sec) or - # "count" (quantity, percent, sum, elapsed time, value, etc) - $doc{'metric_desc'}{'type'} = shift; # A generic name for this metric, like "gigabits-per-second", - # does not include specifics like "/dev/sda" or "cpu1" - $doc{'metric_desc'}{'source'} = shift; # A benchmark or tool where this metric comes from, like - # "iostat" or "fio" - # The instance_name_format tells us how the name for this metric-instance is assembled. - # The instance name is assembled from other fields in this document - # the format is described by joining strings and field names - # (which are identified by % before and after), like: - # - # network-L2-%type%-%host%, - # where type=Gbps and host=perf1, - # so this resolves to: - # network-L2-Gbps-perf1 - # - # or: %source%-%bin%-%pid%/%tid%-%host%-percent-cpu-usage, - # where source=pidstat, bin=qemu pid=1094 tid=1095 host=perf1 - # so this resolves to: - # pidstat-qemu-1094/1095-perf1-percent-cpu-usage - # - # It is vitally important that the metric-instance name use enough fields - # so that the resulting name is different from any other metric-instance. - # For example, you want to ensure that the pidstat data from host perf1 - # does not have the exact same metric-instance name as pidstat data from - # host perf2, and so including %host% in a metric-instance name - # format is almost always required. However, there are potneitally many - # more fields required, and this code can't possibly know all situations - # so it is up to the caller of this function to understand that scenario - # and provide an adequtate instance name format. - $doc{'metric_desc'}{'name_format'} = shift; - $doc{'metric_desc'}{'id'} = get_uuid; - return %doc; -} -sub create_metric_data_doc { - my %doc; - $doc{'metric_data'}{'id'} = shift; # The same id as the metric_desc - $doc{'metric_data'}{'value'} = shift; # The value of the metric - $doc{'metric_data'}{'begin'} = int shift; # The begin epochtime for this value. - $doc{'metric_data'}{'end'} = int shift; # The end epochtime for this value. - $doc{'metric_data'}{'duration'} = $doc{'metric_data'}{'end'} - - $doc{'metric_data'}{'begin'} + 1; - return %doc; -} -sub log_cdm_metric_sample { - my $metric_source = shift; - my $metric_class = shift; - my $metric_type = shift; - my $metric_name_format = shift; - my $names_ref = shift; # contains all field names used in metric_name_format - my $metric_ref = shift; # the metric hash we are populating - my $timestamp_ms = shift; - my $value = shift; - my $metric_interval = shift; # the time in ms between each sample is logged - my $label = $metric_source . "-" . $metric_type . "-"; - # Extract the field names from name_format to create the label - my $tmp_name_format = $metric_name_format; - while ($tmp_name_format =~ s/([^%]*)%([^%]*)%//) { - my $name; - if (not defined ($name = $$names_ref{$2})) { - print "Error: field name $2 not defined\n"; - print "prefix: [$1]\n"; - print "field_name: [$2]\n"; - print "metric_source: [$metric_source]\n"; - print "metric_type: [$metric_type]\n"; - print "field names:\n"; - print Dumper $names_ref; - return 1; - } - $label .= $1 . $name; - } - # This only happens when logging the first sample for a metric - if (not exists $$metric_ref{$label}) { - print "creating label: $label metric_type: $metric_type metric_name_format: $metric_name_format\n"; - for my $name (keys %{ $names_ref }) { - $$metric_ref{$label}{'names'}{$name} = $$names_ref{$name}; - } - $$metric_ref{$label}{'source'} = $metric_source; - $$metric_ref{$label}{'class'} = $metric_class; - $$metric_ref{$label}{'type'} = $metric_type; - $$metric_ref{$label}{'name_format'} = $metric_name_format; - if (defined $metric_interval) { - $$metric_ref{$label}{'interval'} = $metric_interval; - } - } - if (not defined $value) { - print "Error: value is undefined\n"; - print "metric_source: [$metric_source]\n"; - print "metric_type: [$metric_type]\n"; - print "label: $label metric_type: $metric_type metric_name_format: $metric_name_format\n"; - print "field names:\n"; - print Dumper $names_ref; - return 2; - } - if (not $value =~ /^\d+$|^\.\d+$|^\d+\.\d*$/) { - print "Error: $label: value [$value] is not a number\n"; - return 3; - } - if (not defined $timestamp_ms) { - print "Error: timestamp is undefined\n"; - return 4; - } - $$metric_ref{$label}{'samples'}{int $timestamp_ms} = $value; - return 0; -} -sub gen_cdm_metric_data { - my $data_ref = shift; - my $period_doc_path = shift; - my $es_dir = shift; - my $hostname = shift; - my $tool = shift; - my $nr_samples = 0; - my $nr_condensed_samples = 0; - my $coder = JSON->new->ascii->canonical; - my $json_ref = get_json_file($period_doc_path); - my %period_doc = %$json_ref; # this is the CDM doc for the run - my $metric_data_line_num = 0; - my $metric_data_file_num = 0; - open(NDJSON_DESC_FH, ">" . $es_dir . "/metrics/" . $hostname . "/metric_desc-" . $period_doc{"period"}{"id"} . "-" . $tool . ".ndjson"); - open(NDJSON_DATA_FH, ">" . $es_dir . "/metrics/" . $hostname . "/metric_data" . $metric_data_file_num . "-" . $period_doc{"period"}{"id"} . "-" . $tool . ".ndjson"); - print "Generating CDM\n"; - for my $label (sort keys %$data_ref) { - # Limit the size of the metric_data files, so we don't fail on bulk import - if ($metric_data_line_num > 1000000) { - close(NDJSON_DATA_FH); - $metric_data_file_num++; - open(NDJSON_DATA_FH, ">" . $es_dir . "/metrics/" . $hostname . "/metric_data" . $metric_data_file_num . "-" . $period_doc{"period"}{"id"} . "-" . $tool . ".ndjson"); - $metric_data_line_num = 0; - } - my $nr_label_samples = 0; - my $nr_condensed_label_samples = 0; - if (exists $$data_ref{$label}{'samples'}) { - print "processing $label: "; - my $bail = 0; - my %series = %{$$data_ref{$label} }; - my %metric_desc = create_metric_desc_doc( \%period_doc, $series{'class'}, $series{'type'}, - $tool, $series{'name_format'}); - for my $field (keys %{ $series{'names'} }) { - $metric_desc{"metric_desc"}{'names'}{$field} = $series{'names'}{$field}; - } - printf NDJSON_DESC_FH "%s\n", '{ "index": {} }'; - printf NDJSON_DESC_FH "%s\n", $coder->encode(\%metric_desc); - my $begin_value; - my $begin_timestamp; - my $end_timestamp; - for my $timestamp_ms (sort keys %{ $series{'samples'} }) { - $nr_samples++; - $nr_label_samples++; - if (not defined $series{'samples'}{$timestamp_ms}) { - print "Warning: undefined value in\n"; - print Dumper \%series; - $bail = 1; - last; - }; - my $value = $series{'samples'}{$timestamp_ms}; - if (not defined $begin_value) { # The very first value - $begin_value = $value; - if (defined $series{'interval'}) { - # If we know the interval, we can calculate a $begin_timestamp - # and this new timestamp can be an $end_timestamp. If we don't - # know the interval, we have to wait until we get another timestamp - $begin_timestamp = $timestamp_ms - $series{'interval'} + 1; - } else { - $begin_timestamp = $timestamp_ms + 1; - next; - } - } - if ($condense_samples) { - if ($value == $begin_value) { # Keep extending the end timestamp - $end_timestamp = $timestamp_ms; - } elsif (defined $end_timestamp) { # The value changed, so log the previous sample - $nr_condensed_samples++; - $nr_condensed_label_samples++; - my %metric_data = create_metric_data_doc($metric_desc{'metric_desc'}{'id'}, - $begin_value, $begin_timestamp, $end_timestamp); - printf NDJSON_DATA_FH "%s\n", '{ "index": {} }'; - printf NDJSON_DATA_FH "%s\n", $coder->encode(\%metric_data); - $metric_data_line_num += 2; - # Since we start tracking a new value, begin_value and begin/end timestamps must be reasigned - $begin_value = $value; - $begin_timestamp = $end_timestamp + 1; - $end_timestamp = $timestamp_ms; - } else { # End was not defined yet because we only had 1 sample so far - $end_timestamp = $timestamp_ms; - } - } else { - $nr_condensed_samples++; - $nr_condensed_label_samples++; - my %metric_data = create_metric_data_doc($metric_desc{'metric_desc'}{'id'}, - $begin_value, $begin_timestamp, $timestamp_ms); - printf NDJSON_DATA_FH "%s\n", '{ "index": {} }'; - printf NDJSON_DATA_FH "%s\n", $coder->encode(\%metric_data); - $metric_data_line_num += 2; - $begin_value = $value; - $begin_timestamp = $timestamp_ms + 1; - - } - } - if ($condense_samples and not $bail) { - if (not defined $begin_value) { print "no beging_value\n"; print Dumper \%series; next;} - if (not defined $begin_timestamp) { print "no begin_timestamp\n"; print Dumper \%series; next;} - if (not defined $end_timestamp) { print "no end_timestamp\n"; print Dumper \%series; next;} - $nr_condensed_samples++; - $nr_condensed_label_samples++; - # Since we only log the previous value/timestamp in the while loop, we have to log the final value/timestamp - my %metric_data = create_metric_data_doc($metric_desc{'metric_desc'}{'id'}, - $begin_value, $begin_timestamp, $end_timestamp); - printf NDJSON_DATA_FH "%s\n", '{ "index": {} }'; - printf NDJSON_DATA_FH "%s\n", $coder->encode(\%metric_data); - $metric_data_line_num += 2; - } - if ($nr_label_samples > 0) { - printf "dedup reduction: %d%%\n", 100*(1 - $nr_condensed_label_samples/$nr_label_samples); - } else { - print "\n"; - } - } - } - close(NDJSON_DATA_FH); - close(NDJSON_DESC_FH); - if ($nr_samples > 0) { - printf "dedup reduction: %d%%\n", 100*(1 - $nr_condensed_samples/$nr_samples); - } -} - -1; diff --git a/agent/lib/PbenchES.pm b/agent/lib/PbenchES.pm deleted file mode 100644 index 76e8deef75..0000000000 --- a/agent/lib/PbenchES.pm +++ /dev/null @@ -1,849 +0,0 @@ -#!/usr/bin/perl -# -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl -# Author: Andrew Theurer - -package PbenchES; -use strict; -use warnings; -use File::Basename; -use Cwd 'abs_path'; -use Exporter qw(import); -use List::Util qw(max); -use Time::HiRes qw(gettimeofday); -use REST::Client; -use JSON; -use Data::Dumper; -use SigFigs; -use Array::Diff qw(:all); -use PbenchCDM qw(get_cdm_ver get_cdm_rel); - -our @EXPORT_OK = qw(get_primary_metric - get_sample_ids - es_get_query - get_primary_period_id - get_name_format_field_names - get_name_format - get_metric_label - get_metric_ids_from_term - get_metric_ids - get_common_time_domain - get_weighted_avg_agg - get_many_metric_results_from_ids - get_many_metric_results_from_term - get_metric_data - get_aggregation_from_name_format - gen_metric_group_terms - gen_metric_source_types - gen_label_to_terms - list_all_metrics - get_bench_name - get_primary_period_name); - -my %req_header = ("Content-Type" => "application/json"); -my $coder = JSON::MaybeXS->new->ascii->canonical; -my $template = -'{ - "size": 100, - "query" : { - "bool": { - "filter": [ - ] - } - } -}'; - -sub debug_log { - if (exists $ENV{'ES_DEBUG'} and $ENV{'ES_DEBUG'} > 0) { - my @time = gettimeofday; - printf "[%s]%s\n", join(".", @time), shift; - } -} - -sub get_index_basename { - return "cdmv" . get_cdm_ver . get_cdm_rel . "-"; -} - - -sub es_get_query { - my $host = shift; - my $req = get_index_basename . shift; - my $body = shift; - my $desc = shift; - my $client = REST::Client->new(); - $client->setHost($host); - my %req_header = ("Content-Type" => "application/json"); - debug_log(sprintf "%s:\nRequest: http://%s/%s\nbody: %s\n\n", $desc, $host, $req, $body); - $client->request('GET', $req, $body, \%req_header); - my $response = $client->responseContent(); - debug_log($response); - return $response; -} - -sub get_primary_metric { - my $host = shift; - my $iter_id = shift; - my $iter_req_ref = $coder->decode($template); - push @{ $$iter_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"iteration.id": "' . $iter_id . '"}}'); - my $iter_req_json = $coder->encode($iter_req_ref); - my $resp = es_get_query($host, "iteration/iteration/_search", $iter_req_json, "Query to get iteration document"); - my $iter_resp_ref = $coder->decode($resp); - #$bench_name = $$iter_resp_ref{'hits'}{'hits'}[0]{'_source'}{'run'}{'bench'}{'name'}; - #printf "bench_name:\n%s\n\n", $bench_name; - #printf "iteration.params:\n%s\n\n", $$iter_resp_ref{'hits'}{'hits'}[0]{'_source'}{'iteration'}{'params'}; - # get to get all sample docs for an iteration - my $samp_req_ref = $coder->decode($iter_req_json); - $$samp_req_ref{"aggs"}{"source"}{"terms"}{"field"} = "iteration.primary_metric"; - my $samp_req_json = $coder->encode($samp_req_ref); - my $samp_resp = es_get_query($host, "sample/sample/_search", $samp_req_json, "Query to get iteration.primary_metric from sample documents"); - my $samp_resp_ref = $coder->decode($samp_resp); - my $primary_metric; - if (exists $$samp_resp_ref{"aggregations"}{"source"}{"buckets"} and scalar @{ $$samp_resp_ref{"aggregations"}{"source"}{"buckets"} } == 1) { - $primary_metric = $$samp_resp_ref{"aggregations"}{"source"}{"buckets"}[0]{"key"}; - return $primary_metric; - } else { - die "ERROR: Could not find just one primary for this iteration"; - } -} - -sub get_primary_period_name { - my $host = shift; - my $sample_id = shift; - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"sample.id": "' . $sample_id . '"}}'); - my $req_json = $coder->encode($req_ref); - my $resp = es_get_query($host, "sample/sample/_search", $req_json, "Query to get iteration.primary_period sample documents"); - my $resp_ref = $coder->decode($resp); - return $$resp_ref{'hits'}{'hits'}[0]{'_source'}{'iteration'}{'primary_period'}; -} - -sub get_sample_ids { - my $host = shift; - my $iter_id = shift; - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"iteration.id": "' . $iter_id . '"}}'); - my $req_json = $coder->encode($req_ref); - my $resp = es_get_query($host, "sample/sample/_search", $req_json, "Query to get sample documents"); - my $resp_ref = $coder->decode($resp); - #return $$iter_resp_ref{'hits'}{'hits'}[0]{'_source'}{'run'}{'bench'}{'name'}; - my @sample_ids; - foreach my $samp ( @{ $$resp_ref{'hits'}{'hits'} } ) { - push @sample_ids, $$samp{'_source'}{'sample'}{'id'}; - } - return @sample_ids; -} - -sub get_bench_name { - my $host = shift; - my $iter_id = shift; - my $iter_req_ref = $coder->decode($template); - push @{ $$iter_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"iteration.id": "' . $iter_id . '"}}'); - my $iter_req_json = $coder->encode($iter_req_ref); - my $resp = es_get_query($host, "iteration/iteration/_search", $iter_req_json, "Query to get iteration document"); - my $iter_resp_ref = $coder->decode($resp); - return $$iter_resp_ref{'hits'}{'hits'}[0]{'_source'}{'run'}{'bench'}{'name'}; -} - -sub get_primary_period_id { - my $host = shift; - my $sample_id = shift; - my $primary_period = shift; - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.name": "' . $primary_period . '"}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"sample.id": "' . $sample_id . '"}}'); - my $req_json = $coder->encode($req_ref); - my $resp = es_get_query($host, "period/period/_search", $req_json, "Query for primary period"); - my $peri_ref = $coder->decode($resp); - if (scalar @{ $$peri_ref{'hits'}{'hits'} } == 1 ) { # Found exactly 1 primary period, which is what we want - my %peri = %{ $$peri_ref{'hits'}{'hits'}[0] }; - my $period_id = $peri{'_source'}{'period'}{'id'}; - return $period_id; - } -} -# Call with the name_format string, like "%host%-%socket%" and return an array of field names, like ('host', 'socket'); -sub get_name_format_field_names { - my $name_format = shift; - my @field_names; - while ( $name_format =~ /\%\S+\%/ ) { - $name_format =~ s/([^\%]*)\%(\w+)\%(\.*)/$3/; - push @field_names, $2; - } - return @field_names; -} - -sub get_name_format { - my $host = shift; - my $period_id = shift; - my $metric_source = shift; - my $metric_type = shift; - - my $metr_req_ref = $coder->decode($template); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.id": "' . $period_id . '"}}'); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.source": "' . $metric_source . '"}}'); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.type": "' . $metric_type . '"}}'); - $$metr_req_ref{"aggs"}{"source"}{"terms"}{"field"} = "metric_desc.name_format"; - $$metr_req_ref{"size"} = 0; - my $metr_req_json = $coder->encode($metr_req_ref); - my $metr_resp = es_get_query($host, "metric_desc/metric_desc/_search", $metr_req_json, "Query to get name_format"); - my $metr_resp_ref = $coder->decode($metr_resp); - if (scalar @{ $$metr_resp_ref{"aggregations"}{"source"}{"buckets"} } == 1) { - my $name_format = $$metr_resp_ref{"aggregations"}{"source"}{"buckets"}[0]{"key"}; - return $name_format; - } -} - -sub get_metric_ids_from_term { - my $host = shift; - my $period_id = shift; - my $terms = shift; - my %metric_ids_by_terms; - # the next request will start with our base reqeust from the metric request, - # where run/iteration/sample/period/metric.source/metric.type - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.id": "' . $period_id . '"}}'); - my $terms_ref = $coder->decode('[' . $terms . ']'); - # add to, not replace, the existing terms in the filter - my @filter = (@{ $$req_ref{"query"}{"bool"}{"filter"} }, @{ $terms_ref}); - $$req_ref{"query"}{"bool"}{"filter"} = \@filter; - $$req_ref{"size"} = 10000; # let's hope we get nowhere even close to this number, but we'll - # check if we do below - #$$req_ref{"aggs"}{"source"}{"terms"}{"field"} = "metric_desc.id"; - my $req_json = $coder->encode($req_ref); - my $resp = es_get_query($host, "metric_desc/metric_desc/_search", $req_json, "Query to get the mertic id from " . $terms); - my $resp_ref = $coder->decode($resp); - my $found_docs = 0; - #print Dumper - my $num_expected_docs = $$resp_ref{'hits'}{'total'}; - if ($num_expected_docs > $$req_ref{"size"}) { - die "The number of documents requested ($num_expected_docs) exceeds the 'size' parameter ($$req_ref{'size'}) set in this query\n"; - } - my @metric_ids; - for my $hit (@{ $$resp_ref{'hits'}{'hits'} }) { - push @metric_ids, $$hit{'_source'}{'metric_desc'}{'id'}; - $found_docs++; - } - if ($found_docs > 0) { - return @metric_ids; - } else { - print "Failed to find any metric ids for $terms\n"; - print Dumper $resp_ref; - } -} - -# Get *all* metric IDs for a given metric_source and metric_type. -# This is useful if you want to get a single value or data-series -# for any and all metric data-series which match the metric_source -# and metric_value. This is useful for computing a benchmark's -# primary metric value. It is not useful for breaking that value -# out, incrementally, into sub-components, unless you want it fully -# broken out into every single data-series. For example, you could -# use this info to calculate the single, primary metric for a fio -# execution, or you could use it to get the data-series for every -# single fio host/job/io-type, but not that useful if you wanted -# to break it out only by every host (but not job and io-type). -# For that, you need to use gen_metric_group_terms() -sub get_metric_ids { - my $host = shift; - my $period_id = shift; # Required - my $metric_source = shift; # Optional - my $metric_type = shift; # Can only be used if $metric_source is used - my @metric_ids; - if (not defined $period_id) { - return; - } - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.id": "' . $period_id . '"}}'); - if (defined $metric_source) { - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.source": "' . $metric_source . '"}}'); - } - if (defined $metric_type) { - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.type": "' . $metric_type . '"}}'); - } - $$req_ref{"aggs"}{"source"}{"terms"}{"field"} = "metric_desc.id"; - my $req_json = $coder->encode($req_ref); - my $resp = es_get_query($host, "metric_desc/metric_desc/_search", $req_json, - "Query to get all the metric data-series for the primary metric (fio: one data-series per host/job/IOtype)"); - my $resp_ref = $coder->decode($resp); - for my $bucket (@{ $$resp_ref{"aggregations"}{"source"}{"buckets"} }) { - push @metric_ids, $$bucket{"key"}; - } - return @metric_ids; -} - -sub get_common_time_domain { - my $host = shift; - my $period = shift; - my @metric_ids = @_; - my $latest_begin; - my $earliest_end; - # finding the time domain; the period of time where all the metrics ocurred - foreach my $metric_id (@metric_ids) { - my $metr_grp_req_ref = $coder->decode($template); - $$metr_grp_req_ref{'size'} = 0; - push @{ $$metr_grp_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_data.id": "' . $metric_id . '"}}'); - #$coder->decode('{"term": {"metric_data.id": "' . $metric_ids_by_terms{$terms} . '"}}'); - $$metr_grp_req_ref{"aggs"}{"earliest_begin"}{"min"}{"field"} = "metric_data.begin"; - $$metr_grp_req_ref{"aggs"}{"latest_end"}{"max"}{"field"} = "metric_data.end"; - my $metr_grp_req_json = $coder->encode($metr_grp_req_ref); - my $metr_grp_resp = es_get_query($host, "metric_data/metric_data/_search", - $metr_grp_req_json, "Query to find time domain for " . - join(" ", @metric_ids)); - my $metr_grp_resp_ref = $coder->decode($metr_grp_resp); - my $earliest_begin = $$metr_grp_resp_ref{"aggregations"}{"earliest_begin"}{"value"}; - if (not defined $latest_begin or $earliest_begin > $latest_begin) { - $latest_begin = $earliest_begin; - } - my $latest_end = $$metr_grp_resp_ref{"aggregations"}{"latest_end"}{"value"}; - if (not defined $earliest_end or $latest_end < $earliest_end) { - $earliest_end = $latest_end; - } - } - return ($latest_begin, $earliest_end); -} - -sub get_weighted_avg_agg { - my $req_ref = shift; - my $begin = shift; - my $end = shift; - my $mode = shift; - - if ($mode eq 'scripted-expression') { - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"value"}{"field"} = "metric_data.value"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"lang"} = "expression"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"params"}{"end"} = int $end; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"params"}{"begin"} = int $begin; - # The script must check to see if the metric.begin is earlier than the time period we want, and then trim the weight. - # Same goes for metric.end being after the time period we want. - # only type "long" appears to be compatible with .value.millis - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"inline"} = - "(doc['metric_data.end'] < end ? doc['metric_data.end'] : end) - " . - "(doc['metric_data.begin'] > begin ? doc['metric_data.begin'] : begin)"; - } elsif ($mode eq 'scripted-painless') { - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"value"}{"field"} = "metric_data.value"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"lang"} = "painless"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"params"}{"end"} = int $end; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"params"}{"begin"} = int $begin; - # The script must check to see if the metric.begin is earlier than the time period we want, and then trim the weight. - # Same goes for metric.end being after the time period we want. - # only type "long" appears to be compatible with .value.millis - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"source"} = - "long begin = doc['metric_data.begin'].value.millis < params.begin ? params.begin : doc['metric_data.begin'].value.millis;"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"source"} .= - "long end = doc['metric_data.end'].value.millis < params.end ? params.end : doc['metric_data.end'].value.millis;"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"script"}{"source"} .= - "return end - begin + 1;"; - } elsif ($mode eq 'fast') { - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"value"}{"field"} = "metric_data.value"; - $$req_ref{"aggs"}{"metric_avg"}{"weighted_avg"}{"weight"}{"field"} = "metric_data.duration"; - } -} - -sub get_many_metric_results_from_ids { - my $host = shift; - my $begin = shift; - my $end = shift; - my $resolution = shift; # the number of results within $begin and $end - my @metric_ids = @_; - my $num_metric_ids = scalar @metric_ids; - #printf "num_metric_ids: %d\n", $num_metric_ids; - my @values; # where we store the results for each slot in the time domain - my $msearch_req_json = ""; - #my $msearch_wavg_req_json = ""; - #my $msearch_partial_before_req_json = ""; - #my $msearch_partial_after_req_json = ""; - my $duration = int ($end - $begin) / $resolution; - my $this_begin = int $begin; - my $this_end = int $begin + $duration; - # Build the msearch to get a weighted average for each sample we need for $resolution - do { - # This first request is for the weighted average, but does not include the documents - # which are partially outside the time range we need. - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.end": { "lte": "' . $this_end . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.begin": { "gte": "' . $this_begin . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"terms": {"metric_data.id": ' . $coder->encode(\@metric_ids) . '}}'); - # Since we will only use the aggregation data, don't return any _source data - $$req_ref{"size"} = 0; - get_weighted_avg_agg($req_ref, $this_begin, $this_end, 'fast'); - my $index = get_index_basename . "metric_data"; - my $req_json = $coder->encode($req_ref); - $msearch_req_json .= '{"index" : "' . $index . '" }' . "\n"; - $msearch_req_json .= "$req_json\n"; - - # This second request is for the total weight of the previous weighted average. - # We need this because we are going to recompute the weighted average by adding - # a few more documents that are partially outside the time domain. - $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.end": { "lte": "' . $this_end . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.begin": { "gte": "' . $this_begin . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"terms": {"metric_data.id": ' . $coder->encode(\@metric_ids) . '}}'); - # Since we will only use the aggregation data, don't return any _source data - $$req_ref{"size"} = 0; - $$req_ref{'aggs'}{'total_weight'}{'sum'}{'field'} = "metric_data.duration"; - $index = get_index_basename . "metric_data"; - $req_json = $coder->encode($req_ref); - $msearch_req_json .= '{"index" : "' . $index . '" }' . "\n"; - $msearch_req_json .= "$req_json\n"; - - # This third request is for documents that had its begin during or before the time range, but - # its end was after the time range. - $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.end": { "gt": "' . $this_end . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.begin": { "lte": "' . $this_end . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"terms": {"metric_data.id": ' . $coder->encode(\@metric_ids) . '}}'); - $index = get_index_basename . "metric_data"; - $req_json = $coder->encode($req_ref); - $msearch_req_json .= '{"index" : "' . $index . '" }' . "\n"; - $msearch_req_json .= "$req_json\n"; - - # This fourth request is for documents that had its begin before the time range, but - # its end was during or after the time range. - $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.end": { "gte": "' . $this_begin . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"range": {"metric_data.begin": { "lt": "' . $this_begin . '"}}}'); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"terms": {"metric_data.id": ' . $coder->encode(\@metric_ids) . '}}'); - $index = get_index_basename . "metric_data"; - $req_json = $coder->encode($req_ref); - $msearch_req_json .= '{"index" : "' . $index . '" }' . "\n"; - $msearch_req_json .= "$req_json\n"; - - $this_begin = int $this_end + 1; - $this_end += int $duration + 1; - if ($this_end > $end) { - $this_end = int $end; - } - } until ($this_begin > $end); - my $resp_json = es_get_query($host, "metric_data/metric_data/_msearch", $msearch_req_json, "Msearch, aggregate weighted average script to get value"); - my $resp_ref = $coder->decode($resp_json); - #debug_log(Dumper $resp_ref); - $this_begin = int $begin; - $this_end = int $begin + $duration; - my $count = 0; - my $sets = 0; - my $elements = scalar @{ $$resp_ref{'responses'} }; - while ($count < $elements) { # Responses to be processed 4 at a time - my $time_window_duration = $this_end - $this_begin + 1; - my $total_weight_times_metrics = $time_window_duration * $num_metric_ids; - #printf "time_window_duration: %d\n", $time_window_duration; - #printf "total_weight for %d metrics ids: %d\n", $num_metric_ids, $total_weight_times_metrics; - $sets++; - my $agg_avg; - my $agg_weight; - my $agg_avg_times_weight; - my $new_weight; - $agg_avg = $$resp_ref{'responses'}[$count]{'aggregations'}{'metric_avg'}{'value'}; - if (defined $agg_avg) { - # We have the weighted average for documents that don't overlap the time range, - # but we need to combine that with the documents that are partially outside - # the time range. We need to know the total weight from the documents we - # just finished in order to add the new documents and recompute the new weighted - # average. - $agg_weight = $$resp_ref{'responses'}[$count+1]{'aggregations'}{'total_weight'}{'value'}; - #print "From aggregation using documents 100% within the time range:\n"; - #printf " average: %.2f\n", $agg_avg; - #$agg_avg *= $num_metric_ids; # The weighted avg will not add up from different metrics, so we must do it here - # This is for 'throughput' class metrics. For other classess, this may need to - # be handled differently. - #printf " average x num_metric_ids: %.2f\n", $agg_avg; - #printf " weight: %.2f\n", $agg_weight; - #$agg_weight *= $num_metric_ids; - #printf " weight x num_metric_ids: %.2f\n", $agg_weight; - $agg_avg_times_weight = $agg_avg * $agg_weight; - #printf " average x weight: %.2f\n", $agg_avg_times_weight; - } else { - # It is possible that the aggregation returned no results because all of the documents - # were partially outside the time domain. This can happen when - # 1) A metric does not change during the entire test, and therefore only 1 document - # is created with a huge duration with begin before the time range and after after the - # time range. - # 2) The time domain we have is really small because the resolution we are using is - # very big. - # - # In eithr case, we have to set the average and total_weight to 0, and then the - # recompuation of the weighted average [with the last two requests in this set, finding - # all of th docs that are partially in the time domain] will work. - $agg_avg = 0; - $agg_weight = 0; - $agg_avg_times_weight = 0; - } - # Process last 2 of the 4 response 'set' - # Since these docs have a time range partially outside the time range we want, - # we have to get a new, reduced duration and use that to agment our weighted average. - #print "From query using documents partially outside the time range:\n"; - my $sum_value_times_weight = 0; - my $sum_weight = 0; - # It is possible to have the same document returned from the last two queries in this set of 4. - # This can happen when the document's begin is before $this_begin *and* the document's end - # if after $this_end. - # You must not process the document twice. Perform a consolidation by organizing by the - # returned document's '_id' - my %partial_docs; - for (my $k = 2; $k < 4; $k++) { - for my $j (@{ $$resp_ref{'responses'}[$count + $k]{'hits'}{'hits'} }) { - for my $key (keys %{ $$j{'_source'}{'metric_data'} }) { - $partial_docs{$$j{'_id'}}{$key} = $$j{'_source'}{'metric_data'}{$key}; - } - } - } - # Now we can process %partial_docs - for my $id (keys %partial_docs) { - my $duration = $partial_docs{$id}{'duration'}; - if ($partial_docs{$id}{'begin'} < $this_begin) { - $duration -= $this_begin - $partial_docs{$id}{'begin'}; - } - if ($partial_docs{$id}{'end'} > $this_end) { - $duration -= $partial_docs{$id}{'end'} - $this_end; - } - my $value_times_weight = $partial_docs{$id}{'value'} * $duration; - $sum_value_times_weight += $value_times_weight; - $sum_weight += $duration; - } - my $result = ($agg_avg_times_weight + $sum_value_times_weight) / ($total_weight_times_metrics); - #printf "agg_avg: %.2f agg_weight %d\n", $agg_avg, $agg_weight; - #printf "extra_avg: %.2f extra_weight %d\n", $sum_value_times_weight / $sum_weight, $sum_weight; - #printf "weighted avg: %.2f\n", $result; - $result *= $num_metric_ids; - $result=FormatSigFigs($result,4); - my %data_sample = ( 'begin' => $this_begin, 'end' => $this_end, 'value' => $result ); - #printf "weighted avg x num_metric_ids: %.2f\n", $result; - #push @values, $result; - push @values, \%data_sample; - $count += 4; - $this_begin = int $this_end + 1; - $this_end += int $duration + 1; - if ($this_end > $end) { - $this_end = int $end; - } - } - debug_log("processed $sets\n"); - return @values; -} - -sub get_metric_data { - my $host = shift; - my $data_ref = shift; - my $begin = shift; - my $end = shift; - my $period_id = shift; - my $metric_source = shift; - my $metric_type = shift; - my $resolution = shift; - my @breakout = @_; - - # If you are wanting just a single metric (no break-out, no data-series), then - # $resolution should be 1 and $name_format should be blank. - - $$data_ref{'breakouts'} = ''; - $$data_ref{'values'} = (); - my $aggs_json = get_aggregation_from_breakout($metric_source, $metric_type, @breakout); - my $aggs_ref = $coder->decode($aggs_json); - # create a new query with this aggregation - my $metr_req_ref = $coder->decode($template); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.id": "' . $period_id . '"}}'); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.source": "' . $metric_source . '"}}'); - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.type": "' . $metric_type . '"}}'); - while (scalar @breakout > 0) { - my $field = shift(@breakout); - my $value; - #if ($field =~ /([^\=]+)\=([^\=]+)/) { - if ($field =~ /([^\=]+)\=(.+)/) { - $field = $1; - $value = $2; - push @{ $$metr_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"metric_desc.names.' . $field . '": "' . $value . '"}}'); - } - } - $$metr_req_ref{"aggs"} = $$aggs_ref{"aggs"}; - $$metr_req_ref{"size"} = 0; # We don't need the source - my $metr_req_json = $coder->encode($metr_req_ref); - my $metr_resp = es_get_query($host, "metric_desc/metric_desc/_search", $metr_req_json, "Query to get all the metric groups (fio: result data per-host, per-job, per-IOtype)"); - #print $metr_resp; - my $metr_resp_ref = $coder->decode($metr_resp); - # by traversing the aggrgeations in the response, find all of the different metric groups - # (each group represents a single timeseries). - # - # First traverse the aggregation response to build an array of filter_terms needed [in a query] to get the metric_desc document(s) - my @metr_grp_terms = gen_metric_group_terms($$metr_resp_ref{"aggregations"}, ""); - #print "Metric search terms by group:\n", Dumper \@metr_grp_terms; - # Maintain a hash of metric-group label (the name_format with values swapped in for field_names) to the filter terms query - my %metr_terms_by_label; - gen_label_to_terms(\%metr_terms_by_label, @metr_grp_terms); - for my $label (keys %metr_terms_by_label) { - my @metric_ids = get_metric_ids_from_term($host, $period_id, $metr_terms_by_label{$label}); - my $duration = int ($end - $begin) / $resolution; - my $this_begin = int $begin; - my $this_end = int $begin + $duration; - my @results = get_many_metric_results_from_ids($host, $begin, $end, $resolution, @metric_ids); - $$data_ref{'values'}{$label} = \@results; - } - @breakout = sort(@breakout); - my @metric_field_names = sort(get_name_format_field_names(get_name_format($host, $period_id, $metric_source, $metric_type))); - my $diffed_field_names = Array::Diff->diff(\@breakout, \@metric_field_names); - $$data_ref{'breakouts'} = $diffed_field_names->added; -} - -# Build the aggregation-portion of a query (only agg, not including the boolean-filter-terms here) -# based on a metric_source, metric_type, and either all or a *subset* of the name_format -# that this metric uses. For example, fio has one [of its many avail metrics] that has: -# metric_source: fio, metric_type: iops, name_format: %host%-%job%-%type% (type means io-type) -# if you call this function with the name_format = %host%-%job%-%type%, it will -# produce an aggregaton request that [when requested] will produce a nested response where -# every single of fio's host/job/[io-]type is returned individually -the deepest "bucket" -# in the response will have exactly one quantity. -# -# However, if you were to use name_format = %host%-%job%, metrics from the same host and -# same job, but from different io-types would be grouped as one. -# -# Continue this trend and use name_format = %host%, and metric from the same host, but different -# jobs and io-types are grouped as one. -# -# Once you reach the point where name_format is blank, then all metrics are grouped as one. -# If you did not want to break-out the metric by anything at all, this is what you would do. -sub get_aggregation_from_name_format { - my $name_format = shift; - my $metric_source = shift; - my $metric_type = shift; - my $aggs_json = '{'; - # First build the nested aggregation first based on the metric_source and metric_type. - # We do this in case there is no name_format provided (there is no break-out) for - # the metrics, and we can still construct an aggregation. The other option would - # be to have no aggregaton at all, but then we need special code for that case, - # and it becomes more code mgmt, and not worth it. - $aggs_json .= '"aggs": { "metric_desc.source": { "terms": { "field": "metric_desc.source" },'; - $aggs_json .= '"aggs": { "metric_desc.type": { "terms": { "field": "metric_desc.type" }'; - # Next build the nested aggregation based on the field names found in the name_format - my @field_names; - my $field_count = 0; - while ( $name_format =~ /\%\S+\%/ ) { - $name_format =~ s/([^\%]*)\%([^\%]+)\%(\.*)/$3/; - my $field = $2; - # strip off the '=value' if any - $field =~ s/([\^=]+)\=([\^=]+)/$1/; - push @field_names, $field; - $field_count++; - if ($field_count > 0) { - $aggs_json .= ','; - } - # Build the nested aggregation: note the fewer number of }'s - # The 'size' is bumped up to support much bigger aggregations - # we need for large, nested metrics like CPU util for sar/mpstat - # and per-PID CPU usage for pidstat. Eventually this should be - # 'composite' aggregations (if it can work while nested) or - # mutliple queries. - $aggs_json .= '"aggs": { "metric_desc.names.' . $2 . '": { "terms":' . - ' { "show_term_doc_count_error": true, "size": 1000,' . - ' "field": "metric_desc.names.' . $2 . '" }'; - } - while ($field_count > 0) { - $aggs_json .= "}}"; - $field_count--; - } - $aggs_json .= '}}}}}'; # extra closing brackets for metric_source/type aggs - return $aggs_json; -} -sub get_aggregation_from_breakout { - my $metric_source = shift; - my $metric_type = shift; - my @breakout = @_; - my $aggs_json = '{'; - # First build the nested aggregation first based on the metric_source and metric_type. - # We do this in case there is no name_format provided (there is no break-out) for - # the metrics, and we can still construct an aggregation. The other option would - # be to have no aggregaton at all, but then we need special code for that case, - # and it becomes more code mgmt, and not worth it. - $aggs_json .= '"aggs": { "metric_desc.source": { "terms": { "field": "metric_desc.source" },'; - $aggs_json .= '"aggs": { "metric_desc.type": { "terms": { "field": "metric_desc.type" }'; - # Next build the nested aggregation based on the field names found in the name_format - my $field_count = 0; - while (scalar @breakout > 0) { - my $field = shift(@breakout); - if ($field =~ /([^\=]+)\=([^\=]+)/) { - $field = $1; - } - $field_count++; - if ($field_count > 0) { - $aggs_json .= ','; - } - # Build the nested aggregation: note the fewer number of }'s - # The 'size' is bumped up to support much bigger aggregations - # we need for large, nested metrics like CPU util for sar/mpstat - # and per-PID CPU usage for pidstat. Eventually this should be - # 'composite' aggregations (if it can work while nested) or - # mutliple queries. - $aggs_json .= '"aggs": { "metric_desc.names.' . $field . '": { "terms":' . - ' { "show_term_doc_count_error": true, "size": 1000,' . - ' "field": "metric_desc.names.' . $field . '" }'; - } - while ($field_count > 0) { - $aggs_json .= "}}"; - $field_count--; - } - $aggs_json .= '}}}}}'; # extra closing brackets for metric_source/type aggs - return $aggs_json; -} - -# Find (recursively) all of the metric groups that exist for this name_format, return the -# filter terms that [when used in a query] could return the metric_desc document(s). -# This requires that you run get_aggregation_from_name_format() first, then run the -# query based on the nested aggregation returned from get_aggregation_from_name_format(), and then -# finally, you can call this function with the *response* from that query (but only the aggregation -# part of the response) -sub gen_metric_group_terms { - my $pointer = shift; # The aggregations section in the *response* from aggregation query that get_aggregation_from_name_format() formed. - my $terms = shift; # The filter_terms that would return the correct metric_desc doc for this metric group (initially should be empty). - # These terms get constructed as this function is recursively called. - my @metr_grp_queries; - my $value; - # If being called from a previous gen_metric_group_labels_terms, finish the second part of the terms query - if (exists $$pointer{"key"}) { - # This is the value for a field name from previous call to gen_metric_group_labels_terms() (see push statement below) - $value = $$pointer{"key"}; - $terms .= '"' . $value . '"}}'; - } - if (my @metr_desc = grep(/^metric_desc/, (keys %$pointer))) { - die 'Something went wrong, found more than one "metric_desc." in this scope of the aggregation response' if (scalar @metr_desc > 1); - my $field = $metr_desc[0]; - if (exists $$pointer{$field} and exists $$pointer{$field}{"buckets"}) { - foreach my $bucket (@{ $$pointer{$field}{"buckets"} }) { - # Only the first half of the "term", the field name, is here because we need to go one level deeper to get - # the value, in each of the buckets (one level deeper). - push @metr_grp_queries, gen_metric_group_terms($bucket, $terms . "," . '{"term": {"' . $field . '": '); - } - } - return @metr_grp_queries; - } else { - $terms =~ s/^,//; - return $terms; - } -} - -sub gen_metric_source_types { - my $pointer = shift; # The aggregations section in the *response* from aggregation query that get_aggregation_from_name_format() formed. - my $terms = shift; # The filter_terms that would return the correct metric_desc doc for this metric group (initially should be empty). - # These terms get constructed as this function is recursively called. - my @metr_grp_queries; - my $value; - # If being called from a previous gen_metric_group_labels_terms, finish the second part of the terms query - if (exists $$pointer{"key"}) { - # This is the value for a field name from previous call to gen_metric_group_labels_terms() (see push statement below) - $value = $$pointer{"key"}; - $terms .= "-" . $value; - } - if (my @metr_desc = grep(/^metric_desc/, (keys %$pointer))) { - die 'Something went wrong, found more than one "metric_desc." in this scope of the aggregation response' if (scalar @metr_desc > 1); - my $field = $metr_desc[0]; - if (exists $$pointer{$field} and exists $$pointer{$field}{"buckets"}) { - foreach my $bucket (@{ $$pointer{$field}{"buckets"} }) { - # Only the first half of the "term", the field name, is here because we need to go one level deeper to get - # the value, in each of the buckets (one level deeper). - push @metr_grp_queries, gen_metric_source_types($bucket, $terms); - } - } - return @metr_grp_queries; - } else { - $terms =~ s/^,//; - return $terms; - } -} - -sub gen_label_to_terms { - my $metr_terms_by_label_ref = shift; - my @metr_grp_terms = @_; - # Example terms that might be used: - # - # [ - # {"term": {"metric_desc.source": "fio"}},{"term": {"metric_desc.type": "iops"}} - # ] - # - # [ - # '{"term": {"metric_desc.source": "fio"}},{"term": {"metric_desc.type": "iops"}},{"term": {"metric_desc.names.host": "perf84"}},{"term": {"metric_desc.names.job": "1"}},{"term": {"metric_desc.names.type": "0"}}', - # '{"term": {"metric_desc.source": "fio"}},{"term": {"metric_desc.type": "iops"}},{"term": {"metric_desc.names.host": "perf84"}},{"term": {"metric_desc.names.job": "2"}},{"term": {"metric_desc.names.type": "0"}}', - # '{"term": {"metric_desc.source": "fio"}},{"term": {"metric_desc.type": "iops"}},{"term": {"metric_desc.names.host": "perf84"}},{"term": {"metric_desc.names.job": "3"}},{"term": {"metric_desc.names.type": "0"}}' - # ] - for my $term_json (@metr_grp_terms) { - my $term_ref = $coder->decode('[' . $term_json . ']'); - my $label = ""; - for my $this_term (@$term_ref) { - next if $$this_term{'term'}{'metric_desc.source'} or $$this_term{'term'}{'metric_desc.type'}; - for my $field_name (keys %{ $$this_term{"term"} }) { # Should be one field name - $label .= '-' . $$this_term{"term"}{$field_name}; - } - } - $label =~ s/^-//; - $$metr_terms_by_label_ref{$label} = $term_json; - } -} - -# With a period_id, query the metric_desc docs to return -# all unique combinations of 'metric_desc.source'-'metric_desc.type'. -# This represents all of the metrics available during this -# part of the benchmark-iteration-sample-period. -sub list_all_metrics { - my $host = shift; - my $begin = shift; - my $end = shift; - my $resolution = shift; - my $period_id = shift; - my $aggs_json = '{"aggs": { "metric_desc.source": { "terms": { "field": "metric_desc.source" },'; - $aggs_json .= '"aggs": { "metric_desc.type": { "terms": { "field": "metric_desc.type" }}}}}}'; - my $aggs_ref = $coder->decode($aggs_json); - my $req_ref = $coder->decode($template); - push @{ $$req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"period.id": "' . $period_id . '"}}'); - $$req_ref{"aggs"} = $$aggs_ref{"aggs"}; - $$req_ref{"size"} = 0; # We don't need the source - my $req_json = $coder->encode($req_ref); - my $resp_json = es_get_query($host, "metric_desc/metric_desc/_search", $req_json, "Query to aggregate all the metrics available"); - my $resp_ref = $coder->decode($resp_json); - # The aggregations response is only 2 levels deeps, so we just - # process here and don't use recursion like gen_metric_group_terms() - for my $i (@{ $$resp_ref{'aggregations'}{'metric_desc.source'}{'buckets'} }) { - my $source = $$i{'key'}; - printf "source: %s\n", $source; - for my $j (@{ $$i{'metric_desc.type'}{'buckets'} }) { - my $type = $$j{'key'}; - my $name_format = get_name_format($host, $period_id, $source, $type); - my %data = (); - get_metric_data($host, \%data, $begin, $end, $period_id, $source, $type, $resolution); - my @field_names = get_name_format_field_names($name_format); - printf " type: %s avail-break-outs: %s values:\n", $type, join(" ", @field_names); - print " ", $coder->encode($data{'values'}{""}), "\n"; - #print Dumper \%data; - #for my $label (sort keys %{ $data{'values'} }) { - #$printf " %s: %s\n", $label, join(" ", @{ $data{'values'}{$label} }); - #} - } - - } -} - -1; diff --git a/agent/rpm/pbench-agent.spec.j2 b/agent/rpm/pbench-agent.spec.j2 index e4b55cf634..40dae87f48 100644 --- a/agent/rpm/pbench-agent.spec.j2 +++ b/agent/rpm/pbench-agent.spec.j2 @@ -83,14 +83,6 @@ Requires: bc, bzip2, hostname, iproute, iputils, net-tools, numactl Requires: openssh-clients, openssh-server, procps-ng, psmisc, redis Requires: rpmdevtools, rsync, screen, sos, tar, xz -# The condition here should be 7 OR 8.4 OR 8.5 but we don't have a way to deal with minor versions. -# The resulting RPM will NOT work on RHEL8.[45]. -%if 0%{?rhel} == 7 -Requires: ansible -%else -Requires: ansible-core -%endif - Obsoletes: pbench <= 0.34 Conflicts: pbench <= 0.34 # configtools is packaged with pbench-agent, so we specifically do NOT want diff --git a/agent/tool-scripts/postprocess/iostat-postprocess b/agent/tool-scripts/postprocess/iostat-postprocess index 84fc4806e5..1d3496ed09 100755 --- a/agent/tool-scripts/postprocess/iostat-postprocess +++ b/agent/tool-scripts/postprocess/iostat-postprocess @@ -7,23 +7,14 @@ # CSV files and statically generated graphs # with embedded javascript, using d3.js libraries # -# or -# -# CommonDataModel documents -# # Usage: # -# iostat-postprocess [ ] +# iostat-postprocess # # dir = directory where iostat-stdout.txt can be found -# period-doc = path to CDM period document for this iteration-sample -# es-dir = path to base directory where CDM documents should be written -# hostname = the host where mpstat was run # # Note: # -# arguments 2 - 4 are only used when generating CommonDataModel -# # iostat-stdout.txt must be generated from "iostat -N -t -y -x -m " use strict; @@ -31,12 +22,6 @@ use warnings; use lib $ENV{'pbench_lib_dir'}; no lib "."; -BEGIN { - if (scalar @ARGV > 1) { - require PbenchCDM; - PbenchCDM->import( qw(log_cdm_metric_sample gen_cdm_metric_data) ); - } -} use GenData qw(gen_data); my $dir = shift; @@ -45,22 +30,7 @@ die "Non-default options, skipping post-processing for iostat" if (-f $dir . "/i my $dev; my %iostat; # For gen_data() -my %iostat_cdm; # For log_cdm_metric_sample() my $timestamp_ms; -my $es_dir; -my $hostname; -my $period_doc_path; -my $cdm = 0; -# When producing CDM docs, this is typically called from the pbench -# controller only, and the following arguments must be provided: -if (scalar @ARGV > 1) { - #printf "num args: %d args: %s\n", scalar @ARGV, join(" ", @ARGV); - die "Must provide " if (scalar @ARGV != 3); - $cdm = 1; - $period_doc_path = shift; - $es_dir = shift; - $hostname = shift; -} # read the iostat-stdout.txt open(IOSTAT_TXT, $dir . "/iostat-stdout.txt") || die "could not find " . $dir . "/iostat-stdout.txt\n"; @@ -90,59 +60,38 @@ while (my $line = ) { my %rw_stats = ( 'throughput' => \%rw_tput_stats, 'count' => \%rw_count_stats ); my %other_count_stats = ( 'QueueSize' => $12, 'Utilization' => $16 ); - if ($cdm) { - for my $class (keys %rw_stats) { - for my $type (keys %{ $rw_stats{$class} }) { - for my $rw (keys %{ $rw_stats{$class}{$type} }) { - my %md = ( 'host' => $hostname, 'type' => $rw, 'dev' => $dev ); - log_cdm_metric_sample('iostat', 'throughput', $type, '%host%-%dev%-%type%', - \%md, \%iostat_cdm, $timestamp_ms, $rw_stats{$class}{$type}{$rw}); - } - } - } - for my $type (keys %other_count_stats) { - my %md = ( 'host' => $hostname, 'dev' => $dev ); - log_cdm_metric_sample('iostat', 'count', $type, '%host%-%dev%', - \%md, \%iostat_cdm, $timestamp_ms, $other_count_stats{$type}); - } - } else { - # Each data series is for one device (sda). On some graphs, read and write - # series are included in to one graph (instead of having one read-ops chart - # and another write-ops chart), so each series needs "-read" or - # "-write" appended to keep them distinct. - $iostat{disk}{"Utilization_percent"}{$dev}{$timestamp_ms} = $other_count_stats{'Utilization'}; - $iostat{disk}{"Throughput_MB_per_sec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'MegabytesSec'}{'Read'}; - $iostat{disk}{"Throughput_MB_per_sec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'MegabytesSec'}{'Write'}; - $iostat{disk}{"Request_Merges_per_sec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'RequestMergesSec'}{'Read'}; - $iostat{disk}{"Request_Merges_per_sec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'RequestMergesSec'}{'Write'}; - $iostat{disk}{"IOPS"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'IOPS'}{'Read'}; - $iostat{disk}{"IOPS"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'IOPS'}{'Write'}; - $iostat{disk}{"Request_Size_in_512_byte_sectors"}{$dev}{$timestamp_ms} = ($rw_stats{'count'}{'RequestSizeKB'}{'Read'} - + $rw_stats{'count'}{'RequestSizeKB'}{'Write'}) * 2; - $iostat{disk}{"Request_Size_in_kB"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'count'}{'RequestSizeKB'}{'Read'}; - $iostat{disk}{"Request_Size_in_kB"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'count'}{'RequestSizeKB'}{'Write'}; - $iostat{disk}{"Queue_Size"}{$dev}{$timestamp_ms} = $other_count_stats{'QueueSize'}; - $iostat{disk}{"Wait_Time_msec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'count'}{'RequestWaitMs'}{'Read'}; - $iostat{disk}{"Wait_Time_msec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'count'}{'RequestWaitMs'}{'Write'}; - } + # Each data series is for one device (sda). On some graphs, read and write + # series are included in to one graph (instead of having one read-ops chart + # and another write-ops chart), so each series needs "-read" or + # "-write" appended to keep them distinct. + $iostat{disk}{"Utilization_percent"}{$dev}{$timestamp_ms} = $other_count_stats{'Utilization'}; + $iostat{disk}{"Throughput_MB_per_sec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'MegabytesSec'}{'Read'}; + $iostat{disk}{"Throughput_MB_per_sec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'MegabytesSec'}{'Write'}; + $iostat{disk}{"Request_Merges_per_sec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'RequestMergesSec'}{'Read'}; + $iostat{disk}{"Request_Merges_per_sec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'RequestMergesSec'}{'Write'}; + $iostat{disk}{"IOPS"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'throughput'}{'IOPS'}{'Read'}; + $iostat{disk}{"IOPS"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'throughput'}{'IOPS'}{'Write'}; + $iostat{disk}{"Request_Size_in_512_byte_sectors"}{$dev}{$timestamp_ms} = ($rw_stats{'count'}{'RequestSizeKB'}{'Read'} + + $rw_stats{'count'}{'RequestSizeKB'}{'Write'}) * 2; + $iostat{disk}{"Request_Size_in_kB"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'count'}{'RequestSizeKB'}{'Read'}; + $iostat{disk}{"Request_Size_in_kB"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'count'}{'RequestSizeKB'}{'Write'}; + $iostat{disk}{"Queue_Size"}{$dev}{$timestamp_ms} = $other_count_stats{'QueueSize'}; + $iostat{disk}{"Wait_Time_msec"}{$dev . "-read"}{$timestamp_ms} = $rw_stats{'count'}{'RequestWaitMs'}{'Read'}; + $iostat{disk}{"Wait_Time_msec"}{$dev . "-write"}{$timestamp_ms} = $rw_stats{'count'}{'RequestWaitMs'}{'Write'}; } } close(IOSTAT_TXT); -if ($cdm) { - gen_cdm_metric_data(\%iostat_cdm, $period_doc_path, $es_dir, $hostname, "iostat"); -} else { - # define the graph types - # if you want something other than lineChart, put it here - my %graph_type; - # threshold for displying a series in a graph - my %graph_threshold; - $graph_threshold{disk}{"Utilization_percent"} = 1; - $graph_threshold{disk}{"Throughput_MB_per_sec"} = 1; - $graph_threshold{disk}{"IOPS"} = 1; - $graph_threshold{disk}{"Wait_Time_msec"} = 0.1; - $graph_threshold{disk}{"Request_Size_in_512_byte_sectors"} = 0.1; - $graph_threshold{disk}{"Request_Size_in_kB"} = 0.1; - $graph_threshold{disk}{"Request_Merges_per_sec"} = 0.1; - $graph_threshold{disk}{"Queue_Size"} = 0.01; - gen_data(\%iostat, \%graph_type, \%graph_threshold, $dir); -} +# define the graph types +# if you want something other than lineChart, put it here +my %graph_type; +# threshold for displying a series in a graph +my %graph_threshold; +$graph_threshold{disk}{"Utilization_percent"} = 1; +$graph_threshold{disk}{"Throughput_MB_per_sec"} = 1; +$graph_threshold{disk}{"IOPS"} = 1; +$graph_threshold{disk}{"Wait_Time_msec"} = 0.1; +$graph_threshold{disk}{"Request_Size_in_512_byte_sectors"} = 0.1; +$graph_threshold{disk}{"Request_Size_in_kB"} = 0.1; +$graph_threshold{disk}{"Request_Merges_per_sec"} = 0.1; +$graph_threshold{disk}{"Queue_Size"} = 0.01; +gen_data(\%iostat, \%graph_type, \%graph_threshold, $dir); diff --git a/agent/tool-scripts/postprocess/iostat-postprocess-cdm b/agent/tool-scripts/postprocess/iostat-postprocess-cdm deleted file mode 120000 index bae60ef173..0000000000 --- a/agent/tool-scripts/postprocess/iostat-postprocess-cdm +++ /dev/null @@ -1 +0,0 @@ -iostat-postprocess \ No newline at end of file diff --git a/agent/tool-scripts/postprocess/mpstat-postprocess b/agent/tool-scripts/postprocess/mpstat-postprocess index 2bb022362b..6d21571f35 100755 --- a/agent/tool-scripts/postprocess/mpstat-postprocess +++ b/agent/tool-scripts/postprocess/mpstat-postprocess @@ -8,23 +8,14 @@ # CSV files and statically generated graphs # with embedded javascript, using d3.js libraries # -# or -# -# CommonDataModel documents -# # Usage: # -# mpstat-postprocess [ ] +# mpstat-postprocess # # dir = directory where mpstat-stdout.txt can be found -# period-doc = path to CDM period document for this iteration-sample -# es-dir = path to base directory where CDM documents should be written -# hostname = the host where mpstat was run # # Note: # -# arguments 2 - 4 are only used when generating CommonDataModel -# # mpstat-stdout.txt must be generated from "mpstat -P ALL " use strict; @@ -32,12 +23,6 @@ use warnings; use lib $ENV{'pbench_lib_dir'}; no lib "."; -BEGIN { - if (scalar @ARGV > 1) { - require PbenchCDM; - PbenchCDM->import( qw(log_cdm_metric_sample gen_cdm_metric_data) ); - } -} use GenData qw(gen_data); use SysStat qw(get_mpstat_cpumode_attributes build_cpu_topology get_cpu_socket_core); @@ -48,7 +33,6 @@ die "Non-default options, skipping post-processing for mpstat" if (-f $dir . "/m my $date_ms; my $prev_time = ""; my %online_cpus; -my $hostname; # for CDM # We'll force mpstat into a LANG=C environment: two-digit year on the first line, # 24-hour time with *no* AM/PM designation in the timestamp. @@ -106,27 +90,12 @@ sub cpu_is_online { } my %mpstat; # For gen_data() -my %mpstat_cdm; # For log_cdm_metric_sample() my $timestamp_ms; -my $es_dir; # for CDM -my $period_doc_path; # for CDM -my @cpu_topology; # for CDM -my $cdm = 0; my @cpumode_attributes = get_mpstat_cpumode_attributes; my @nonbusy = (qw(idle iowait steal)); my @busy = grep(!/^idle?|^iowait?|^steal?/, @cpumode_attributes); my %cpu_modes = ( 'idle' => \@nonbusy, 'busy' => \@busy ); -# When producing CDM docs, this is typically called from the pbench -# controller only, and the following arguments must be provided: -if (scalar @ARGV > 1) { - die "Must provide " if (scalar @ARGV != 3); - $cdm = 1; - $period_doc_path = shift; - $es_dir = shift; - $hostname = shift; - build_cpu_topology(\@cpu_topology, $dir); -} open(MPSTAT_TXT, "$dir/mpstat-stdout.txt") || die "could not find $dir/mpstat-stdout.txt\n"; # All mpstat output processing assumes LANG=C environment when mpstat was run. # @@ -156,47 +125,16 @@ while (my $line = ) { my %cpumode_util; @cpumode_util{@cpumode_attributes} = @values; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - if ($cpuid ne "all") { - # Note that an "all" cpu metric is not logged - # with CDM like with traditional pbench. The - # "all" metric represents all CPUs in a host - # as a range of 0-100%. We do not represent - # CPU utilization this way in CDM. Instead, - # we calculate system CPU by aggregating per- - # CPU stats into a single stat, dynamically, - # via the Web UI. A per-system CPU utiliza- - # tion will be from 0 to (100 * nr_cpus) - # instead of 0-100%. - # - # Addtionally, cpu busy and idle stats have - # different name_format, so that only busy or - # only idle stats are aggregated. Allowing - # aggregation of all the modes (idle and busy) - # together makes little sense, since it will - # just add up to 100. - (my $socket, my $core) = get_cpu_socket_core($cpuid, \@cpu_topology); - for my $util_type (keys %cpu_modes ) { - my $type = "Processor" . ucfirst $util_type . "Util"; - for my $mode (@{ $cpu_modes{$util_type} }) { - my %md = ( 'host' => $hostname, 'socket' => $socket, 'core' => $core, 'id' => $cpuid, 'mode' => $mode ); - log_cdm_metric_sample('mpstat', 'throughput', $type, '%host%-%socket%:%core%:%id%-%mode%', - \%md, \%mpstat_cdm, $timestamp_ms, $cpumode_util{$mode}); - } - } - } - } else { - for my $mode (@cpumode_attributes) { - next if ($mode eq "gnice"); - my $mode_label; - if ($mode eq 'soft') { # Stay compatible with legacy label - $mode_label = 'softirq'; - } else { - $mode_label = $mode; - } - my $cpu_label = "cpu" . $cpuid; - $mpstat{$cpu_label}{$cpu_label}{$mode_label}{$timestamp_ms} = $cpumode_util{$mode}; + for my $mode (@cpumode_attributes) { + next if ($mode eq "gnice"); + my $mode_label; + if ($mode eq 'soft') { # Stay compatible with legacy label + $mode_label = 'softirq'; + } else { + $mode_label = $mode; } + my $cpu_label = "cpu" . $cpuid; + $mpstat{$cpu_label}{$cpu_label}{$mode_label}{$timestamp_ms} = $cpumode_util{$mode}; } } elsif ( $line =~ /(\d\d:\d\d:\d\d)\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+.*/ ) { # Run the timestamp calculation since the first head might @@ -205,18 +143,13 @@ while (my $line = ) { } } close(MPSTAT_TXT); -if ($cdm) { - print "cdm\n"; - gen_cdm_metric_data(\%mpstat_cdm, $period_doc_path, $es_dir, $hostname, "mpstat"); -} else { - my $file; - my $graph; - my %graph_threshold; - my %graph_type; - foreach $file (keys %mpstat) { - foreach $graph (keys %{ $mpstat{$file} }) { - $graph_type{$file}{$graph} = "stackedAreaChart"; - } +my $file; +my $graph; +my %graph_threshold; +my %graph_type; +foreach $file (keys %mpstat) { + foreach $graph (keys %{ $mpstat{$file} }) { + $graph_type{$file}{$graph} = "stackedAreaChart"; } - gen_data(\%mpstat, \%graph_type, \%graph_threshold, $dir); } +gen_data(\%mpstat, \%graph_type, \%graph_threshold, $dir); diff --git a/agent/tool-scripts/postprocess/mpstat-postprocess-cdm b/agent/tool-scripts/postprocess/mpstat-postprocess-cdm deleted file mode 120000 index f56579dea0..0000000000 --- a/agent/tool-scripts/postprocess/mpstat-postprocess-cdm +++ /dev/null @@ -1 +0,0 @@ -mpstat-postprocess \ No newline at end of file diff --git a/agent/tool-scripts/postprocess/pidstat-postprocess b/agent/tool-scripts/postprocess/pidstat-postprocess index 95603147b9..65289f51ee 100755 --- a/agent/tool-scripts/postprocess/pidstat-postprocess +++ b/agent/tool-scripts/postprocess/pidstat-postprocess @@ -9,23 +9,14 @@ # CSV files and statically generated graphs # with embedded javascript, using d3.js libraries # -# or -# -# CommonDataModel documents -# # Usage: # -# pidstat-postprocess [ ] +# pidstat-postprocess # # dir = directory where pidstat-stdout.txt can be found -# period-doc = path to CDM period document for this iteration-sample -# es-dir = path to base directory where CDM documents should be written -# hostname = the host where pidstat was run # # Note: # -# arguments 2 - 4 are only used when generating CommonDataModel -# # pidstat output must be generated with "pidstat -l -H -w -u -h -d -r [-t] -p ALL" # and piped to pidstat-convert @@ -36,22 +27,12 @@ use lib $ENV{'pbench_lib_dir'}; no lib "."; use GenData qw(gen_data); use SysStat qw(get_pidstat_attributes); -BEGIN { - if (scalar @ARGV > 1) { - require PbenchCDM; - PbenchCDM->import( qw(log_cdm_metric_sample gen_cdm_metric_data) ); - } -} my $dir = shift; die "Non-default options, skipping post-processing for pidstat" if (-f $dir . "/pidstat.options"); my %pidstat; -my $cdm = 0; -my $es_dir; -my $period_doc_path; -my $hostname; my %graph_type; my %graph_threshold; my $vm_name; @@ -60,35 +41,10 @@ my $line2; my $tid; my @prev_stats; my @curr_stats; -my $interval; # sampling interval my $threads; -# When producing CDM docs, this is typically called from the pbench -# controller only, and the following arguments must be provided: -if (scalar @ARGV > 2) { - #printf "num args: %d args: %s\n", scalar @ARGV, join(" ", @ARGV); - die "Must provide " if (scalar @ARGV != 3); - $cdm = 1; - $period_doc_path = shift; - $es_dir = shift; - $hostname = shift; - my $pidstat_cmd_file = $dir . "/pidstat.cmd"; - open (FH, $pidstat_cmd_file) || die "Could not open $pidstat_cmd_file"; - while ($line = ) { - #LANG=C /usr/local/bin/pidstat -l -H -w -u -h -d -r -p ALL 1 - if ($line =~ /-p\sALL\s+(\d+)/ ){ - $interval = $1; - } - } - close FH; -} - my %timestamps; my @attributes = get_pidstat_attributes(); -if ($cdm and not defined $interval) { - print "Could not determine the sampling interval for pidstat\n"; - exit 1; -} my $pids_dir = $dir . "/pids"; if (opendir(my $pids_dh, $pids_dir) || die "Could not open $pids_dir") { for my $pid (grep(/\d+/, readdir($pids_dh))) { @@ -154,36 +110,15 @@ if (opendir(my $pids_dh, $pids_dir) || die "Could not open $pids_dir") { my %noncpu_tput = ( 'FileIoKbSec' => \%file_io_kb_sec, 'MemoryFaultsSec' => \%memory_faults_sec, 'ContextSwitchesSec' => \%ctxsw_sec ); my %memory_usage_kb = ( 'Resident' => $stats{'RSS'}, 'NonResident' => $stats{'VSZ'} - $stats{'RSS'} ); - if ($cdm) { - for my $metric_type (keys %noncpu_tput) { - for my $type (keys %{ $noncpu_tput{$metric_type} }) { - my %md = ( 'host' => $hostname, 'type' => $type, 'pid' => $pid, 'cmd' => $cmd ); - log_cdm_metric_sample('pidstat', 'throughput', $metric_type, '%host%-%pid%-%cmd%-%type%', - \%md, \%pidstat, $timestamp_ms, $noncpu_tput{$metric_type}{$type}, - $interval); - } - } - for my $type (keys %memory_usage_kb) { - my %md = ( 'host' => $hostname, 'type' => $type, 'pid' => $pid, 'cmd' => $cmd ); - log_cdm_metric_sample('pidstat', 'count', 'MemoryUsageKb', '%host%-%pid%-%cmd%-%type%', - \%md, \%pidstat, $timestamp_ms, $memory_usage_kb{$type}, - $interval); - } - my %md = ( 'host' => $hostname, 'pid' => $pid, 'cmd' => $cmd ); - log_cdm_metric_sample('pidstat', 'throughput', 'PercentCPUutil', '%host%-%pid%-%cmd%', - \%md, \%pidstat, $timestamp_ms, $stats{'CPU_PCT'}, - $interval); - } else { - $pidstat{'cpu_usage'}{'percent_cpu'}{$pid_cmd}{$timestamp_ms} = $stats{'CPU_PCT'}; - $pidstat{'file_io'}{'io_reads_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_rd_s'}; - $pidstat{'file_io'}{'io_writes_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_wr_s'}; - $pidstat{'memory_faults'}{'minor_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'minflt_s'}; - $pidstat{'memory_faults'}{'major_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'majflt_s'}; - $pidstat{'context_switches'}{'voluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'cswch_s'}; - $pidstat{'context_switches'}{'nonvoluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'nvcswch_s'}; - $pidstat{'memory_usage'}{'virtual_size'}{$pid_cmd}{$timestamp_ms} = $stats{'VSZ'}; - $pidstat{'memory_usage'}{'resident_set_size'}{$pid_cmd}{$timestamp_ms} = $stats{'RSS'}; - } + $pidstat{'cpu_usage'}{'percent_cpu'}{$pid_cmd}{$timestamp_ms} = $stats{'CPU_PCT'}; + $pidstat{'file_io'}{'io_reads_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_rd_s'}; + $pidstat{'file_io'}{'io_writes_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_wr_s'}; + $pidstat{'memory_faults'}{'minor_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'minflt_s'}; + $pidstat{'memory_faults'}{'major_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'majflt_s'}; + $pidstat{'context_switches'}{'voluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'cswch_s'}; + $pidstat{'context_switches'}{'nonvoluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'nvcswch_s'}; + $pidstat{'memory_usage'}{'virtual_size'}{$pid_cmd}{$timestamp_ms} = $stats{'VSZ'}; + $pidstat{'memory_usage'}{'resident_set_size'}{$pid_cmd}{$timestamp_ms} = $stats{'RSS'}; $timestamps{$timestamp_ms}++; } } @@ -191,35 +126,30 @@ if (opendir(my $pids_dh, $pids_dir) || die "Could not open $pids_dir") { } } -if ($cdm) { - print "cdm\n"; - gen_cdm_metric_data(\%pidstat, $period_doc_path, $es_dir, $hostname, "pidstat"); -} else { - # fill in any missing data with zeros - my $htmlpage; - my $graph; - my $pid; - my $timestamp_ms; - foreach $htmlpage ( keys %pidstat ) { - foreach $graph ( keys %{ $pidstat{$htmlpage} } ) { - foreach $pid ( keys %{ $pidstat{$htmlpage}{$graph} } ) { - foreach $timestamp_ms ( sort {$a <=> $b} (keys %timestamps ) ) { - if (! defined($pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms})) { - $pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms} = 0; - } +# fill in any missing data with zeros +my $htmlpage; +my $graph; +my $pid; +my $timestamp_ms; +foreach $htmlpage ( keys %pidstat ) { + foreach $graph ( keys %{ $pidstat{$htmlpage} } ) { + foreach $pid ( keys %{ $pidstat{$htmlpage}{$graph} } ) { + foreach $timestamp_ms ( sort {$a <=> $b} (keys %timestamps ) ) { + if (! defined($pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms})) { + $pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms} = 0; } } } } - $graph_type{cpu_usage}{percent_cpu} = "stackedAreaChart"; - $graph_threshold{cpu_usage}{percent_cpu} = 1; - $graph_threshold{file_io}{io_reads_KB_sec} = 50; - $graph_threshold{file_io}{io_writes_KB_sec} = 50; - $graph_threshold{memory_faults}{minor_faults_sec} = 150; - $graph_threshold{memory_faults}{major_faults_sec} = 150; - $graph_threshold{context_switches}{voluntary_switches_sec} = 100; - $graph_threshold{context_switches}{nonvoluntary_switches_sec} = 100; - $graph_threshold{memory_usage}{virtual_size} = 100; - $graph_threshold{memory_usage}{resident_set_size} = 100; - gen_data(\%pidstat, \%graph_type, \%graph_threshold, $dir); } +$graph_type{cpu_usage}{percent_cpu} = "stackedAreaChart"; +$graph_threshold{cpu_usage}{percent_cpu} = 1; +$graph_threshold{file_io}{io_reads_KB_sec} = 50; +$graph_threshold{file_io}{io_writes_KB_sec} = 50; +$graph_threshold{memory_faults}{minor_faults_sec} = 150; +$graph_threshold{memory_faults}{major_faults_sec} = 150; +$graph_threshold{context_switches}{voluntary_switches_sec} = 100; +$graph_threshold{context_switches}{nonvoluntary_switches_sec} = 100; +$graph_threshold{memory_usage}{virtual_size} = 100; +$graph_threshold{memory_usage}{resident_set_size} = 100; +gen_data(\%pidstat, \%graph_type, \%graph_threshold, $dir); diff --git a/agent/tool-scripts/postprocess/pidstat-postprocess-cdm b/agent/tool-scripts/postprocess/pidstat-postprocess-cdm deleted file mode 120000 index d3ec1bd33c..0000000000 --- a/agent/tool-scripts/postprocess/pidstat-postprocess-cdm +++ /dev/null @@ -1 +0,0 @@ -pidstat-postprocess \ No newline at end of file diff --git a/agent/tool-scripts/postprocess/sar-postprocess b/agent/tool-scripts/postprocess/sar-postprocess index 477bed540d..866273e796 100755 --- a/agent/tool-scripts/postprocess/sar-postprocess +++ b/agent/tool-scripts/postprocess/sar-postprocess @@ -10,23 +10,14 @@ # CSV files and statically generated graphs # with embedded javascript, using d3.js libraries # -# or -# -# CommonDataModel documents -# # Usage: # -# sar-postprocess [ ] +# sar-postprocess # # dir = directory where sar-stdout.txt can be found -# period-doc = path to CDM period document for this iteration-sample -# es-dir = path to base directory where CDM documents should be written -# hostname = the host where sar was run # # Note: # -# arguments 2 - 4 are only used when generating CommonDataModel -# # sar-stdout.txt must be generated with "sar -A " use strict; @@ -34,12 +25,6 @@ use warnings; use lib $ENV{'pbench_lib_dir'}; no lib "."; -BEGIN { - if (scalar @ARGV > 1) { - require PbenchCDM; - PbenchCDM->import( qw(log_cdm_metric_sample gen_cdm_metric_data) ); - } -} use GenData qw(gen_data); use SysStat qw(get_sar_cpumode_attributes build_cpu_topology get_cpu_socket_core); @@ -47,39 +32,9 @@ my $dir = shift; die "Non-default options, skipping post-processing for sar" if (-f $dir . "/sar.options"); -my $es_dir; -my $hostname; -my $period_doc_path; -my $cdm = 0; my @cpu_topology; -my %sar_cdm; my $interval; # sampling interval -# When producing CDM docs, this is typically called from the pbench -# controller only, and the following arguments must be provided: -if (scalar @ARGV > 1) { - die "Must provide " if (scalar @ARGV != 3); - $cdm = 1; - $period_doc_path = shift; - $es_dir = shift; - $hostname = shift; - build_cpu_topology(\@cpu_topology, $dir); - my $sar_cmd_file = $dir . "/sar.cmd"; - open (FH, $sar_cmd_file) || die "Could not open $sar_cmd_file"; - while (my $line = ) { - if ($line =~ /\S+\s+(\d+)$/ ){ - $interval = $1; - print "Found sar interval: $interval\n"; - } else { - print "interval not found in sar.cmd: $line\n"; - } - } - close FH; - if (not defined $interval) { - print "Could not determine the sampling interval for sar\n"; - exit 1; - } -} my %sar; my $prev_time = ""; my $date_ms; @@ -198,31 +153,13 @@ while ($line = ) { (my $socket, my $core) = get_cpu_socket_core($cpuid, \@cpu_topology); @cpumode_util{@cpumode_attributes} = @values; $timestamp_ms = calc_timestamp($time); - if ($cdm and $cpuid ne "all") { - # Note that an "all_cpu_busy" metric is not calculated - # like traditional pbench. This metric is dynamically - # calculated from the Web UI. However, cpu busy vs idle - # stats have different name_format, so that only busy - # or # only idle stats are aggregated. Allowing - # aggregation of all the modes (idle and busy) together - # makes little sense, since it will just add up to 100. - for my $util_type (keys %cpu_modes ) { - my $type = "Processor" . ucfirst $util_type . "Util"; - for my $mode (@{ $cpu_modes{$util_type} }) { - my %md = ( 'host' => $hostname, 'socket' => $socket, 'core' => $core, 'id' => $cpuid, 'mode' => $mode ); - log_cdm_metric_sample('sar', 'throughput', $type, '%host%-%socket%:%core%:%id%-%mode%', - \%md, \%sar, $timestamp_ms, $cpumode_util{$mode}, $interval); - } - } - } else { - $cpuid = sprintf "%02d", $cpuid; - my $cpulabel = "cpu_" . $cpuid; - my $busy = $cpumode_util{'usr'} + $cpumode_util{'nice'} + $cpumode_util{'sys'} + $cpumode_util{'irq'} + $cpumode_util{'soft'} + $cpumode_util{'guest'} + $cpumode_util{'gnice'}; - for my $mode (@cpumode_attributes) { - $sar{per_cpu}{$cpulabel}{$mode}{$timestamp_ms} = $cpumode_util{$mode}; - } - $sar{cpu}{"all_cpu_busy"}{$cpulabel}{$timestamp_ms} = $busy; + $cpuid = sprintf "%02d", $cpuid; + my $cpulabel = "cpu_" . $cpuid; + my $busy = $cpumode_util{'usr'} + $cpumode_util{'nice'} + $cpumode_util{'sys'} + $cpumode_util{'irq'} + $cpumode_util{'soft'} + $cpumode_util{'guest'} + $cpumode_util{'gnice'}; + for my $mode (@cpumode_attributes) { + $sar{per_cpu}{$cpulabel}{$mode}{$timestamp_ms} = $cpumode_util{$mode}; } + $sar{cpu}{"all_cpu_busy"}{$cpulabel}{$timestamp_ms} = $busy; next; } @@ -238,16 +175,8 @@ while ($line = ) { $time = $1; my %proc_ctx = ( 'ProcessesCreatedSec' => $2, 'ContextSwitchesSec' => $3 ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %proc_ctx) { - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', $type, '%host%', - \%md, \%sar, $timestamp_ms, $proc_ctx{$type}); - } - } else { - $sar{system}{proc_cswch_sec}{processes_created_sec}{$timestamp_ms} = $proc_ctx{'ProcessesCreatedSec'}; - $sar{system}{proc_cswch_sec}{context_switches_sec}{$timestamp_ms} = $proc_ctx{'ContextSwitchesSec'}; - } + $sar{system}{proc_cswch_sec}{processes_created_sec}{$timestamp_ms} = $proc_ctx{'ProcessesCreatedSec'}; + $sar{system}{proc_cswch_sec}{context_switches_sec}{$timestamp_ms} = $proc_ctx{'ContextSwitchesSec'}; next; } @@ -264,13 +193,7 @@ while ($line = ) { $time = $1; my $intr_sec = $2; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', 'InterruptsSec', '%host%', - \%md, \%sar, $timestamp_ms, $intr_sec); - } else { - $sar{system}{interrupts_sec}{all_interrupts}{$timestamp_ms} = $intr_sec; - } + $sar{system}{interrupts_sec}{all_interrupts}{$timestamp_ms} = $intr_sec; next; } @@ -286,16 +209,8 @@ while ($line = ) { $time = $1; my %pswp_sec = ( 'In' => $2, 'Out' => $3 ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %pswp_sec) { - my %md = ( 'host' => $hostname, 'direction' => $dir ); - log_cdm_metric_sample('sar', 'througput', 'PageSwapsSec', '%host%-%direction%', - \%md, \%sar, $timestamp_ms, $pswp_sec{$dir}); - } - } else { - $sar{memory}{memory_activity}{page_swaps_in_sec}{$timestamp_ms} = $pswp_sec{'In'}; - $sar{memory}{memory_activity}{page_swaps_out_sec}{$timestamp_ms} = $pswp_sec{'Out'}; - } + $sar{memory}{memory_activity}{page_swaps_in_sec}{$timestamp_ms} = $pswp_sec{'In'}; + $sar{memory}{memory_activity}{page_swaps_out_sec}{$timestamp_ms} = $pswp_sec{'Out'}; next; } @@ -314,36 +229,15 @@ while ($line = ) { my %other_pg_sec = ( 'Free' => $6, 'KswapdSacn' => $7, 'Scan' => $8, 'Steal' => $9 ); my $vmeff_pct = $10; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %disk_pg_kb_sec) { - my %md = ( 'host' => $hostname, 'direction' => $dir ); - log_cdm_metric_sample('sar', 'througput', 'DiskPageFaultKbytesSec', '%host%-%direction%', - \%md, \%sar, $timestamp_ms, $disk_pg_kb_sec{$dir}); - } - for my $type (keys %fault_sec) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'througput', 'PageFaultPagesSec', '%host%-%type%', - \%md, \%sar, $timestamp_ms, $fault_sec{$type}); - } - for my $action (keys %other_pg_sec) { - my %md = ( 'host' => $hostname, 'action' => $action ); - log_cdm_metric_sample('sar', 'througput', 'OtherProcessedPagesSec', '%host%-%action%', - \%md, \%sar, $timestamp_ms, $other_pg_sec{$action}); - } - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', 'VirtMemPctEff', '%host%', - \%md, \%sar, $timestamp_ms, $vmeff_pct); - } else { - $sar{memory}{memory_activity}{pgpgin_sec}{$timestamp_ms} = $disk_pg_kb_sec{'In'}; - $sar{memory}{memory_activity}{pgpgout_sec}{$timestamp_ms} = $disk_pg_kb_sec{'Out'}; - $sar{memory}{memory_activity}{fault_sec}{$timestamp_ms} = sprintf("%.2f", $fault_sec{'Minor'} + $fault_sec{'Major'}); - $sar{memory}{memory_activity}{majflt_sec}{$timestamp_ms} = $fault_sec{'Major'}; - $sar{memory}{memory_activity}{pgfree_sec}{$timestamp_ms} = $other_pg_sec{'Free'}; - $sar{memory}{memory_activity}{pgscank_sec}{$timestamp_ms} = $other_pg_sec{'KswapdSacn'}; - $sar{memory}{memory_activity}{pgscand_sec}{$timestamp_ms} = $other_pg_sec{'Scan'}; - $sar{memory}{memory_activity}{pgsteal_sec}{$timestamp_ms} = $other_pg_sec{'Steal'}; - $sar{memory}{memory_activity}{vmeff_pct}{$timestamp_ms} = $vmeff_pct; - } + $sar{memory}{memory_activity}{pgpgin_sec}{$timestamp_ms} = $disk_pg_kb_sec{'In'}; + $sar{memory}{memory_activity}{pgpgout_sec}{$timestamp_ms} = $disk_pg_kb_sec{'Out'}; + $sar{memory}{memory_activity}{fault_sec}{$timestamp_ms} = sprintf("%.2f", $fault_sec{'Minor'} + $fault_sec{'Major'}); + $sar{memory}{memory_activity}{majflt_sec}{$timestamp_ms} = $fault_sec{'Major'}; + $sar{memory}{memory_activity}{pgfree_sec}{$timestamp_ms} = $other_pg_sec{'Free'}; + $sar{memory}{memory_activity}{pgscank_sec}{$timestamp_ms} = $other_pg_sec{'KswapdSacn'}; + $sar{memory}{memory_activity}{pgscand_sec}{$timestamp_ms} = $other_pg_sec{'Scan'}; + $sar{memory}{memory_activity}{pgsteal_sec}{$timestamp_ms} = $other_pg_sec{'Steal'}; + $sar{memory}{memory_activity}{vmeff_pct}{$timestamp_ms} = $vmeff_pct; next; } @@ -368,32 +262,22 @@ while ($line = ) { my %memusage = ( 'SystemMemUsageKB' => \%kbmem_system, 'SystemMemActivenessKB' => \%kbmem_activeness, 'KernelMemUsageKB' => \%kbmem_kernel, 'OtherMemUsageKB' => \%kbmem_other, 'PercentMemUsageKB' => \%pctmem ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $memtype (keys %memusage) { - for my $type (keys %{ $memusage{$memtype} }) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'count', $memtype, '%host%-%type%', - \%md, \%sar, $timestamp_ms, $memusage{$memtype}{$type}); - } - } - } else { - $sar{memory}{memory_usage}{kbmemfree}{$timestamp_ms} = $kbmem_system{'Free'}; - $sar{memory}{memory_usage}{kbavail}{$timestamp_ms} = $kbmem_other{'Avail'}; - $sar{memory}{memory_usage}{kbmemused}{$timestamp_ms} = $kbmem_other{'Used'}; - $sar{memory}{memory_usage}{memused_pct}{$timestamp_ms} = $pctmem{'Used'}; - $sar{memory}{memory_usage}{kbbuffers}{$timestamp_ms} = $kbmem_system{'Buffers'}; - $sar{memory}{memory_usage}{kbcached}{$timestamp_ms} = $kbmem_system{'Cached'}; - $sar{memory}{memory_usage}{kbcommit}{$timestamp_ms} = $kbmem_other{'Commit'}; - $sar{memory}{memory_usage}{commit_pct}{$timestamp_ms} = $pctmem{'Commit'}; - $sar{memory}{memory_usage}{kbactive}{$timestamp_ms} = $kbmem_activeness{'Active'}; - $sar{memory}{memory_usage}{kbinactive}{$timestamp_ms} = $kbmem_activeness{'Inactive'}; - $sar{memory}{memory_usage}{kbdirty}{$timestamp_ms} = $kbmem_kernel{'Dirty'}; - $sar{memory}{memory_usage}{kbanonpg}{$timestamp_ms} = $kbmem_kernel{'NoPage'}; - $sar{memory}{memory_usage}{kbslab}{$timestamp_ms} = $kbmem_system{'Slab'}; - $sar{memory}{memory_usage}{kbkstack}{$timestamp_ms} = $kbmem_kernel{'Stack'}; - $sar{memory}{memory_usage}{kbpgtbl}{$timestamp_ms} = $kbmem_kernel{'PageTable'}; - $sar{memory}{memory_usage}{kbvmused}{$timestamp_ms} = $kbmem_other{'VmUsed'}; - } + $sar{memory}{memory_usage}{kbmemfree}{$timestamp_ms} = $kbmem_system{'Free'}; + $sar{memory}{memory_usage}{kbavail}{$timestamp_ms} = $kbmem_other{'Avail'}; + $sar{memory}{memory_usage}{kbmemused}{$timestamp_ms} = $kbmem_other{'Used'}; + $sar{memory}{memory_usage}{memused_pct}{$timestamp_ms} = $pctmem{'Used'}; + $sar{memory}{memory_usage}{kbbuffers}{$timestamp_ms} = $kbmem_system{'Buffers'}; + $sar{memory}{memory_usage}{kbcached}{$timestamp_ms} = $kbmem_system{'Cached'}; + $sar{memory}{memory_usage}{kbcommit}{$timestamp_ms} = $kbmem_other{'Commit'}; + $sar{memory}{memory_usage}{commit_pct}{$timestamp_ms} = $pctmem{'Commit'}; + $sar{memory}{memory_usage}{kbactive}{$timestamp_ms} = $kbmem_activeness{'Active'}; + $sar{memory}{memory_usage}{kbinactive}{$timestamp_ms} = $kbmem_activeness{'Inactive'}; + $sar{memory}{memory_usage}{kbdirty}{$timestamp_ms} = $kbmem_kernel{'Dirty'}; + $sar{memory}{memory_usage}{kbanonpg}{$timestamp_ms} = $kbmem_kernel{'NoPage'}; + $sar{memory}{memory_usage}{kbslab}{$timestamp_ms} = $kbmem_system{'Slab'}; + $sar{memory}{memory_usage}{kbkstack}{$timestamp_ms} = $kbmem_kernel{'Stack'}; + $sar{memory}{memory_usage}{kbpgtbl}{$timestamp_ms} = $kbmem_kernel{'PageTable'}; + $sar{memory}{memory_usage}{kbvmused}{$timestamp_ms} = $kbmem_other{'VmUsed'}; next; } @@ -410,19 +294,11 @@ while ($line = ) { my %kbswap = ( 'Used' => $3, 'Free' => $2, 'Cached' => $5 ); my %pctswap = ( 'Used' => $4, 'Cached' => $6 ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %kbswap) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'count', "SwapUsageKB", '%host%-%type%', - \%md, \%sar, $timestamp_ms, $kbswap{$type}); - } - } else { - $sar{memory}{memory_usage}{kbswpfree}{$timestamp_ms} = $kbswap{'Free'}; - $sar{memory}{memory_usage}{kbswpused}{$timestamp_ms} = $kbswap{'Used'}; - $sar{memory}{memory_usage}{swpused_pct}{$timestamp_ms} = $pctswap{'Used'}; - $sar{memory}{memory_usage}{kbswpcad}{$timestamp_ms} = $kbswap{'Cached'}; - $sar{memory}{memory_usage}{swpcad_pct}{$timestamp_ms} = $pctswap{'Cached'}; - } + $sar{memory}{memory_usage}{kbswpfree}{$timestamp_ms} = $kbswap{'Free'}; + $sar{memory}{memory_usage}{kbswpused}{$timestamp_ms} = $kbswap{'Used'}; + $sar{memory}{memory_usage}{swpused_pct}{$timestamp_ms} = $pctswap{'Used'}; + $sar{memory}{memory_usage}{kbswpcad}{$timestamp_ms} = $kbswap{'Cached'}; + $sar{memory}{memory_usage}{swpcad_pct}{$timestamp_ms} = $pctswap{'Cached'}; next; } @@ -440,19 +316,9 @@ while ($line = ) { my %pcthuge = ( 'Used' => $4 ); my %hugemem = ( 'HugePageUsageKB' => \%kbhuge, 'HugePageUsagePercent' => \%pcthuge ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $hugetype (keys %hugemem) { - for my $type (keys %{ $hugemem{$hugetype} }) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'count', $hugetype, '%host%-%type%', - \%md, \%sar, $timestamp_ms, $hugemem{$hugetype}{$type}); - } - } - } else { - $sar{memory}{memory_usage}{kbhugefree}{$timestamp_ms} = $kbhuge{'Free'}; - $sar{memory}{memory_usage}{kbhugeused}{$timestamp_ms} = $kbhuge{'Used'}; - $sar{memory}{memory_usage}{hugeused_pct}{$timestamp_ms} = $pcthuge{'Used'}; - } + $sar{memory}{memory_usage}{kbhugefree}{$timestamp_ms} = $kbhuge{'Free'}; + $sar{memory}{memory_usage}{kbhugeused}{$timestamp_ms} = $kbhuge{'Used'}; + $sar{memory}{memory_usage}{hugeused_pct}{$timestamp_ms} = $pcthuge{'Used'}; next; } @@ -495,26 +361,13 @@ while ($line = ) { 'NetdevCompressedPacketsSec' => \%comp_packets_sec, 'NetdevMulticastPacketsSec' => \%mult_packets_sec ); my $pct_util = $10; my $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $nettype (keys %network_sec) { - for my $dir (keys %{ $network_sec{$nettype} }) { - my %md = ( 'host' => $hostname, 'direction' => $dir); - log_cdm_metric_sample('sar', 'throughput', $nettype, '%host%-%direction%', - \%md, \%sar, $timestamp_ms, $network_sec{$nettype}{$dir}); - } - } - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', 'NetdevPercentUtilization', '%host%', - \%md, \%sar, $timestamp_ms, $pct_util); - } else { - $sar{network_l2}{network_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $packets_sec{'rx'}; - $sar{network_l2}{network_packets_sec}{"$interface" . "-tx"}{$timestamp_ms} = $packets_sec{'tx'}; - $sar{network_l2}{network_compressed_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $comp_packets_sec{'rx'}; - $sar{network_l2}{network_compressed_packets_sec}{"$interface" . "-tx"}{$timestamp_ms} = $comp_packets_sec{'tx'}; - $sar{network_l2}{network_multicast_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $mult_packets_sec{'rx'}; - $sar{network_l2}{network_Mbits_sec}{"$interface" . "-rx"}{$timestamp_ms} = $mbits_sec{'rx'}; - $sar{network_l2}{network_Mbits_sec}{"$interface" . "-tx"}{$timestamp_ms} = $mbits_sec{'tx'}; - } + $sar{network_l2}{network_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $packets_sec{'rx'}; + $sar{network_l2}{network_packets_sec}{"$interface" . "-tx"}{$timestamp_ms} = $packets_sec{'tx'}; + $sar{network_l2}{network_compressed_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $comp_packets_sec{'rx'}; + $sar{network_l2}{network_compressed_packets_sec}{"$interface" . "-tx"}{$timestamp_ms} = $comp_packets_sec{'tx'}; + $sar{network_l2}{network_multicast_packets_sec}{"$interface" . "-rx"}{$timestamp_ms} = $mult_packets_sec{'rx'}; + $sar{network_l2}{network_Mbits_sec}{"$interface" . "-rx"}{$timestamp_ms} = $mbits_sec{'rx'}; + $sar{network_l2}{network_Mbits_sec}{"$interface" . "-tx"}{$timestamp_ms} = $mbits_sec{'tx'}; next; } @@ -538,24 +391,14 @@ while ($line = ) { my %err_rx_packets_sec = ( 'Drop' => $6, 'Fifo' => $11, 'Frame' => $9 ); my %err_packets_sec = ( 'tx' => \%err_tx_packets_sec, 'rx' => \%err_rx_packets_sec ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %err_packets_sec) { - for my $error (keys %{ $err_packets_sec{$dir} } ){ - my %md = ( 'host' => $hostname, 'direction' => $dir, 'type' => $error ); - log_cdm_metric_sample('sar', 'throughput', 'NetdevErrorPacketsSec', '%host%-%direction%-%type%', - \%md, \%sar, $timestamp_ms, $err_packets_sec{$dir}{$error}); - } - } - } else { - $sar{network_l2}{errors_sec}{"$interface" . "-rx"}{$timestamp_ms} = $rx_err_sec; - $sar{network_l2}{errors_sec}{"$interface" . "-tx"}{$timestamp_ms} = $tx_err_sec; - $sar{network_l2}{drops_sec}{"$interface" . "-rx"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Drop'}; - $sar{network_l2}{drops_sec}{"$interface" . "-tx"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Drop'}; - $sar{network_l2}{carrier_errors}{"$interface"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Carrier'}; - $sar{network_l2}{frame_alignment_errors}{"$interface"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Frame'}; - $sar{network_l2}{fifo_overrun_errors}{"$interface" . "-rx"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Fifo'}; - $sar{network_l2}{fifo_overrun_errors}{"$interface" . "-tx"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Fifo'}; - } + $sar{network_l2}{errors_sec}{"$interface" . "-rx"}{$timestamp_ms} = $rx_err_sec; + $sar{network_l2}{errors_sec}{"$interface" . "-tx"}{$timestamp_ms} = $tx_err_sec; + $sar{network_l2}{drops_sec}{"$interface" . "-rx"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Drop'}; + $sar{network_l2}{drops_sec}{"$interface" . "-tx"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Drop'}; + $sar{network_l2}{carrier_errors}{"$interface"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Carrier'}; + $sar{network_l2}{frame_alignment_errors}{"$interface"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Frame'}; + $sar{network_l2}{fifo_overrun_errors}{"$interface" . "-rx"}{$timestamp_ms} = $err_packets_sec{'rx'}{'Fifo'}; + $sar{network_l2}{fifo_overrun_errors}{"$interface" . "-tx"}{$timestamp_ms} = $err_packets_sec{'tx'}{'Fifo'}; next; } @@ -571,21 +414,12 @@ while ($line = ) { $time = $1; my $calls_sec = $2; my %nfs_client_calls_sec = ( 'Retrans' => $3, 'Read' => $4, 'Write' => $5, 'Access' => $6, 'GetAttr' => $7 ); - if ($cdm) { - for my $type (keys %nfs_client_calls_sec) { - # these metrics will end up being grouped with NFS server calls/sec - my %md = ( 'host' => $hostname, 'type' => $type, 'clientserver' => 'client' ); - log_cdm_metric_sample('sar', 'throughput', 'NFSCallsSec', '%host%-%clientserver%-%type%', - \%md, \%sar, $timestamp_ms, $nfs_client_calls_sec{$type}); - } - } else { - my $timestamp_ms = calc_timestamp($time); - $sar{network_l345}{nfs_client}{calls_sec}{$timestamp_ms} = $calls_sec; - $sar{network_l345}{nfs_client}{reads_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Read'}; - $sar{network_l345}{nfs_client}{writes_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Write'}; - $sar{network_l345}{nfs_client}{access_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Access'}; - $sar{network_l345}{nfs_client}{getattr_sec}{$timestamp_ms} = $nfs_client_calls_sec{'GetAttr'}; - } + my $timestamp_ms = calc_timestamp($time); + $sar{network_l345}{nfs_client}{calls_sec}{$timestamp_ms} = $calls_sec; + $sar{network_l345}{nfs_client}{reads_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Read'}; + $sar{network_l345}{nfs_client}{writes_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Write'}; + $sar{network_l345}{nfs_client}{access_sec}{$timestamp_ms} = $nfs_client_calls_sec{'Access'}; + $sar{network_l345}{nfs_client}{getattr_sec}{$timestamp_ms} = $nfs_client_calls_sec{'GetAttr'}; next; } @@ -604,40 +438,16 @@ while ($line = ) { my %nfs_server_hitmiss_sec = ( 'Hit' => $7, 'Miss' => $8 ); my %nfs_server_calls_sec = ( 'Read' => $9, 'Write' => $10, 'Access' => $11, 'GetAttr' => $12 ); my $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %nfs_server_calls_sec) { - # these metrics will end up being grouped with NFS client calls/sec - my %md = ( 'host' => $hostname, 'type' => $type, 'clientserver' => 'server' ); - log_cdm_metric_sample('sar', 'throughput', 'NFSCallsSec', '%host%-%clientserver%-%type%', - \%md, \%sar, $timestamp_ms, $nfs_server_calls_sec{$type}); - } - for my $status (keys %nfs_server_goodbad_sec) { - my %md = ( 'host' => $hostname, 'status' => $status ); - log_cdm_metric_sample('sar', 'throughput', 'NFSServerGoodBadCallsSec', '%host%-%status%', - \%md, \%sar, $timestamp_ms, $nfs_server_goodbad_sec{$status}); - } - for my $status (keys %nfs_server_hitmiss_sec) { - my %md = ( 'host' => $hostname, 'status' => $status ); - log_cdm_metric_sample('sar', 'throughput', 'NFSServerCacheHitMissSec', '%host%-%status%', - \%md, \%sar, $timestamp_ms, $nfs_server_hitmiss_sec{$status}); - } - for my $proto (keys %nfs_server_proto_sec) { - my %md = ( 'host' => $hostname, 'protocol' => $proto ); - log_cdm_metric_sample('sar', 'throughput', 'NFSServerPacketsSec', '%host%-%protocol%', - \%md, \%sar, $timestamp_ms, $nfs_server_proto_sec{$proto}); - } - } else { - $sar{network_l345}{nfs_server}{calls_sec}{$timestamp_ms} = sprintf("%.2f", $nfs_server_goodbad_sec{'Good'} + $nfs_server_goodbad_sec{'Bad'}); - $sar{network_l345}{nfs_server}{badcalls_sec}{$timestamp_ms} = $nfs_server_goodbad_sec{'Bad'}; - $sar{network_l345}{nfs_server}{rx_udp_packets_sec}{$timestamp_ms} = $nfs_server_proto_sec{'Udp'}; - $sar{network_l345}{nfs_server}{rx_tcp_packets_sec}{$timestamp_ms} = $nfs_server_proto_sec{'Tcp'}; - $sar{network_l345}{nfs_server}{reply_cache_hits_sec}{$timestamp_ms} = $nfs_server_hitmiss_sec{'Hit'}; - $sar{network_l345}{nfs_server}{reply_cache_misses_sec}{$timestamp_ms} = $nfs_server_hitmiss_sec{'Miss'}; - $sar{network_l345}{nfs_server}{rpc_read_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Read'}; - $sar{network_l345}{nfs_server}{rpc_write_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Write'}; - $sar{network_l345}{nfs_server}{rpc_access_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Access'}; - $sar{network_l345}{nfs_server}{rpc_getattr_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'GetAttr'}; - } + $sar{network_l345}{nfs_server}{calls_sec}{$timestamp_ms} = sprintf("%.2f", $nfs_server_goodbad_sec{'Good'} + $nfs_server_goodbad_sec{'Bad'}); + $sar{network_l345}{nfs_server}{badcalls_sec}{$timestamp_ms} = $nfs_server_goodbad_sec{'Bad'}; + $sar{network_l345}{nfs_server}{rx_udp_packets_sec}{$timestamp_ms} = $nfs_server_proto_sec{'Udp'}; + $sar{network_l345}{nfs_server}{rx_tcp_packets_sec}{$timestamp_ms} = $nfs_server_proto_sec{'Tcp'}; + $sar{network_l345}{nfs_server}{reply_cache_hits_sec}{$timestamp_ms} = $nfs_server_hitmiss_sec{'Hit'}; + $sar{network_l345}{nfs_server}{reply_cache_misses_sec}{$timestamp_ms} = $nfs_server_hitmiss_sec{'Miss'}; + $sar{network_l345}{nfs_server}{rpc_read_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Read'}; + $sar{network_l345}{nfs_server}{rpc_write_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Write'}; + $sar{network_l345}{nfs_server}{rpc_access_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'Access'}; + $sar{network_l345}{nfs_server}{rpc_getattr_calls_sec}{$timestamp_ms} = $nfs_server_calls_sec{'GetAttr'}; next; } @@ -659,25 +469,12 @@ while ($line = ) { my $ipfrag = $6; my $tcptw = $7; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $proto (keys %sck) { - my %md = ( 'host' => $hostname, 'protocol' => $proto ); - log_cdm_metric_sample('sar', 'count', 'SocketsByProtocol', '%host%-%protocol%', - \%md, \%sar, $timestamp_ms, int $sck{$proto}); - } - my %md = ( 'host' => $hostname ); # works for both metrics - log_cdm_metric_sample('sar', 'count', 'IpFragments', '%host%', - \%md, \%sar, $timestamp_ms, int $ipfrag); - log_cdm_metric_sample('sar', 'count', 'TcpTimewaitSockets', '%host%', \%md, \%sar, $timestamp_ms, int $tcptw); - - } else { - $sar{network_l345}{sockets}{total_sockets}{$timestamp_ms} = $totsck; - $sar{network_l345}{sockets}{tcp_sockets}{$timestamp_ms} = $sck{'Tcp'}; - $sar{network_l345}{sockets}{udp_sockets}{$timestamp_ms} = $sck{'Udp'}; - $sar{network_l345}{sockets}{raw_sockets}{$timestamp_ms} = $sck{'Raw'}; - $sar{network_l345}{sockets}{raw_sockets}{$timestamp_ms} = $ipfrag; - $sar{network_l345}{sockets}{tcp_timewait_sockets}{$timestamp_ms} = $tcptw; - } + $sar{network_l345}{sockets}{total_sockets}{$timestamp_ms} = $totsck; + $sar{network_l345}{sockets}{tcp_sockets}{$timestamp_ms} = $sck{'Tcp'}; + $sar{network_l345}{sockets}{udp_sockets}{$timestamp_ms} = $sck{'Udp'}; + $sar{network_l345}{sockets}{raw_sockets}{$timestamp_ms} = $sck{'Raw'}; + $sar{network_l345}{sockets}{raw_sockets}{$timestamp_ms} = $ipfrag; + $sar{network_l345}{sockets}{tcp_timewait_sockets}{$timestamp_ms} = $tcptw; next; } @@ -702,22 +499,14 @@ while ($line = ) { 'FragmentedDatagramsSec' => $8, # The number of IP datagrams that have been successfully fragmented at this entity per second [ipFragOKs]. 'FragmentCreatedSec' => $9 );# The number of IP datagram fragments that have been generated per second as a result of fragmentation at this entity [ipFragCreates]. my $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %net_ip) { - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', $type, '%host%', - \%md, \%sar, $timestamp_ms, $net_ip{$type}); - } - } else { - $sar{network_l345}{ip}{rx_datagrams_sec}{$timestamp_ms} = $net_ip{'RxDatagramsSec'}; - $sar{network_l345}{ip}{rx_datagrams_forwarded_sec}{$timestamp_ms} = $net_ip{'FwdDatagramsSec'}; - $sar{network_l345}{ip}{datagrams_delivered_sec}{$timestamp_ms} = $net_ip{'DelDatagramsSec'}; - $sar{network_l345}{ip}{datagrams_output_requests_sec}{$timestamp_ms} = $net_ip{'OutputReqDatagramsSec'}; - $sar{network_l345}{ip}{fragments_rx_need_reassemble_sec}{$timestamp_ms} = $net_ip{'RxNeedReassmbleDatagramsSec'}; - $sar{network_l345}{ip}{datagrams_reassmbled_sec}{$timestamp_ms} = $net_ip{'ReassembledDatagramsSec'}; - $sar{network_l345}{ip}{datagrams_fragmented_ok_sec}{$timestamp_ms} = $net_ip{'FragmentedDatagramsSec'}; - $sar{network_l345}{ip}{datagram_fragments_gen_sec}{$timestamp_ms} = $net_ip{'FragmentCreatedSec'}; - } + $sar{network_l345}{ip}{rx_datagrams_sec}{$timestamp_ms} = $net_ip{'RxDatagramsSec'}; + $sar{network_l345}{ip}{rx_datagrams_forwarded_sec}{$timestamp_ms} = $net_ip{'FwdDatagramsSec'}; + $sar{network_l345}{ip}{datagrams_delivered_sec}{$timestamp_ms} = $net_ip{'DelDatagramsSec'}; + $sar{network_l345}{ip}{datagrams_output_requests_sec}{$timestamp_ms} = $net_ip{'OutputReqDatagramsSec'}; + $sar{network_l345}{ip}{fragments_rx_need_reassemble_sec}{$timestamp_ms} = $net_ip{'RxNeedReassmbleDatagramsSec'}; + $sar{network_l345}{ip}{datagrams_reassmbled_sec}{$timestamp_ms} = $net_ip{'ReassembledDatagramsSec'}; + $sar{network_l345}{ip}{datagrams_fragmented_ok_sec}{$timestamp_ms} = $net_ip{'FragmentedDatagramsSec'}; + $sar{network_l345}{ip}{datagram_fragments_gen_sec}{$timestamp_ms} = $net_ip{'FragmentCreatedSec'}; next; } @@ -735,24 +524,14 @@ while ($line = ) { my %ipv4_tx_discarded_sec = ( 'NoRoute' => $7, 'FragmentFail' => $9, 'NoBuff' => $6 ); my %ipv4_discarded_sec = ( 'Rx' => \%ipv4_rx_discarded_sec, 'Tx' => \%ipv4_tx_discarded_sec ); my $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %ipv4_discarded_sec) { - for my $type (keys %{ $ipv4_discarded_sec{$dir} }) { - my %md = ( 'host' => $hostname, 'direction' => $dir, 'type' => $type ); - log_cdm_metric_sample('sar', 'throughput', 'Ipv4DiscardsSec', '%host%-%direction%-%type%', - \%md, \%sar, $timestamp_ms, $ipv4_discarded_sec{$dir}{$type}); - } - } - } else { - $sar{network_l345}{ip}{rx_datagrams_header_err_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'HeaderError'}; - $sar{network_l345}{ip}{rx_datagrams_invalid_addr_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'DestIpError'}; - $sar{network_l345}{ip}{rx_datagrams_unknown_proto_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'UnknownProto'}; - $sar{network_l345}{ip}{rx_datagrams_nobuff_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'NoBuff'}; - $sar{network_l345}{ip}{tx_datagrams_nobuff_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'NoBuff'}; - $sar{network_l345}{ip}{tx_datagrams_noroute_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'NoRoute'}; - $sar{network_l345}{ip}{rx_datagrams_reassmbly_fail_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'ReassemblyFail'}; - $sar{network_l345}{ip}{tx_datagrams_fragment_fail_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'FragmentFail'}; - } + $sar{network_l345}{ip}{rx_datagrams_header_err_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'HeaderError'}; + $sar{network_l345}{ip}{rx_datagrams_invalid_addr_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'DestIpError'}; + $sar{network_l345}{ip}{rx_datagrams_unknown_proto_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'UnknownProto'}; + $sar{network_l345}{ip}{rx_datagrams_nobuff_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'NoBuff'}; + $sar{network_l345}{ip}{tx_datagrams_nobuff_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'NoBuff'}; + $sar{network_l345}{ip}{tx_datagrams_noroute_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'NoRoute'}; + $sar{network_l345}{ip}{rx_datagrams_reassmbly_fail_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Rx'}{'ReassemblyFail'}; + $sar{network_l345}{ip}{tx_datagrams_fragment_fail_sec}{$timestamp_ms} = $ipv4_discarded_sec{'Tx'}{'FragmentFail'}; next; } @@ -778,32 +557,20 @@ while ($line = ) { $icmp_sec{'Tx'}{'Request'} = \%icmp_tx_req_sec; $icmp_sec{'Tx'}{'Reply'} = \%icmp_tx_reply_sec; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %icmp_sec) { - for my $msgtype (keys %{ $icmp_sec{$dir} }) { - for my $reqtype (keys %{ $icmp_sec{$dir}{$msgtype} }) { - my %md = ( 'host' => $hostname, 'direction' => $dir, 'msgtype' => $msgtype, 'reqtype' => $msgtype ); - log_cdm_metric_sample('sar', 'throughput', 'IcmpMessagesSec', '%host%-%direction%-%msgtype%-%reqtype%', - \%md, \%sar, $timestamp_ms, $icmp_sec{$dir}{$msgtype}{$reqtype}); - } - } - } - } else { - $sar{network_l345}{icmp}{rx_messages_sec}{$timestamp_ms} = $imsg_sec; - $sar{network_l345}{icmp}{tx_messages_sec}{$timestamp_ms} = $omsg_sec; - $sar{network_l345}{icmp}{rx_echo_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'Echo'}; - $sar{network_l345}{icmp}{rx_echo_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'Echo'}; - $sar{network_l345}{icmp}{tx_echo_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'Echo'}; - $sar{network_l345}{icmp}{tx_echo_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Reply'}{'Echo'}; - $sar{network_l345}{icmp}{rx_timestamp_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'Timestamp'}; - $sar{network_l345}{icmp}{rx_timestamp_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'Timestamp'}; - $sar{network_l345}{icmp}{tx_timestamp_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'Timestamp'}; - $sar{network_l345}{icmp}{tx_timestamp_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Reply'}{'Timestamp'}; - $sar{network_l345}{icmp}{rx_addrmask_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'AddrMask'}; - $sar{network_l345}{icmp}{rx_addrmask_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'AddrMask'}; - $sar{network_l345}{icmp}{tx_addrmask_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'AddrMask'}; - $sar{network_l345}{icmp}{tx_addrmask_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'AddrMask'}; - } + $sar{network_l345}{icmp}{rx_messages_sec}{$timestamp_ms} = $imsg_sec; + $sar{network_l345}{icmp}{tx_messages_sec}{$timestamp_ms} = $omsg_sec; + $sar{network_l345}{icmp}{rx_echo_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'Echo'}; + $sar{network_l345}{icmp}{rx_echo_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'Echo'}; + $sar{network_l345}{icmp}{tx_echo_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'Echo'}; + $sar{network_l345}{icmp}{tx_echo_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Reply'}{'Echo'}; + $sar{network_l345}{icmp}{rx_timestamp_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'Timestamp'}; + $sar{network_l345}{icmp}{rx_timestamp_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'Timestamp'}; + $sar{network_l345}{icmp}{tx_timestamp_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'Timestamp'}; + $sar{network_l345}{icmp}{tx_timestamp_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Reply'}{'Timestamp'}; + $sar{network_l345}{icmp}{rx_addrmask_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Request'}{'AddrMask'}; + $sar{network_l345}{icmp}{rx_addrmask_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Rx'}{'Reply'}{'AddrMask'}; + $sar{network_l345}{icmp}{tx_addrmask_request_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'AddrMask'}; + $sar{network_l345}{icmp}{tx_addrmask_reply_messages_sec}{$timestamp_ms} = $icmp_sec{'Tx'}{'Request'}{'AddrMask'}; next; } @@ -823,28 +590,18 @@ while ($line = ) { $icmperr_sec{'Rx'} = \%icmperr_rx_sec; $icmperr_sec{'Tx'} = \%icmperr_tx_sec; $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %icmperr_sec) { - for my $type (keys %{ $icmperr_sec{$dir} }) { - my %md = ( 'host' => $hostname, 'direction' => $dir, 'type' => $type ); - log_cdm_metric_sample('sar', 'throughput', 'IcmpErrorsSec', '%host%-%direction%-%type%', - \%md, \%sar, $timestamp_ms, $icmperr_sec{$dir}{$type}); - } - } - } else { - $sar{network_l345}{icmp}{rx_icmp_err_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'IcmpError'}; - $sar{network_l345}{icmp}{tx_icmp_err_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'IcmpError'}; - $sar{network_l345}{icmp}{rx_dest_unreachable_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'DestUnreachable'}; - $sar{network_l345}{icmp}{tx_dest_unreachable_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'DestUnreachable'}; - $sar{network_l345}{icmp}{rx_time_exceed_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'TimeExceeded'}; - $sar{network_l345}{icmp}{tx_time_exceed_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'TimeExceeded'}; - $sar{network_l345}{icmp}{rx_param_prob_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'ParamProblem'}; - $sar{network_l345}{icmp}{tx_param_prob_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'ParamProblem'}; - $sar{network_l345}{icmp}{rx_src_quench_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'SourceQuench'}; - $sar{network_l345}{icmp}{tx_src_quench_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'SourceQuench'}; - $sar{network_l345}{icmp}{rx_redir_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'Redirect'}; - $sar{network_l345}{icmp}{tx_redir_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'Redirect'}; - } + $sar{network_l345}{icmp}{rx_icmp_err_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'IcmpError'}; + $sar{network_l345}{icmp}{tx_icmp_err_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'IcmpError'}; + $sar{network_l345}{icmp}{rx_dest_unreachable_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'DestUnreachable'}; + $sar{network_l345}{icmp}{tx_dest_unreachable_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'DestUnreachable'}; + $sar{network_l345}{icmp}{rx_time_exceed_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'TimeExceeded'}; + $sar{network_l345}{icmp}{tx_time_exceed_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'TimeExceeded'}; + $sar{network_l345}{icmp}{rx_param_prob_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'ParamProblem'}; + $sar{network_l345}{icmp}{tx_param_prob_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'ParamProblem'}; + $sar{network_l345}{icmp}{rx_src_quench_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'SourceQuench'}; + $sar{network_l345}{icmp}{tx_src_quench_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'SourceQuench'}; + $sar{network_l345}{icmp}{rx_redir_sec}{$timestamp_ms} = $icmperr_sec{'Rx'}{'Redirect'}; + $sar{network_l345}{icmp}{tx_redir_sec}{$timestamp_ms} = $icmperr_sec{'Tx'}{'Redirect'}; next; } @@ -861,23 +618,10 @@ while ($line = ) { my %tcp_opens_sec = ( 'Active' => $2, 'Passive' => $3 ); my %tcp_segments_sec = ( 'Input' => $4, 'Output' => $5 ); $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %tcp_opens_sec) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'throughput', 'TcpOpensSec', '%host%-%type%', - \%md, \%sar, $timestamp_ms, $tcp_opens_sec{$type}); - } - for my $dir (keys %tcp_segments_sec) { - my %md = ( 'host' => $hostname, 'direction' => $dir ); - log_cdm_metric_sample('sar', 'throughput', 'TcpSegmentsSec', '%host%-%direction%', - \%md, \%sar, $timestamp_ms, $tcp_segments_sec{$dir}); - } - } else { - $sar{network_l345}{tcp_sockets}{active_opens_sec}{$timestamp_ms} = $tcp_opens_sec{'Active'}; - $sar{network_l345}{tcp_sockets}{passive_opens_sec}{$timestamp_ms} = $tcp_opens_sec{'Passive'}; - $sar{network_l345}{tcp_sockets}{rx_segments_sec}{$timestamp_ms} = $tcp_segments_sec{'Input'}; - $sar{network_l345}{tcp_sockets}{tx_segments_sec}{$timestamp_ms} = $tcp_segments_sec{'Output'}; - } + $sar{network_l345}{tcp_sockets}{active_opens_sec}{$timestamp_ms} = $tcp_opens_sec{'Active'}; + $sar{network_l345}{tcp_sockets}{passive_opens_sec}{$timestamp_ms} = $tcp_opens_sec{'Passive'}; + $sar{network_l345}{tcp_sockets}{rx_segments_sec}{$timestamp_ms} = $tcp_segments_sec{'Input'}; + $sar{network_l345}{tcp_sockets}{tx_segments_sec}{$timestamp_ms} = $tcp_segments_sec{'Output'}; next; } @@ -896,26 +640,11 @@ while ($line = ) { my $tcp_input_segerr_sec = $5; my $tcp_output_rsts_sec = $5; # The number of TCP segments sent per second containing the RST flag [tcpOutRsts]. $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $type (keys %tcperr_closes_sec ) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'throughput', 'TcpErrorClosesSec', '%host%-%type%', - \%md, \%sar, $timestamp_ms, $tcperr_closes_sec{$type}); - } - my %md = ( 'host' => $hostname ); - log_cdm_metric_sample('sar', 'throughput', 'TcpRetransmitsSec', '%host%', - \%md, \%sar, $timestamp_ms, $tcp_retrans_sec); - log_cdm_metric_sample('sar', 'throughput', 'TcpInputSegmentErrorsSec', '%host%', - \%md, \%sar, $timestamp_ms, $tcp_input_segerr_sec); - log_cdm_metric_sample('sar', 'throughput', 'TcpOutputRstsSec', '%host%', - \%md, \%sar, $timestamp_ms, $tcp_output_rsts_sec); - } else { - $sar{network_l345}{tcp_sockets}{tcp_attempt_fails_sec}{$timestamp_ms} = $tcperr_closes_sec{'AttemptFail'}; - $sar{network_l345}{tcp_sockets}{tcp_establish_resets_sec}{$timestamp_ms} = $tcperr_closes_sec{'EstablishReset'}; - $sar{network_l345}{tcp_sockets}{tcp_segments_retransmitted_sec}{$timestamp_ms} = $tcp_retrans_sec; - $sar{network_l345}{tcp_sockets}{tcp_rx_segments_error_sec}{$timestamp_ms} = $tcp_input_segerr_sec; - $sar{network_l345}{tcp_sockets}{tcp_tx_segments_RSTflag_sec}{$timestamp_ms} = $tcp_output_rsts_sec; - } + $sar{network_l345}{tcp_sockets}{tcp_attempt_fails_sec}{$timestamp_ms} = $tcperr_closes_sec{'AttemptFail'}; + $sar{network_l345}{tcp_sockets}{tcp_establish_resets_sec}{$timestamp_ms} = $tcperr_closes_sec{'EstablishReset'}; + $sar{network_l345}{tcp_sockets}{tcp_segments_retransmitted_sec}{$timestamp_ms} = $tcp_retrans_sec; + $sar{network_l345}{tcp_sockets}{tcp_rx_segments_error_sec}{$timestamp_ms} = $tcp_input_segerr_sec; + $sar{network_l345}{tcp_sockets}{tcp_tx_segments_RSTflag_sec}{$timestamp_ms} = $tcp_output_rsts_sec; next; } @@ -934,23 +663,10 @@ while ($line = ) { my $noport_sec = $4; # The total number of received UDP datagrams per second for which there was no application at the destination port [udpNoPorts]. my $idgmerr_sec = $5; # The number of received UDP datagrams per second that could not be delivered for reasons other than the lack of an application at the destination port [udpInErrors]. my $timestamp_ms = calc_timestamp($time); - if ($cdm) { - for my $dir (keys %udp_datagrams_sec) { - my %md = ( 'host' => $hostname, 'direction' => $dir ); - log_cdm_metric_sample('sar', 'throughput', 'UdpDatagramsSec', '%host%-%direction%', - \%md, \%sar, $timestamp_ms, $udp_datagrams_sec{$dir}); - } - for my $type (keys %udp_error_datagrams_sec) { - my %md = ( 'host' => $hostname, 'type' => $type ); - log_cdm_metric_sample('sar', 'throughput', 'UdpErrorDatagramsSec', '%host%-%type%', - \%md, \%sar, $timestamp_ms, $udp_error_datagrams_sec{$type}); - } - } else { - $sar{network_l345}{udp}{rx_datagrams_sec}{$timestamp_ms} = $udp_datagrams_sec{'Rx'}; - $sar{network_l345}{udp}{tx_datagrams_sec}{$timestamp_ms} = $udp_datagrams_sec{'Tx'}; - $sar{network_l345}{udp}{rx_datagrams_discard_no_dest_port_sec}{$timestamp_ms} = $udp_error_datagrams_sec{'NoPort'}; - $sar{network_l345}{udp}{rx_datagrams_discard_other_error}{$timestamp_ms} = $udp_error_datagrams_sec{'Other'}; - } + $sar{network_l345}{udp}{rx_datagrams_sec}{$timestamp_ms} = $udp_datagrams_sec{'Rx'}; + $sar{network_l345}{udp}{tx_datagrams_sec}{$timestamp_ms} = $udp_datagrams_sec{'Tx'}; + $sar{network_l345}{udp}{rx_datagrams_discard_no_dest_port_sec}{$timestamp_ms} = $udp_error_datagrams_sec{'NoPort'}; + $sar{network_l345}{udp}{rx_datagrams_discard_other_error}{$timestamp_ms} = $udp_error_datagrams_sec{'Other'}; next; } @@ -991,13 +707,7 @@ while ($line = ) { (my $socket, my $core) = get_cpu_socket_core($cpuid, \@cpu_topology); $timestamp_ms = calc_timestamp($time); - if ($cdm and $cpuid ne "all") { - my %md = ( 'host' => $hostname, 'socket' => $socket, 'core' => $core, 'id' => $cpuid ); - log_cdm_metric_sample('sar', 'count', 'ProcessorFrequencyMhz', '%host%-%socket%:%core%:%id%', - \%md, \%sar, $timestamp_ms, $mhz); - } else { - $sar{cpu}{"frequency_MHz"}{sprintf "cpu_%02d", $cpuid}{$timestamp_ms} = $mhz; - } + $sar{cpu}{"frequency_MHz"}{sprintf "cpu_%02d", $cpuid}{$timestamp_ms} = $mhz; next; } @@ -1016,14 +726,9 @@ while ($line = ) { # 03:25:56 AM 19754 31421 61.40 61.40 52237292 191508 0.37 /dev/mapper/rhel72_perf48-root } close(SAR_TXT); -if ($cdm) { - print "cdm\n"; - gen_cdm_metric_data(\%sar, $period_doc_path, $es_dir, $hostname, "sar"); -} else { - $graph_type{cpu}{"all_cpu_busy"} = "stackedAreaChart"; - my $label; - foreach $label (keys %{ $sar{per_cpu}} ) { - $graph_type{per_cpu}{$label} = "stackedAreaChart"; - } - gen_data(\%sar, \%graph_type, \%graph_threshold, $dir); +$graph_type{cpu}{"all_cpu_busy"} = "stackedAreaChart"; +my $label; +foreach $label (keys %{ $sar{per_cpu}} ) { + $graph_type{per_cpu}{$label} = "stackedAreaChart"; } +gen_data(\%sar, \%graph_type, \%graph_threshold, $dir); diff --git a/agent/tool-scripts/postprocess/sar-postprocess-cdm b/agent/tool-scripts/postprocess/sar-postprocess-cdm deleted file mode 120000 index a49c6938ff..0000000000 --- a/agent/tool-scripts/postprocess/sar-postprocess-cdm +++ /dev/null @@ -1 +0,0 @@ -sar-postprocess \ No newline at end of file diff --git a/agent/util-scripts/cdm-get-iterations b/agent/util-scripts/cdm-get-iterations deleted file mode 100755 index 5f2ff1d772..0000000000 --- a/agent/util-scripts/cdm-get-iterations +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/perl -## -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl - -use strict; -use warnings; -use File::Basename; -use File::Find; -use REST::Client; - -use lib $ENV{'pbench_lib_dir'}; -use JSON::MaybeXS; -use Data::Dumper; -use PbenchCDM qw(get_cdm_ver get_cdm_rel); -use PbenchBase qw(get_json_file put_json_file remove_element); - -sub gen_metric_groups { - my $pointer = shift; - my $query = shift; - my @fields = @_; - my @metr_grp_queries; - # If being called from a previous gen_metric_groups, finish the second part of the terms query - if (exists $$pointer{"key"}) { - $query .= '"' . $$pointer{"key"} . '"}}'; - } - if (scalar @fields > 0) { - my $field = shift @fields; - if (exists $$pointer{$field} and exists $$pointer{$field}{"buckets"}) { - foreach my $bucket (@{ $$pointer{$field}{"buckets"} }) { - # Only the first half of the "term", the field name, is here because we need to go one level deeper to get - # the value, in each of the buckets (one levep deeper). - push @metr_grp_queries, gen_metric_groups($bucket, $query . "," . '{"term": {"metric.' . $field . '": ', @fields); - } - } - return @metr_grp_queries; - } else { - $query =~ s/^,//; - return $query; - } -} - -my $script_name = basename($0); -my $es_host = shift(@ARGV); -my $run_id = shift(@ARGV); -my %req_header = ("Content-Type" => "application/json"); -my $template = -'{ - "size": 100, - "query" : { - "bool": { - "filter": [ - ] - } - } -}'; -my $client = REST::Client->new(); -$client->setHost($es_host); - -my $bench_name; -my $coder = JSON::MaybeXS->new->ascii->canonical; -# with run.id get the run doc(s) -my $run_req_ref = $coder->decode($template); -push @{ $$run_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"run.id": "' . $run_id . '"}}'); -$$run_req_ref{"aggs"}{"source"}{"terms"}{"field"} = "run.bench.name"; -my $run_req_json = $coder->encode($run_req_ref); -$client->request('GET', "cdmv" . get_cdm_ver . get_cdm_rel . "-iteration/iteration/_search", - $run_req_json, \%req_header); -my $resp = $client->responseContent(); -my $run_resp_ref = $coder->decode($resp); -print Dumper $run_resp_ref; -if (scalar @{ $$run_resp_ref{"aggregations"}{"source"}{"buckets"} } == 1) { - $bench_name = $$run_resp_ref{"aggregations"}{"source"}{"buckets"}[0]{"key"}; - print "benchmark: $bench_name\n"; -} else { - print "Could not find just one benchmark for this run\n"; - printf "%s\n", $resp; - exit 1; -} -# with run.id get the iterations -my $iter_req_ref = $coder->decode($template); -push @{ $$iter_req_ref{"query"}{"bool"}{"filter"} }, - $coder->decode('{"term": {"run.id": "' . $run_id . '"}}'); -my $iter_req_json = $coder->encode($iter_req_ref); -$client->request('GET', "cdmv" . get_cdm_ver . get_cdm_rel . "-iteration/iteration/_search", - $iter_req_json, \%req_header); -my $response = $client->responseContent(); -$run_resp_ref = $coder->decode($response); -foreach my $iter ( @{ $$run_resp_ref{'hits'}{'hits'} } ) { - printf "\niteration.id: %s\n", $$iter{'_source'}{'iteration'}{'id'}; -} diff --git a/agent/util-scripts/pbench-get-iteration-metrics b/agent/util-scripts/pbench-get-iteration-metrics deleted file mode 100755 index fc624bd563..0000000000 --- a/agent/util-scripts/pbench-get-iteration-metrics +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/perl -## -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl - -# Usage: pbench-get-metrics -# -# You can get the iteraton-id with pbench-get-iterations. -# -# This script will output all of the metrics (benchmark and tools) -# for an iteration-id - -use strict; -use warnings; -use File::Basename; -use File::Find; -use REST::Client; -use Time::HiRes qw(gettimeofday); - -use lib $ENV{'pbench_lib_dir'}; -use JSON::MaybeXS; -use Data::Dumper; -use Getopt::Long 'HelpMessage'; -use PbenchCDM qw(get_cdm_ver get_cdm_rel); -use PbenchBase qw(get_json_file put_json_file remove_element); -use PbenchES qw(get_primary_metric get_sample_ids get_primary_period_id get_name_format get_metric_label get_common_time_domain gen_label_to_terms list_all_metrics get_bench_name get_primary_period_name get_metric_ids); - -sub usage { - print "--es-host= The hostname and port for Elasticsearch. You can also set ES_HOST\n"; - print "--iteration-id= The UUID (iteration.id)\n"; - print "--debug-level= 0 = no debug, 1 = output ES requests and responses\n"; - exit; -} - -GetOptions( - 'es-host=s' => \(my $es_host = $ENV{'ES_HOST'}), - 'iteration-id=s' => \ my $iter_id, - 'debug-level=i' => \(my $debug = 0), - 'resolution=i' => \(my $resolution = 1) -) or usage; - -$es_host and $iter_id or usage; - -my $index_basename = "cdmv" . get_cdm_ver . get_cdm_rel; -my $script_name = basename($0); -my $coder = JSON::MaybeXS->new->ascii->canonical; -my $bench_name = get_bench_name($es_host, $iter_id);; -printf "bench_name:\n%s\n\n", $bench_name; -my $primary_metric = get_primary_metric($es_host, $iter_id); -my $sample_num = 0; -my $samples = ""; -my $sum = 0; -foreach my $sample_id ( get_sample_ids($es_host, $iter_id) ) { - printf "sample.id: %s\n", $sample_id; - my $primary_period_name = get_primary_period_name($es_host, $sample_id); - if (my $period_id = get_primary_period_id($es_host, $sample_id, $primary_period_name)) { - printf "period.id: %s\n", $period_id; - # Find all of the individual metrics for the benchmark - my @metr_ids = get_metric_ids($es_host, $period_id, $bench_name, $primary_metric); - # From all of those metrics, get the time domain where they all - # had samples - (my $begin, my $end) = get_common_time_domain($es_host, @metr_ids); - printf "begin: %d end: %d\n", $begin, $end; - list_all_metrics($es_host, $begin, $end, $resolution, $period_id); - } -} - - diff --git a/agent/util-scripts/pbench-get-metric-data b/agent/util-scripts/pbench-get-metric-data deleted file mode 100755 index 506906cc2f..0000000000 --- a/agent/util-scripts/pbench-get-metric-data +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/perl -## -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl - -# Usage: pbench-get-metrics -# -# You can get the iteraton-id with pbench-get-iterations. -# -# This script will output all of the metrics (benchmark and tools) -# for an iteration-id - -use strict; -use warnings; -use File::Basename; -use File::Find; -use REST::Client; -use Time::HiRes qw(gettimeofday); - -use lib $ENV{'pbench_lib_dir'}; -use JSON::MaybeXS; -use Data::Dumper; -use PbenchCDM qw(get_cdm_ver get_cdm_rel); -use PbenchBase qw(get_json_file put_json_file remove_element); -use PbenchES qw(get_name_format get_metric_data); -use Getopt::Long 'HelpMessage'; - - -sub usage { - print "--es-host= The hostname and port for Elasticsearch. You can also set ES_HOST\n"; - print "--period-id= The UUID (period.id) for a time-period from a benchmark-iteration-sample.\n"; - print " Running 'pbench-get-iteration-metrics' will provide a period.id for the\n"; - print " primary period (the measurement phase) of the benchmark-iteration-samples\n"; - print "--metric-source= The name of the metric source, like 'fio' or 'iostat'\n"; - print "--metric-type= The name of the metric type, like 'iops' for fio, or 'BusyCpuUtil' for mpstat\n"; - print "--breakout= If this option is not used, a single data-series will be returned which aggregates\n"; - print " all metrics which match the metric-source and metric-type. However, a query can\n"; - print " return multiple data-series, broken-out by any of the field names found in the\n"; - print " name_format. To get these available fields, either (a) inspect the returned\n"; - print " data from a call to this script, specifically, {{'breakouts'}:[]},\n"; - print " or, (b) run the get-iteraton-metrics script to see the available break-outs\n"; - print " for each of the metric source/tyops.\n"; - print "--begin= The epochtime in milliseconds for the beginning of the time domain.\n"; - print " This must be within the same time period for period.id and less than\n"; - print " 'end' value provided\n"; - print "--end= The epochtime in milliseconds for the ending of the time domain.\n"; - print " This must be within the same time period for period.id and greater than\n"; - print " 'begin' value provided\n"; - exit; -} - -GetOptions( - 'es-host=s' => \(my $es_host = $ENV{'ES_HOST'}), - 'period-id=s' => \ my $period_id, - 'metric-source=s' => \ my $metric_source, - 'metric-type=s' => \ my $metric_type, - 'breakout=s' => \(my $breakout = ""), - 'begin=i' => \ my $begin, - 'end=i' => \ my $end, - 'resolution=i' => \(my $resolution = 1) -) or usage; - -$es_host and $period_id and $metric_source and $metric_type and $begin and $end or usage; - -my $index_basename = "cdmv" . get_cdm_ver . get_cdm_rel; -my $script_name = basename($0); -my %data; -my $coder = JSON::MaybeXS->new->ascii->canonical->pretty; -$data{'name_format'} = get_name_format($es_host, $period_id, $metric_source, $metric_type); -$data{'metric_source'} = $metric_source; -$data{'metric_type'} = $metric_type; -get_metric_data($es_host, \%data, $begin, $end, $period_id, $metric_source, $metric_type, $resolution, split(/,/, $breakout)); -my $data_json = $coder->encode(\%data); -print $data_json; diff --git a/agent/util-scripts/pbench-get-primary-metric b/agent/util-scripts/pbench-get-primary-metric deleted file mode 100755 index e3f8b478e8..0000000000 --- a/agent/util-scripts/pbench-get-primary-metric +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/perl -## -*- mode: perl; indent-tabs-mode: t; perl-indent-level: 4 -*- -## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl - -# Usage: pbench-get-primary-metric -# -# You can get the iteraton-id with pbench-get-iterations. -# -# This script will output the primary metric for the benchmark. -# For example, this is "iops" for Fio. - -use strict; -use warnings; -use File::Basename; -use File::Find; -use REST::Client; -use Time::HiRes qw(gettimeofday); - -use lib $ENV{'pbench_lib_dir'}; -use JSON::MaybeXS; -use Data::Dumper; -use PbenchES qw(get_primary_metric); -use PbenchCDM qw(get_cdm_ver get_cdm_rel); -use PbenchBase qw(get_json_file put_json_file remove_element); - -if (scalar @ARGV != 2) { - print "Usage: pbench-get-metrics \n"; - exit 1; -} -my $debug = 0; -my $script_name = basename($0); -my $es_host = shift(@ARGV); -my $iter_id = shift(@ARGV); -my $depth = shift; -my $coder = JSON::MaybeXS->new->ascii->canonical; -my $primary_metric = get_primary_metric($es_host, $iter_id); -printf "%s\n", $primary_metric; diff --git a/agent/util-scripts/pbench-import-cdm b/agent/util-scripts/pbench-import-cdm deleted file mode 100755 index 7df91e3138..0000000000 --- a/agent/util-scripts/pbench-import-cdm +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use File::Basename; -use File::Find; - -use lib $ENV{'pbench_lib_dir'}; -use JSON; -use Data::Dumper; -use PbenchCDM qw(get_cdm_ver get_cdm_rel); -use PbenchBase qw(get_json_file put_json_file remove_element); - -my $coder = JSON->new; -my $count = 0; - -open(FH, ">import.log") || die "Could not open import.log";; - -sub process_dir { - my $dir = shift; - my $index = shift; - my $host = shift; - opendir(my $dh, $dir) || die "process_dir(): could not open directory: $!\n"; - for my $entry (grep(!/^\./, readdir($dh))) { - if (-d $dir . "/" . $entry) { - process_dir($dir . "/" . $entry, $index, $host); - } else { - # separate the UUID (like -23316532-b21a-448c-8634-fa91c2a3410c) from the index name) - if ($entry =~ /(\D+)\d*-[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}[^\.]*\.(ndjson|json$)/) { - my $doc_type = $1; - my $file_type = $2; - my $curl_cmd = ""; - if ($file_type eq "json") { - $curl_cmd = sprintf "curl --stderr /dev/null -X POST \"%s/%s\" -H \'Content-Type: application/json\' -d@%s", $host, "cdmv" . get_cdm_ver . get_cdm_rel . "-" . $doc_type . "/" . $doc_type, $dir . "/" . $entry; - } else { - $curl_cmd = sprintf "curl --stderr /dev/null -X POST \"%s/%s\" -H \'Content-Type: application/x-ndjson\' --data-binary @%s", $host, "cdmv" . get_cdm_ver . get_cdm_rel . "-" . $doc_type . "/" . $doc_type . "/_bulk", $dir . "/" . $entry; - } - my $output = `$curl_cmd`; - if ($? > 0) { - printf "curl failed, so aborting now\n"; - die; - } - my $json_ref = $coder->decode($output); - printf FH "%s\n", $output; - if ($file_type eq "ndjson" and $$json_ref{'errors'} eq JSON::true) { - print "import of $entry failed:\n\n$output\n\n"; - exit; - } - if ($file_type eq "json" and exists $$json_ref{'error'}) { - print "import of $entry failed:\n\n$output\n\n"; - exit; - } - $count++; - } else { - printf "file %s did not match the name format\n", $entry; - } - } - } - closedir $dh; -} - -my $script_name = basename($0); -my $es_dir = shift(@ARGV); -my $es_host = shift(@ARGV); - -# under $es_dir, process the CDM files -opendir(my $es_dh, $es_dir) || die "$script_name: could not open directory clients: $!\n"; -my @es_dirs = grep(!/^\./, (sort readdir($es_dh))); -closedir $es_dh; - -for my $index (@es_dirs) { - if ( -d $index) { - process_dir($es_dir . "/" . $index, $index, $es_host); - } -} -close(FH); -printf "Processed %d files\n", $count; diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests index 42bb131ac9..2bf786a979 100755 --- a/agent/util-scripts/unittests +++ b/agent/util-scripts/unittests @@ -59,8 +59,6 @@ cp -L ${_tdir}/../tool-scripts/datalog/*-convert ${_testopt}/tool-scripts/datalo let res=res+${?} cp -L ${_tdir}/../tool-scripts/postprocess/*-postprocess ${_testopt}/tool-scripts/postprocess/ let res=res+${?} -cp ${_tdir}/../tool-scripts/postprocess/*-postprocess-cdm ${_testopt}/tool-scripts/postprocess/ -let res=res+${?} cp -rL ${_tdir}/../lib/* ${_testopt}/lib/ let res=res+${?} cp -rL ${_tdir}/test-lib/* ${_testopt}/unittest-lib/