From 5fe6a7b24ae5da43e1bc264dac5d63db303bdefa Mon Sep 17 00:00:00 2001 From: Nick Dokos Date: Thu, 1 Jun 2023 17:14:25 -0400 Subject: [PATCH] Fixes for user-tool (#3440) * Fixes for user-tool The tool meister was treating tools options as a concatenated string that was passed as a single argument to `pbench-install/start/stop-tools`. They are now split into separate arguments, allowing the scripts to work correctly. `pbench-postprocess-tools` was being called without empty tool options. Instead of burrowing even deeper into the filesystem to grab the options, we get rid of most of that by rewriting it to use `pbench-list-tools -g -o` and parsing the output of that script. The parsing is ugly but is arguably less ugly than the file system ad-hockery that we used to go through. * Fix failures in the python tests `test_install` was using a NullObject for a logger, but the `install` method now logs its command. So we set up a basic logger to satisfy that need. Two other tests (`test_start` and `test_stop`) failed on an `is` test. We now test for equality, rather than identity and the tests pass. I'm not sure what caused these two failures, but the cause is probably the breakup of the options into separate arguments. * Fix legacy util-scripts tests Tests -51, -52 and -61 were failing because of the additional `install` logging. Tests -53, -56 and -57 now pass: the new output looks correct to me and the old output looks wrong. The util-scripts also pass in the CI container (run locally with `jenkins/run tox -- agent util-scripts`), except for -51 and -52 which fail with identical errors: - pbench-tool-meister-stop: waiting for tool-data-sink (#####) to exit The line is missing for some reason - I hope that the real Jenkins will pass them, but it's not clear to me why they fail locally. OTOH, they do pass locally when run with `tox -- agent util-scripts`. Explanations/theories/guesses are welcome. PBENCH-1172 --- .../gold/test-client-tool-meister/test-53.txt | 12 ++-- .../gold/test-client-tool-meister/test-56.txt | 44 +++++++++----- .../gold/test-client-tool-meister/test-57.txt | 44 +++++++++----- .../gold/test-client-tool-meister/test-61.txt | 6 ++ .../test-start-stop-tool-meister/test-51.txt | 58 ++++++++++--------- .../test-start-stop-tool-meister/test-52.txt | 58 ++++++++++--------- agent/util-scripts/pbench-postprocess-tools | 55 ++++++++++-------- lib/pbench/agent/tool_meister.py | 26 +++------ .../test/unit/agent/test_tool_meister.py | 7 +-- 9 files changed, 174 insertions(+), 136 deletions(-) diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt index 874b98c587..f2e0735860 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt @@ -17,6 +17,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42 @@ -28,6 +29,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/metadata.log /var/tmp/pbench-test-utils/pbench/mock-run/ssh_config @@ -171,6 +173,7 @@ port 17001 # Redis is now ready to exit, bye bye... --- mock-run/tm/redis.log file contents +++ mock-run/tm/tm.err file contents +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -190,15 +193,16 @@ WARNING pbench-tool-meister end_tools -- testhost.example.com: unexpected temp f INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up +testhost.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two testhost.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel testhost.example.com INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -218,10 +222,10 @@ testhost.example.com WARNING pbench-tool-meister end_tools -- testhost.example.c testhost.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com block,security_mitigations,sos parallel testhost.example.com INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.logs file contents diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt index 34f50228e5..ecee4dc8f1 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt @@ -23,6 +23,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat @@ -32,6 +33,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat @@ -40,6 +42,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42 @@ -52,6 +55,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat @@ -61,6 +65,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat @@ -69,6 +74,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/metadata.log /var/tmp/pbench-test-utils/pbench/mock-run/ssh_config @@ -176,6 +182,7 @@ Collecting system information /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/dcgm /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/mpstat === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com --interval=42 --options=forty-two @@ -191,14 +198,15 @@ INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT tool-data INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.out: === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com INFO pbench-tool-meister start -- Started persistent tool node-exporter, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter'] @@ -218,10 +226,10 @@ WARNING pbench-tool-meister end_tools -- remote-b.example.com: unexpected temp f INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com INFO pbench-tool-meister __exit__ -- remote-b.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.out: @@ -450,6 +458,7 @@ port 17001 # Redis is now ready to exit, bye bye... --- mock-run/tm/redis.log file contents +++ mock-run/tm/tm.err file contents +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -469,15 +478,16 @@ WARNING pbench-tool-meister end_tools -- testhost.example.com: unexpected temp f INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up +remote-a.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-a.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com remote-a.example.com INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com --interval=42 --options=forty-two @@ -493,6 +503,7 @@ remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.exampl remote-a.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com remote-a.example.com INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating +remote-b.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-b.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com block,security_mitigations,sos parallel remote-b.example.com INFO pbench-tool-meister _send_directory -- remote-b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com remote-b.example.com INFO pbench-tool-meister start -- Started persistent tool node-exporter, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter'] @@ -524,6 +535,7 @@ remote-c.example.com WARNING pbench-tool-meister end_tools -- remote-c.example.c remote-c.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-c.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/red:remote-c.example.com block,security_mitigations,sos parallel remote-c.example.com INFO pbench-tool-meister _send_directory -- remote-c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-c.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/red:remote-c.example.com remote-c.example.com INFO pbench-tool-meister __exit__ -- remote-c.example.com: terminating +testhost.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two testhost.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel testhost.example.com INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -543,22 +555,22 @@ testhost.example.com WARNING pbench-tool-meister end_tools -- testhost.example.c testhost.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com block,security_mitigations,sos parallel testhost.example.com INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.logs file contents @@ -682,10 +694,10 @@ sysinfo_full_hostname='testhost.example.com' /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/cp -rL /etc/ssh/ssh_config.d /var/tmp/pbench-test-utils/pbench/mock-run/ /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com block,security_mitigations,sos parallel diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt index 4af5ea4e09..2a2c7a595d 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt @@ -23,6 +23,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/blue:remote-b.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat @@ -32,6 +33,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/remote-a.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat @@ -40,6 +42,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42 @@ -52,6 +55,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/blue:remote-b.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat @@ -61,6 +65,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.cmd /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.file +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/remote-a.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat @@ -69,6 +74,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat-stop-postprocess.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat.cmd +/var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/mpstat.options /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/mpstat/online-cpus.txt /var/tmp/pbench-test-utils/pbench/mock-run/metadata.log /var/tmp/pbench-test-utils/pbench/mock-run/ssh_config @@ -155,6 +161,7 @@ pbench-tool-meister-stop: system information not collected when --interrupt spec /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/dcgm /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/mpstat === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com --interval=42 --options=forty-two @@ -168,14 +175,15 @@ INFO pbench-tool-meister _wait_for_process_with_kill -- Waiting for Transient to INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.out: === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister _send_directory -- remote-b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com INFO pbench-tool-meister start -- Started persistent tool node-exporter, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter'] @@ -193,10 +201,10 @@ INFO pbench-tool-meister stop -- Terminate issued for persistent tool node-expor INFO pbench-tool-meister _wait_for_process_with_kill -- Waiting for Persistent tool node-exporter process WARNING pbench-tool-meister end_tools -- remote-b.example.com: unexpected temp files blue:remote-b.example.com/node-exporter/node_exporter.file INFO pbench-tool-meister __exit__ -- remote-b.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.out: @@ -420,6 +428,7 @@ port 17001 # Redis is now ready to exit, bye bye... --- mock-run/tm/redis.log file contents +++ mock-run/tm/tm.err file contents +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -437,15 +446,16 @@ INFO pbench-tool-meister stop -- Terminate issued for persistent tool dcgm INFO pbench-tool-meister _wait_for_process_with_kill -- Waiting for Persistent tool dcgm process WARNING pbench-tool-meister end_tools -- testhost.example.com: unexpected temp files dcgm/dcgm-exporter.file INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up +remote-a.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-a.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com remote-a.example.com INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com --interval=42 --options=forty-two @@ -459,6 +469,7 @@ remote-a.example.com INFO pbench-tool-meister _wait_for_process_with_kill -- Wai remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com remote-a.example.com INFO pbench-tool-meister _send_directory -- remote-a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com remote-a.example.com INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating +remote-b.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-b.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com block,security_mitigations,sos parallel remote-b.example.com INFO pbench-tool-meister _send_directory -- remote-b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/blue:remote-b.example.com remote-b.example.com INFO pbench-tool-meister start -- Started persistent tool node-exporter, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter'] @@ -486,6 +497,7 @@ remote-c.example.com INFO pbench-tool-meister _wait_for_process_with_kill -- Wai remote-c.example.com INFO pbench-tool-meister _wait_for_process_with_kill -- Waiting for Persistent tool pcp process remote-c.example.com WARNING pbench-tool-meister end_tools -- remote-c.example.com: unexpected temp files red:remote-c.example.com/dcgm/dcgm-exporter.file,red:remote-c.example.com/pcp/pmcd.file remote-c.example.com INFO pbench-tool-meister __exit__ -- remote-c.example.com: terminating +testhost.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two testhost.example.com INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel testhost.example.com INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com testhost.example.com INFO pbench-tool-meister start -- Started persistent tool dcgm, ['/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter'] @@ -503,22 +515,22 @@ testhost.example.com INFO pbench-tool-meister stop -- Terminate issued for persi testhost.example.com INFO pbench-tool-meister _wait_for_process_with_kill -- Waiting for Persistent tool dcgm process testhost.example.com WARNING pbench-tool-meister end_tools -- testhost.example.com: unexpected temp files dcgm/dcgm-exporter.file testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-a.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-a.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +remote-b.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping remote-b.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop -testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 --options=forty-two " +testhost.example.com INFO pbench-tool-meister.logger-start log_raw_io_output -- mpstat: running "/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/datalog/mpstat-datalog 42 forty-two" testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: stopping testhost.example.com INFO pbench-tool-meister.logger-stop log_raw_io_output -- mpstat: post-processing following stop --- mock-run/tm/tm.logs file contents @@ -622,10 +634,10 @@ sysinfo_full_hostname='testhost.example.com' /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/cp -rL /etc/ssh/ssh_config.d /var/tmp/pbench-test-utils/pbench/mock-run/ /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/dcgm-exporter -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/node_exporter /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com block,security_mitigations,sos parallel /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.lite.NNNNN.nnnnnnnn/remote-a.example.com block,security_mitigations,sos parallel diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-61.txt b/agent/util-scripts/gold/test-client-tool-meister/test-61.txt index 8a55de3ac1..ec0a4d8748 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-61.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-61.txt @@ -72,9 +72,11 @@ ERROR - "pbench-tool-meister-start --sysinfo='block,security_mitigations,sos' 'l /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/dcgm /var/tmp/pbench-test-utils/pbench/tools-v1-lite/testhost.example.com/mpstat === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating === /var/tmp/pbench-test-utils/pbench/remote/remote-a.example.com/tmp/tm.out: === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.err: +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister __exit__ -- remote-b.example.com: terminating === /var/tmp/pbench-test-utils/pbench/remote/remote-b.example.com/tmp/tm.out: === /var/tmp/pbench-test-utils/pbench/remote/remote-c.example.com/tmp/tm.err: @@ -283,13 +285,17 @@ port 17001 # Redis is now ready to exit, bye bye... --- mock-run/tm/redis.log file contents +++ mock-run/tm/tm.err file contents +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up +remote-a.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-a.example.com INFO pbench-tool-meister __exit__ -- remote-a.example.com: terminating +remote-b.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two remote-b.example.com INFO pbench-tool-meister __exit__ -- remote-b.example.com: terminating remote-c.example.com INFO pbench-tool-meister __exit__ -- remote-c.example.com: terminating +testhost.example.com INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install --interval=42 --options=forty-two testhost.example.com INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating --- mock-run/tm/tm.logs file contents +++ mock-run/tm/tm.out file contents diff --git a/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt b/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt index d499594e89..9133fd2ca8 100644 --- a/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt +++ b/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt @@ -200,6 +200,8 @@ port 17001 DEBUG pbench-tool-meister daemon -- re-constructing Redis server object DEBUG pbench-tool-meister daemon -- re-constructed Redis server object DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'hostname': 'testhost.example.com', 'instance_uuid': '00000000-0000-0000-0000-000000000001', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_group': 'default', 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install +INFO pbench-tool-meister install -- perf: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/perf --install --record-opts=-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions --report-opts=-I -g DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms DEBUG pbench-tool-meister driver -- waiting ... @@ -231,33 +233,35 @@ DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-t +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'hostname': 'testhost.example.com', 'instance_uuid': '00000000-0000-0000-0000-000000000001', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_group': 'default', 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} -testhost.example.com 0001 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms -testhost.example.com 0002 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms -testhost.example.com 0003 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0004 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: wait_for_command startup -testhost.example.com 0005 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0006 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}'} -testhost.example.com 0007 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}' -testhost.example.com 0008 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} -testhost.example.com 0009 DEBUG pbench-tool-meister driver -- acting ... init_tools, {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} -testhost.example.com 0010 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} -testhost.example.com 0011 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' -testhost.example.com 0012 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0013 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0014 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}'} -testhost.example.com 0015 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}' -testhost.example.com 0016 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} -testhost.example.com 0017 DEBUG pbench-tool-meister driver -- acting ... end_tools, {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} -testhost.example.com 0018 DEBUG pbench-tool-meister end_tools -- testhost.example.com: deleting persistent tool tmp directory default /var/tmp/pbench-test-utils/pbench/mock-run/tools-default/testhost.example.com -testhost.example.com 0019 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} -testhost.example.com 0020 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' -testhost.example.com 0021 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0022 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0023 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}'} -testhost.example.com 0024 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}' -testhost.example.com 0025 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'terminate', 'args': {'interrupt': False}, 'directory': None, 'group': 'default'} -testhost.example.com 0026 INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -testhost.example.com 0027 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"} +testhost.example.com 0001 INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install +testhost.example.com 0002 INFO pbench-tool-meister install -- perf: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/perf --install --record-opts=-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions --report-opts=-I -g +testhost.example.com 0003 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms +testhost.example.com 0004 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms +testhost.example.com 0005 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0006 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: wait_for_command startup +testhost.example.com 0007 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0008 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}'} +testhost.example.com 0009 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}' +testhost.example.com 0010 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} +testhost.example.com 0011 DEBUG pbench-tool-meister driver -- acting ... init_tools, {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} +testhost.example.com 0012 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} +testhost.example.com 0013 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' +testhost.example.com 0014 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0015 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0016 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}'} +testhost.example.com 0017 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "group": "default"}' +testhost.example.com 0018 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} +testhost.example.com 0019 DEBUG pbench-tool-meister driver -- acting ... end_tools, {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'group': 'default'} +testhost.example.com 0020 DEBUG pbench-tool-meister end_tools -- testhost.example.com: deleting persistent tool tmp directory default /var/tmp/pbench-test-utils/pbench/mock-run/tools-default/testhost.example.com +testhost.example.com 0021 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} +testhost.example.com 0022 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' +testhost.example.com 0023 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0024 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0025 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}'} +testhost.example.com 0026 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}' +testhost.example.com 0027 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'terminate', 'args': {'interrupt': False}, 'directory': None, 'group': 'default'} +testhost.example.com 0028 INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating +testhost.example.com 0029 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"} --- mock-run/tm/tm.logs file contents +++ mock-run/tm/tm.out file contents --- mock-run/tm/tm.out file contents diff --git a/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt b/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt index c1201bc643..c1499b8f79 100644 --- a/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt +++ b/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt @@ -200,6 +200,8 @@ port 17001 DEBUG pbench-tool-meister daemon -- re-constructing Redis server object DEBUG pbench-tool-meister daemon -- re-constructed Redis server object DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'hostname': 'testhost.example.com', 'instance_uuid': '00000000-0000-0000-0000-000000000001', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_group': 'mygroup', 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install +INFO pbench-tool-meister install -- perf: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/perf --install --record-opts=-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions --report-opts=-I -g DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms DEBUG pbench-tool-meister driver -- waiting ... @@ -231,33 +233,35 @@ DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-t +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'hostname': 'testhost.example.com', 'instance_uuid': '00000000-0000-0000-0000-000000000001', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_group': 'mygroup', 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} -testhost.example.com 0001 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms -testhost.example.com 0002 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms -testhost.example.com 0003 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0004 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: wait_for_command startup -testhost.example.com 0005 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0006 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}'} -testhost.example.com 0007 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}' -testhost.example.com 0008 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} -testhost.example.com 0009 DEBUG pbench-tool-meister driver -- acting ... init_tools, {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} -testhost.example.com 0010 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} -testhost.example.com 0011 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' -testhost.example.com 0012 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0013 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0014 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}'} -testhost.example.com 0015 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}' -testhost.example.com 0016 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} -testhost.example.com 0017 DEBUG pbench-tool-meister driver -- acting ... end_tools, {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} -testhost.example.com 0018 DEBUG pbench-tool-meister end_tools -- testhost.example.com: deleting persistent tool tmp directory mygroup /var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup/testhost.example.com -testhost.example.com 0019 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} -testhost.example.com 0020 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' -testhost.example.com 0021 DEBUG pbench-tool-meister driver -- waiting ... -testhost.example.com 0022 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms -testhost.example.com 0023 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "mygroup"}'} -testhost.example.com 0024 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "mygroup"}' -testhost.example.com 0025 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'terminate', 'args': {'interrupt': False}, 'directory': None, 'group': 'mygroup'} -testhost.example.com 0026 INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating -testhost.example.com 0027 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"} +testhost.example.com 0001 INFO pbench-tool-meister install -- mpstat: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --install +testhost.example.com 0002 INFO pbench-tool-meister install -- perf: install_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/perf --install --record-opts=-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions --report-opts=-I -g +testhost.example.com 0003 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms +testhost.example.com 0004 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms +testhost.example.com 0005 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0006 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: wait_for_command startup +testhost.example.com 0007 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0008 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}'} +testhost.example.com 0009 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "init", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}' +testhost.example.com 0010 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} +testhost.example.com 0011 DEBUG pbench-tool-meister driver -- acting ... init_tools, {'action': 'init', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} +testhost.example.com 0012 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} +testhost.example.com 0013 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' +testhost.example.com 0014 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0015 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0016 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}'} +testhost.example.com 0017 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "end", "args": null, "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup", "group": "mygroup"}' +testhost.example.com 0018 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} +testhost.example.com 0019 DEBUG pbench-tool-meister driver -- acting ... end_tools, {'action': 'end', 'args': None, 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup', 'group': 'mygroup'} +testhost.example.com 0020 DEBUG pbench-tool-meister end_tools -- testhost.example.com: deleting persistent tool tmp directory mygroup /var/tmp/pbench-test-utils/pbench/mock-run/tools-mygroup/testhost.example.com +testhost.example.com 0021 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "success"} +testhost.example.com 0022 DEBUG pbench-tool-meister _send_client_status -- posted client status message, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}' +testhost.example.com 0023 DEBUG pbench-tool-meister driver -- waiting ... +testhost.example.com 0024 DEBUG pbench-tool-meister fetch_message -- next pbench-agent-cli-to-tms +testhost.example.com 0025 DEBUG pbench-tool-meister fetch_message -- payload from pbench-agent-cli-to-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-tms', 'data': b'{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "mygroup"}'} +testhost.example.com 0026 DEBUG pbench-tool-meister fetch_message -- channel pbench-agent-cli-to-tms payload, '{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "mygroup"}' +testhost.example.com 0027 DEBUG pbench-tool-meister wait_for_command -- testhost.example.com: msg - {'action': 'terminate', 'args': {'interrupt': False}, 'directory': None, 'group': 'mygroup'} +testhost.example.com 0028 INFO pbench-tool-meister __exit__ -- testhost.example.com: terminating +testhost.example.com 0029 DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-tms {"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"} --- mock-run/tm/tm.logs file contents +++ mock-run/tm/tm.out file contents --- mock-run/tm/tm.out file contents diff --git a/agent/util-scripts/pbench-postprocess-tools b/agent/util-scripts/pbench-postprocess-tools index 11a3ae9579..0b84d4ef2f 100755 --- a/agent/util-scripts/pbench-postprocess-tools +++ b/agent/util-scripts/pbench-postprocess-tools @@ -1,5 +1,5 @@ #!/bin/bash -# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*- +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 4; sh-indentation: 4; tab-width: 4 -*- script_path="$(dirname ${0})" script_name="$(basename ${0})" @@ -70,7 +70,7 @@ fi # This tool group's directory which stores the list of tools and their # options, etc. -tool_group_dir="$(verify_tool_group "${group}")" +tool_group_dir=$(verify_tool_group ${group}) if [[ ${?} -ne 0 || -z "${tool_group_dir}" ]]; then exit 1 fi @@ -82,37 +82,42 @@ if [[ ! -d "${tool_output_dir}" ]]; then exit 1 fi +# We run `pbench-list-tools', collect the output and parse it +# to determine which host runs which tools with what options. +tmp=${pbench_tmp}/pbench-postprocess-tools.$$ +trap "rm -f ${tmp}" EXIT QUIT INT TERM +pbench-list-tools --group ${group} --with-option >${tmp} 2>/dev/null + let failures=0 -for dirent in $(/bin/ls -1 ${tool_group_dir}); do - if [[ "${dirent}" == "__trigger__" ]]; then - # Ignore trigger files - continue - elif [[ ! -d ${tool_group_dir}/${dirent} ]]; then - # Skip spurious files of ${tool_group_dir} - warn_log "[${script_name}] \"${this_tool_file}\" is a file in \"${tool_group_dir}\"; that should not happen. Please consider deleting it." - continue - fi +while read hostentry ;do + # Parse an entry from the output of `pbench-list-tools` above. + # The format is: "group: ; host: ; tools: , ... + IFS=';' read group_spec host_spec tools_spec <<<"${hostentry}" + IFS=':' read dummy host junk <<<"${host_spec}" + IFS=':' read dummy tools_entry junk <<<"${tools_spec}" + host=${host# *} + # Associative array: the keys are the tool names, and the values are the options + declare -A tools=() + IFS=',' read -a otools <<<"${tools_entry# *}" + for tool in ${otools[@]} ;do + IFS=' ' read -a atool <<<${tool} + tools[${atool[0]}]=${atool[@]:1} + done + # FIXME: add support for label applied to the hostname directory. - host_tool_output_dir="${tool_output_dir}/${dirent}" + host_tool_output_dir="${tool_output_dir}/${host}" if [[ -d "${host_tool_output_dir}" ]]; then - for filent in $(/bin/ls -1 ${tool_group_dir}/${dirent}); do - if [[ ! -f "${tool_group_dir}/${dirent}/${filent}" ]]; then - # Ignore unrecognized directories or symlinks - continue - fi - if [[ "${filent}" == "__label__" ]]; then - # Ignore label files - continue - fi - if [[ ! -x ${pbench_bin}/tool-scripts/${filent} ]]; then + for tool_name in ${!tools[@]} ;do + if [[ ! -x "${pbench_bin}/tool-scripts/${tool_name}" ]]; then # Ignore unrecognized tools continue fi - if [[ "${filent}" == "node-exporter" || "${filent}" == "dcgm" || "${filent}" == "pcp" || "${filent}" == "pcp-transient" ]]; then + if [[ "${tool_name}" == "node-exporter" || "${tool_name}" == "dcgm" || "${tool_name}" == "pcp" || "${tool_name}" == "pcp-transient" ]]; then # To be removed when converted to python continue fi - ${pbench_bin}/tool-scripts/${filent} --postprocess --dir=${host_tool_output_dir} "${tool_opts[@]}" >> ${host_tool_output_dir}/postprocess.log 2>&1 + tool_options="${tools[${tool_name}]}" + ${pbench_bin}/tool-scripts/${tool_name} --postprocess --dir="${host_tool_output_dir}" ${tool_options} >> ${host_tool_output_dir}/postprocess.log 2>&1 if [[ ${?} -ne 0 ]]; then cat ${host_tool_output_dir}/postprocess.log >&2 (( failures++ )) @@ -122,6 +127,6 @@ for dirent in $(/bin/ls -1 ${tool_group_dir}); do warn_log "[${script_name}] Missing tool output directory, '${host_tool_output_dir}'" (( failures++ )) fi -done +done < ${tmp} exit ${failures} diff --git a/lib/pbench/agent/tool_meister.py b/lib/pbench/agent/tool_meister.py index 4bdebcb1b2..7ef736c40f 100644 --- a/lib/pbench/agent/tool_meister.py +++ b/lib/pbench/agent/tool_meister.py @@ -41,6 +41,7 @@ import logging.handlers import os from pathlib import Path +import shlex import shutil import signal import subprocess @@ -271,11 +272,8 @@ def install(self) -> InstallationResult: """Synchronously runs the tool --install mode capturing the return code and output, returning them as a tuple to the caller. """ - args = [ - self.tool_script, - "--install", - self.tool_opts, - ] + args = [self.tool_script, "--install"] + shlex.split(self.tool_opts) + self.logger.info("%s: install_tool -- %s", self.name, " ".join(args)) cp = subprocess.run( args, stdin=None, @@ -305,12 +303,9 @@ def start(self, tool_dir: Path): self.stop_process is None ), f"Tool({self.name}) has an unexpected stop process running" - args = [ - self.tool_script, - "--start", - f"--dir={tool_dir}", - self.tool_opts, - ] + args = [self.tool_script, "--start", f"--dir={tool_dir}"] + shlex.split( + self.tool_opts + ) self.logger.info("%s: start_tool -- %s", self.name, " ".join(args)) self.start_process = self._create_process_with_logger(args, tool_dir, "start") self.tool_dir = tool_dir @@ -342,12 +337,9 @@ def stop(self): break time.sleep(0.1) - args = [ - self.tool_script, - "--stop", - f"--dir={self.tool_dir}", - self.tool_opts, - ] + args = [self.tool_script, "--stop", f"--dir={self.tool_dir}"] + shlex.split( + self.tool_opts + ) self.logger.info("%s: stop_tool -- %s", self.name, " ".join(args)) self.stop_process = self._create_process_with_logger( args, self.tool_dir, "stop" diff --git a/lib/pbench/test/unit/agent/test_tool_meister.py b/lib/pbench/test/unit/agent/test_tool_meister.py index 2f3ad79867..74ecbc75fe 100644 --- a/lib/pbench/test/unit/agent/test_tool_meister.py +++ b/lib/pbench/test/unit/agent/test_tool_meister.py @@ -504,12 +504,11 @@ class TestTransientTool: @staticmethod def test_install(monkeypatch): mocked_install_dir = MockedPath() - mocked_logger = NullObject() tool = TransientTool( name="test-tool", tool_opts="opt1;opt2", pbench_install_dir=mocked_install_dir, - logger=mocked_logger, + logger=logging.getLogger("pbench.test"), ) class CompletedProcess(NamedTuple): @@ -559,7 +558,7 @@ def test_start(caplog): assert tool.start_process.args[0] is tool.tool_script assert tool.start_process.args[1] == "--start" assert tool.start_process.args[2] == f"--dir={the_tool_dir}" - assert tool.start_process.args[3] is tool.tool_opts + assert tool.start_process.args[3] == tool.tool_opts assert tool.start_process.tool_dir is the_tool_dir assert tool.start_process.ctx_name == "start" assert len(caplog.record_tuples) == 1 @@ -580,7 +579,7 @@ def test_stop(caplog): assert tool.stop_process.args[0] is tool.tool_script assert tool.stop_process.args[1] == "--stop" assert tool.stop_process.args[2] == f"--dir={the_tool_dir}" - assert tool.stop_process.args[3] is tool.tool_opts + assert tool.stop_process.args[3] == tool.tool_opts assert tool.stop_process.tool_dir is the_tool_dir assert tool.stop_process.ctx_name == "stop" assert len(caplog.record_tuples) == 1