State Signals in Pbench

PBENCH-181 Added the use of `state-signals` to drive the Tool Meister sub-system. Also fixes a `pbench-tool-meister-start` race condition. Co-authored-by: Peter Portante <[email protected]>
distributed-system-analysis · Sep 30, 2022 · 76d1aeb · 76d1aeb
1 parent 835510d
commit 76d1aeb
Show file tree

Hide file tree

Showing 12 changed files with 264 additions and 297 deletions.
diff --git a/agent/requirements.txt b/agent/requirements.txt
@@ -9,3 +9,4 @@ python-pidfile
 redis
 requests
 sh
+state-signals>=1.0.1
diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt
@@ -144,7 +144,7 @@ Listening on http://localhost:8080/
 Hit Ctrl-C to quit.
 
 INFO pbench-tool-data-sink tm_log_capture -- Running Tool Meister log capture ...
-INFO pbench-tool-data-sink execute -- Tool Data Sink terminating
+INFO pbench-tool-data-sink execute_action -- Tool Data Sink terminating
 INFO pbench-tool-data-sink web_server_run -- Bottle web server exited
 --- mock-run/tm/pbench-tool-data-sink.err file contents
 +++ mock-run/tm/pbench-tool-data-sink.out file contents

diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt
@@ -410,7 +410,7 @@ install_check_output = mpstat: pbench-sysstat-12.0.3-1 is installed
 
 Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=<class 'pbench.agent.tool_data_sink.DataSinkWsgiServer.__init__.<locals>.DataSinkWsgiRequestHandler'>))...
 Hit Ctrl-C to quit.
-INFO pbench-tool-data-sink execute -- Tool Data Sink terminating
+INFO pbench-tool-data-sink execute_action -- Tool Data Sink terminating
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-a.example.com HTTP/1.1" 200 0
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-b.example.com HTTP/1.1" 200 0
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-c.example.com HTTP/1.1" 200 0

diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt
@@ -383,7 +383,7 @@ install_check_output = mpstat: pbench-sysstat-12.0.3-1 is installed
 
 Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=<class 'pbench.agent.tool_data_sink.DataSinkWsgiServer.__init__.<locals>.DataSinkWsgiRequestHandler'>))...
 Hit Ctrl-C to quit.
-INFO pbench-tool-data-sink execute -- Tool Data Sink terminating
+INFO pbench-tool-data-sink execute_action -- Tool Data Sink terminating
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-a.example.com HTTP/1.1" 200 0
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-b.example.com HTTP/1.1" 200 0
 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote-c.example.com HTTP/1.1" 200 0

diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-61.txt b/agent/util-scripts/gold/test-client-tool-meister/test-61.txt
@@ -253,7 +253,7 @@ install_check_output = mpstat: pbench-sysstat-12.0.3-1 is not installed
 
 Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=<class 'pbench.agent.tool_data_sink.DataSinkWsgiServer.__init__.<locals>.DataSinkWsgiRequestHandler'>))...
 Hit Ctrl-C to quit.
-INFO pbench-tool-data-sink execute -- Tool Data Sink terminating
+INFO pbench-tool-data-sink execute_action -- Tool Data Sink terminating
 INFO pbench-tool-data-sink tm_log_capture -- Running Tool Meister log capture ...
 INFO pbench-tool-data-sink web_server_run -- Bottle web server exited
 INFO pbench-tool-data-sink web_server_run -- Running Bottle web server ...

diff --git a/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt b/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt
@@ -16,22 +16,16 @@ pbench-tool-meister-start: 5. starting tool data sink
 pbench-tool-meister-start: 6a. starting localhost tool meister
 pbench-tool-meister-start: 7. waiting for all successfully created Tool Meister processes to show up as subscribers
 pbench-tool-meister-start: 8. Initialize persistent tools
-pbench-tool-meister-start: channel pbench-agent-cli-to-client payload, '{"action": "init", "kind": "ds", "status": "success"}'
 pbench-tool-meister-start: channel pbench-agent-cli-to-client payload, '{"action": "startup", "kind": "ds", "status": "success"}'
+pbench-tool-meister-start: constructed SignalExporter() object
+pbench-tool-meister-start: constructing SignalExporter() object using existing Redis connection, name: start-pbench-client
 pbench-tool-meister-start: next pbench-agent-cli-to-client
-pbench-tool-meister-start: next pbench-agent-cli-to-client
-pbench-tool-meister-start: payload from pbench-agent-cli-to-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-client', 'data': b'{"action": "init", "kind": "ds", "status": "success"}'}
 pbench-tool-meister-start: payload from pbench-agent-cli-to-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-client', 'data': b'{"action": "startup", "kind": "ds", "status": "success"}'}
-pbench-tool-meister-start: publish init on chan pbench-agent-cli-from-client
-pbench-tool-meister-start: published pbench-agent-cli-from-client
-pbench-tool-meister-stop: channel pbench-agent-cli-to-client payload, '{"action": "end", "kind": "ds", "status": "success"}'
-pbench-tool-meister-stop: constructed Redis() object
-pbench-tool-meister-stop: constructing Redis() object
-pbench-tool-meister-stop: next pbench-agent-cli-to-client
-pbench-tool-meister-stop: payload from pbench-agent-cli-to-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-to-client', 'data': b'{"action": "end", "kind": "ds", "status": "success"}'}
-pbench-tool-meister-stop: publish end on chan pbench-agent-cli-from-client
-pbench-tool-meister-stop: publish terminate on chan pbench-agent-cli-from-client
-pbench-tool-meister-stop: published pbench-agent-cli-from-client
+pbench-tool-meister-start: publish state signal for state init with metadata: {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
+pbench-tool-meister-stop: constructed SignalExporter() object
+pbench-tool-meister-stop: constructing SignalExporter() object using host localhost:17001, name: stop-pbench-client
+pbench-tool-meister-stop: publish state signal for state end with metadata: {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
+pbench-tool-meister-stop: publish state signal for state terminate with metadata: {'group': 'default', 'directory': None, 'args': {'interrupt': False}}
 pbench-tool-meister-stop: waiting for tool-data-sink (#####) to exit
 --- Finished test-51 test-start-stop-tool-meister (status=0)
 +++ pbench tree state
@@ -147,42 +141,32 @@ DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-
 DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-tms payload, '{"hostdata": {"alias": "bond", "all-fqdns": "agent.example.com bond.example.com", "all-ip-addresses": "192.168.0.1 172.0.0.1 127.0.0.1", "domain": "example.com", "fqdn": "agent.example.com", "ip-address": "172.0.0.1 127.0.0.1", "nis": "hostname[mock]: Local domain name not set", "short": "agent"}, "hostname": "testhost.example.com", "installs": {"mpstat": [0, "mpstat: pbench-sysstat-12.0.3-1 is installed"], "perf": [0, "perf: perf is installed"]}, "kind": "tm", "label": "", "pid": NNNNN, "seqno": "", "sha1": "(unknown)", "version": "(unknown)"}'
 DEBUG pbench-tool-data-sink execute -- publish pbench-agent-cli-to-client
 DEBUG pbench-tool-data-sink execute -- published pbench-agent-cli-to-client
-DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-client
-DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-client', 'data': b'{"action": "init", "group": "default", "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "args": null}'}
-DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-client payload, '{"action": "init", "group": "default", "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "args": null}'
+DEBUG pbench-tool-data-sink _fetch_action -- client 'start-pbench-client-00000000-0000-0000-0000-000000000001', action 'initialization', metadata None
+DEBUG pbench-tool-data-sink _fetch_action -- client 'start-pbench-client-00000000-0000-0000-0000-000000000001', action 'init', metadata {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
 DEBUG pbench-tool-data-sink _forward_tms -- publish pbench-agent-cli-to-tms
 DEBUG pbench-tool-data-sink _forward_tms -- published pbench-agent-cli-to-tms
-DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'action': 'init', 'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
+DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None, 'action': 'init'}
 DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-tms
 DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-tms', 'data': b'{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}'}
 DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-tms payload, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}'
 DEBUG pbench-tool-data-sink execute_action -- No persistent tools to init
-DEBUG pbench-tool-data-sink _send_client_status -- publish pbench-agent-cli-to-client
-DEBUG pbench-tool-data-sink _send_client_status -- published pbench-agent-cli-to-client
-DEBUG pbench-tool-data-sink _send_client_status -- posted client status, 'success'
-DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-client
-DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-client', 'data': b'{"action": "end", "group": "default", "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "args": null}'}
-DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-client payload, '{"action": "end", "group": "default", "directory": "/var/tmp/pbench-test-utils/pbench/mock-run/tools-default", "args": null}'
+DEBUG pbench-tool-data-sink _fetch_action -- client 'stop-pbench-client-00000000-0000-0000-0000-000000000001', action 'initialization', metadata None
+DEBUG pbench-tool-data-sink _fetch_action -- client 'stop-pbench-client-00000000-0000-0000-0000-000000000001', action 'end', metadata {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
 DEBUG pbench-tool-data-sink _forward_tms -- publish pbench-agent-cli-to-tms
 DEBUG pbench-tool-data-sink _forward_tms -- published pbench-agent-cli-to-tms
-DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'action': 'end', 'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None}
+DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'group': 'default', 'directory': '/var/tmp/pbench-test-utils/pbench/mock-run/tools-default', 'args': None, 'action': 'end'}
 DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-tms
 DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-tms', 'data': b'{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}'}
 DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-tms payload, '{"hostname": "testhost.example.com", "kind": "tm", "status": "success"}'
-DEBUG pbench-tool-data-sink _send_client_status -- publish pbench-agent-cli-to-client
-DEBUG pbench-tool-data-sink _send_client_status -- published pbench-agent-cli-to-client
-DEBUG pbench-tool-data-sink _send_client_status -- posted client status, 'success'
-DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-client
-DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-client: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-client', 'data': b'{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}'}
-DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-client payload, '{"action": "terminate", "args": {"interrupt": false}, "directory": null, "group": "default"}'
+DEBUG pbench-tool-data-sink _fetch_action -- client 'stop-pbench-client-00000000-0000-0000-0000-000000000001', action 'terminate', metadata {'group': 'default', 'directory': None, 'args': {'interrupt': False}}
 DEBUG pbench-tool-data-sink _forward_tms -- publish pbench-agent-cli-to-tms
 DEBUG pbench-tool-data-sink _forward_tms -- published pbench-agent-cli-to-tms
-DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'action': 'terminate', 'args': {'interrupt': False}, 'directory': None, 'group': 'default'}
+DEBUG pbench-tool-data-sink _forward_tms -- posted TM action message, {'group': 'default', 'directory': None, 'args': {'interrupt': False}, 'action': 'terminate'}
 DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-tms
 DEBUG pbench-tool-data-sink fetch_message -- payload from pbench-agent-cli-from-tms: {'type': 'message', 'pattern': None, 'channel': b'pbench-agent-cli-from-tms', 'data': b'{"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"}'}
 DEBUG pbench-tool-data-sink fetch_message -- channel pbench-agent-cli-from-tms payload, '{"hostname": "testhost.example.com", "kind": "tm", "status": "terminated"}'
-DEBUG pbench-tool-data-sink fetch_message -- next pbench-agent-cli-from-client
-INFO pbench-tool-data-sink execute -- Tool Data Sink terminating
+INFO pbench-tool-data-sink execute_action -- Tool Data Sink terminating
+DEBUG pbench-tool-data-sink execute -- done
 DEBUG pbench-tool-data-sink __exit__ -- web server stop
 INFO pbench-tool-data-sink web_server_run -- Bottle web server exited
 DEBUG pbench-tool-data-sink __exit__ -- Waiting for the web server thread to exit ...
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,3 +9,4 @@ python-pidfile @@
     redis
     requests
     sh
+    state-signals>=1.0.1