PBENCH-697 Delete stockpile submodule (#2971)

PBENCH-697

- Delete the git submodule.
- Delete the stockpile patch file and clean up the agent spec file.
- Delete stockpile collection function and reference in pbench-syssinfo-dump.
- Delete references in pbench-run-benchmark.pl and
  lib/PbenchAnsbile.pm and delete comment in Systat.pm.
- Delete references in docs/CONFIG_DATA.md.
- Delete reference in lib/pbench/agent/constants.py.
- Delete references in build.sh and pyproject.toml.
- Delete references in agent/Makefile.
- Clean up agent/base.
- Clean up .flake8 and .pre-commit-config.yaml.
- Fix tests

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 doctests = True
 exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*web-server/*,
-        *agent/stockpile/*,*agent/bench-scripts/test-bin/fio-histo-log-pctiles.py,
+        *agent/bench-scripts/test-bin/fio-histo-log-pctiles.py,
         *agent/bench-scripts/tests/pbench-trafficgen/test-39.trafficgen/*,
         *agent/bench-scripts/tests/pbench-trafficgen/test-40.trafficgen/*
 ignore = E121,E123,E126,E203,E226,E501,W503,E704

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "agent/stockpile"]
-	path = agent/stockpile
-	url = https://github.com/cloud-bulldozer/stockpile

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 # black complains loudly that fio-histo-log-pctile.py is not a python file
 # because it isn't, so ignore it.
-exclude: (agent/bench-scripts/test-bin/fio-histo-log-pctiles.py|agent/stockpile/roles|agent/bench-scripts/tests/pbench-trafficgen/test-39.trafficgen|agent/bench-scripts/tests/pbench-trafficgen/test-40.trafficgen)
+exclude: (agent/bench-scripts/test-bin/fio-histo-log-pctiles.py|agent/bench-scripts/tests/pbench-trafficgen/test-39.trafficgen|agent/bench-scripts/tests/pbench-trafficgen/test-40.trafficgen)
 repos:
   - repo: https://gitlab.com/pycqa/flake8
     rev: 4.0.1

agent/Makefile

@@ -10,7 +10,6 @@ ANSIBLEDIR=${DESTDIR}/ansible
 BENCHDIR=${DESTDIR}/bench-scripts
 CONFIGDIR=${DESTDIR}/config
 LIBDIR=${DESTDIR}/lib
-STOCKPILEDIR=${DESTDIR}/stockpile
 TOOLDIR=${DESTDIR}/tool-scripts
 UTILDIR=${DESTDIR}/util-scripts
 
@@ -241,8 +240,6 @@ install: install-destdir install-python3-setup install-bench-scripts install-too
 	${INSTALL} ${INSTALLOPTS} ${CONFIGDIR}/benchmark
 	${COPY} config ${DESTDIR}
 	${COPY} templates ${DESTDIR}
-	${INSTALL} ${INSTALLOPTS} ${STOCKPILEDIR}
-	${COPY} stockpile ${DESTDIR}
 
 install-destdir:
 	${INSTALL} ${INSTALLOPTS} ${DESTDIR}

agent/base

@@ -250,11 +250,7 @@ function verify_common_bench_script_options {
 function generate_inventory {
     local component="${1}"
     local preamble=""
-    if [[ ${component} == "stockpile" ]]; then
-        preamble="[stockpile]\n%s\n\n[all]\n"
-    else
-        preamble="[controller]\n%s\n\n[remote]\n"
-    fi
+    preamble="[controller]\n%s\n\n[remote]\n"
     printf -- ${preamble} "${_pbench_full_hostname}"
     for dirent in $(/bin/ls -1 ${tool_group_dir}); do
         if [[ "${dirent}" == "__trigger__" ]]; then

agent/bench-scripts/pbench-run-benchmark.pl

@@ -28,7 +28,7 @@
                         get_pbench_datetime);
 use PbenchAnsible    qw(ssh_hosts ping_hosts copy_files_to_hosts copy_files_from_hosts
                         remove_files_from_hosts remove_dir_from_hosts create_dir_hosts
-                        sync_dir_from_hosts verify_success stockpile_hosts);
+                        sync_dir_from_hosts verify_success);
 
 my %defaults = (
     "num-samples" => 1,
@@ -183,14 +183,6 @@ sub find_common_parameters {
     mkdir($es_dir . "/" . $es_subdir);
 }
 
-# Use stockpile to collect configuration information
-my @config_hosts = split(/,/, $params{"clients"});
-if (-e "/tmp/stockpile") {
-    print "Collecting confguration information with stockpile\n";
-    stockpile_hosts(\@config_hosts, $base_bench_dir,"stockpile_output_path=".
-                    $base_bench_dir . "/stockpile.json");
-}
-
 my $tool_group = $params{"tool-group"};
 my $iteration_id = 0;
 my $run_id = get_uuid;
@@ -436,34 +428,4 @@ sub find_common_parameters {
     print "finished\n";
 }
 
-# Convert the stockpile data with scribe, then create CDM docs in ./es/config
-if (-e $base_bench_dir . "/stockpile.json") {
-    system('python3 -m venv /var/lib/pbench-agent/tmp/scribe && cd ' .
-        $base_bench_dir . ' && scribe -t stockpile -ip ./stockpile.json >scribe.json');
-    open(my $scribe_fh, "<" . $base_bench_dir . "/scribe.json") || die "Could not open " .
-        $base_bench_dir . "/scribe.json";
-    my $json_text = "";
-    # Instead of 1 json document, there are actually multiple documents, but no separator
-    # between them or organized in an array
-    while (<$scribe_fh>) {
-        $json_text .= $_;
-        if (/^\}/) { # Assume this is the end of a json doc
-            my %config_doc = create_config_doc(\%last_run_doc, from_json($json_text));
-            if ($config_doc{'cdm'}{'doctype'} =~ /^config_(.+)/) {
-                my $config_subname = $1;
-                if (exists $config_doc{'config'}{'id'}) {
-                    put_json_file(\%config_doc, $es_dir . "/config/" . $config_doc{'cdm'}{'doctype'} .
-                                "-" . $config_doc{'config'}{'id'} . ".json");
-                } else {
-                    printf "Error: config doc's config.%s not found\n", $config_subname;
-                }
-            } else {
-                printf "Error: config doc's cdm.doctype does not start with \"config_\"\n";
-            }
-            $json_text = "";
-        }
-    }
-    close($scribe_fh);
-}
-
 printf "Run complete\n\n";

agent/bench-scripts/tests/pbench-fio/test-14.txt

@@ -84,7 +84,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-14 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-17.txt

@@ -84,7 +84,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-17 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-18.txt

@@ -87,7 +87,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-18 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-21.txt

@@ -87,7 +87,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-21 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-26.txt

@@ -87,7 +87,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-26 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-27.txt

@@ -87,7 +87,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-27 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-fio/test-29.txt

@@ -87,7 +87,7 @@ The following options are available:
 		set the histogram logging interval in seconds (default 10)
 
 	--sysinfo=str            str= comma separated values of sysinfo to be collected
-		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 
 	--unique-ports           Use unique ports for each server
 --- Finished test-29 pbench-fio (status=1)

agent/bench-scripts/tests/pbench-uperf/test-59.txt

@@ -46,7 +46,7 @@
 				   part of the label, just use --tool-label-pattern= to tell this script
 				   the prefix pattern to use for CPU information.
 	--sysinfo=str,             str= comma separated values of sysinfo to be collected
-	                                available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+	                                available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 --- Finished test-59 pbench-uperf (status=1)
 +++ pbench tree state
 /var/tmp/pbench-test-bench/pbench-agent

agent/bench-scripts/tests/pbench-uperf/test-60.txt

@@ -46,7 +46,7 @@
 				   part of the label, just use --tool-label-pattern= to tell this script
 				   the prefix pattern to use for CPU information.
 	--sysinfo=str,             str= comma separated values of sysinfo to be collected
-	                                available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+	                                available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 --- Finished test-60 pbench-uperf (status=1)
 +++ pbench tree state
 /var/tmp/pbench-test-bench/pbench-agent

agent/bench-scripts/tests/pbench-user-benchmark/test-33.txt

@@ -14,7 +14,7 @@ Usage: pbench-user-benchmark [options] -- <script to run>
 		                      provided as arguments to the script;
 		                        NOTE: --iteration-list is not compatible with --use-tool-triggers
 		--sysinfo=str,[str]   comma separated values of system information to be collected;
-		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 		--pbench-pre=str      path to the script which will be executed before tools are started
 		                        NOTE: --pbench-pre is not compatible with --use-tool-triggers
 		--pbench-post=str     path to the script which will be executed after tools are stopped and postprocessing is complete

agent/bench-scripts/tests/pbench-user-benchmark/test-34.txt

@@ -14,7 +14,7 @@ Usage: pbench-user-benchmark [options] -- <script to run>
 		                      provided as arguments to the script;
 		                        NOTE: --iteration-list is not compatible with --use-tool-triggers
 		--sysinfo=str,[str]   comma separated values of system information to be collected;
-		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 		--pbench-pre=str      path to the script which will be executed before tools are started
 		                        NOTE: --pbench-pre is not compatible with --use-tool-triggers
 		--pbench-post=str     path to the script which will be executed after tools are stopped and postprocessing is complete

agent/bench-scripts/tests/pbench-user-benchmark/test-35.txt

@@ -14,7 +14,7 @@ Usage: pbench-user-benchmark [options] -- <script to run>
 		                      provided as arguments to the script;
 		                        NOTE: --iteration-list is not compatible with --use-tool-triggers
 		--sysinfo=str,[str]   comma separated values of system information to be collected;
-		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 		--pbench-pre=str      path to the script which will be executed before tools are started
 		                        NOTE: --pbench-pre is not compatible with --use-tool-triggers
 		--pbench-post=str     path to the script which will be executed after tools are stopped and postprocessing is complete

agent/bench-scripts/tests/pbench-user-benchmark/test-36.txt

@@ -14,7 +14,7 @@ Usage: pbench-user-benchmark [options] -- <script to run>
 		                      provided as arguments to the script;
 		                        NOTE: --iteration-list is not compatible with --use-tool-triggers
 		--sysinfo=str,[str]   comma separated values of system information to be collected;
-		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, stockpile, topology
+		                        available: default, none, all, ara, block, insights, kernel_config, libvirt, security_mitigations, sos, topology
 		--pbench-pre=str      path to the script which will be executed before tools are started
 		                        NOTE: --pbench-pre is not compatible with --use-tool-triggers
 		--pbench-post=str     path to the script which will be executed after tools are stopped and postprocessing is complete