From f6a49cf2676c7537f39a99f89404a8864fc94466 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 25 May 2023 09:21:28 -0400 Subject: [PATCH 1/3] add generator for templated flag testdata Signed-off-by: Andrew Mason --- Makefile | 5 ++++- go/flags/endtoend/mysqlctl.txt | 2 +- go/flags/endtoend/mysqlctld.txt | 2 +- go/flags/endtoend/vtaclcheck.txt | 2 +- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtctlclient.txt | 2 +- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vtexplain.txt | 2 +- go/flags/endtoend/vtgate.txt | 2 +- go/flags/endtoend/vtgr.txt | 2 +- go/flags/endtoend/vtorc.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/flags/endtoend/zkctl.txt | 2 +- go/flags/endtoend/zkctld.txt | 2 +- tools/generate_flag_testdata.sh | 17 +++++++++++++++++ 16 files changed, 35 insertions(+), 15 deletions(-) create mode 100755 tools/generate_flag_testdata.sh diff --git a/Makefile b/Makefile index 0e2cf442155..719425748a7 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ export REWRITER=go/vt/sqlparser/rewriter.go # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time. .NOTPARALLEL: -.PHONY: all build install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools generate_ci_workflows +.PHONY: all build install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools generate_ci_workflows generate-flag-testdata all: build @@ -457,6 +457,9 @@ vtadmin_authz_testgen: generate_ci_workflows: cd test && go run ci_workflow_gen.go && cd .. +generate-flag-testdata: + ./tools/generate_flag_testdata.sh + release-notes: go run ./go/tools/release-notes --from "$(FROM)" --to "$(TO)" --version "$(VERSION)" --summary "$(SUMMARY)" diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index 2472528260b..4af44804749 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -18,7 +18,7 @@ Global flags: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 8b0ca101cb4..6fbbd059492 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -6,7 +6,7 @@ Usage of mysqlctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index d28eb7351b9..001d3a5b192 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -4,7 +4,7 @@ Usage of vtaclcheck: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index f825e764e6c..aa9c98ea666 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -22,7 +22,7 @@ Usage of vtbackup: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index cbb6d74cb9d..7fa186acbd0 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -4,7 +4,7 @@ Usage of vtctlclient: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --datadog-agent-host string host to send spans to. if empty, no tracing will be done diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 7bc57f53630..4e5357c0423 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -21,7 +21,7 @@ Usage of vtctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index 29ffb35f343..4d39f0d0734 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -4,7 +4,7 @@ Usage of vtexplain: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --dbname string Optional database target to override normal routing diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index bb576961c29..1c2b4b4fcf8 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -14,7 +14,7 @@ Usage of vtgate: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index c553ee8015c..cbee799bf3e 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -5,7 +5,7 @@ Usage of vtgr: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index b38cca7374b..3f184b7e8d5 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -10,7 +10,7 @@ Usage of vtorc: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index bdb55724d0b..311f4330f0d 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -37,7 +37,7 @@ Usage of vttablet: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 4254f58c398..eaaca50b9b8 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -18,7 +18,7 @@ Usage of vttestserver: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index f52a5f7fe1b..36ddace46db 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -3,7 +3,7 @@ Usage of zkctl: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index e2187ead03f..76f19523660 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -3,7 +3,7 @@ Usage of zkctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/tools/generate_flag_testdata.sh b/tools/generate_flag_testdata.sh new file mode 100755 index 00000000000..00e408be7f5 --- /dev/null +++ b/tools/generate_flag_testdata.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +templatize_help_text() { + $1 --help 2>&1 | \ + sed 's/{{/{{ "{{/' | \ + sed 's/}}/}}" }}/' | \ + sed 's/Paths to search for config files in. (default .*)/Paths to search for config files in. (default [{{ .Workdir }}])/' + return 0 +} + +export -f templatize_help_text + +find go/flags/endtoend -iname '*.txt' | \ + xargs basename | \ + cut -d. -f1 | \ + xargs -I{} \ + bash -c 'templatize_help_text "{}" >go/flags/endtoend/{}.txt' \ No newline at end of file From 65c4b2533a8e1337a5a2f874f3e5c54152cef550 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 25 May 2023 13:38:40 -0400 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Matt Lord Signed-off-by: Andrew Mason --- tools/generate_flag_testdata.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/generate_flag_testdata.sh b/tools/generate_flag_testdata.sh index 00e408be7f5..0f70d65947e 100755 --- a/tools/generate_flag_testdata.sh +++ b/tools/generate_flag_testdata.sh @@ -1,10 +1,11 @@ #!/bin/bash +set -euo pipefail templatize_help_text() { $1 --help 2>&1 | \ - sed 's/{{/{{ "{{/' | \ - sed 's/}}/}}" }}/' | \ - sed 's/Paths to search for config files in. (default .*)/Paths to search for config files in. (default [{{ .Workdir }}])/' + sed -e 's/{{/{{ "{{/' \ + -e 's/}}/}}" }}/' \ + -e 's/Paths to search for config files in. (default .*)/Paths to search for config files in. (default [{{ .Workdir }}])/' return 0 } From 4d5998b4dc901757ccaa8554aa8e430b651791dd Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 25 May 2023 13:45:30 -0400 Subject: [PATCH 3/3] while loop + variable directory Signed-off-by: Andrew Mason --- tools/generate_flag_testdata.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/generate_flag_testdata.sh b/tools/generate_flag_testdata.sh index 0f70d65947e..f5e5293685b 100755 --- a/tools/generate_flag_testdata.sh +++ b/tools/generate_flag_testdata.sh @@ -1,6 +1,9 @@ #!/bin/bash set -euo pipefail + +DIR="go/flags/endtoend" + templatize_help_text() { $1 --help 2>&1 | \ sed -e 's/{{/{{ "{{/' \ @@ -11,8 +14,8 @@ templatize_help_text() { export -f templatize_help_text -find go/flags/endtoend -iname '*.txt' | \ - xargs basename | \ - cut -d. -f1 | \ - xargs -I{} \ - bash -c 'templatize_help_text "{}" >go/flags/endtoend/{}.txt' \ No newline at end of file +while read -r testfile; do + base="${testfile##*/}" + binary="${base%.*}" + templatize_help_text "${binary}" > "${testfile}" +done < <(find "${DIR}" -iname '*.txt')