From c1861d3bf79ee564dc6838adbe7c28d9d6f0bb89 Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Tue, 23 Nov 2021 14:05:04 -0500 Subject: [PATCH] bazel, ci: use test2json format for nightly logging Fixes #71930 Previously, bazel based nightlies didn't report errors as github issues. This PR uses a custom feature of `rules_go` to save the log output in `test2json` format. A new shell function is added to post-process the generated log and ports the `run_json_test` features to submit failures to GitHub. Release note: None --- WORKSPACE | 8 +- build/teamcity-bazel-support.sh | 80 +++++++++++++++++++ .../nightlies/sqllogic_hi_vmodule_nightly.sh | 3 +- .../sqllogic_hi_vmodule_nightly_impl.sh | 24 +++++- 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 51a4657429f5..08ce59e0ede4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,12 +13,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Load go bazel tools. This gives us access to the go bazel SDK/toolchains. http_archive( name = "io_bazel_rules_go", - sha256 = "f22ee17cb3fca740bca01cf4a6d95f5ccc3a411b77044357edff4f2524965261", - strip_prefix = "cockroachdb-rules_go-442d7c1", + sha256 = "adf650cf3dfded434e3a46c9063863579cdf8960a6e2f27e3e2537eb83a9413b", + strip_prefix = "rules_go-1ce9e349278b6e9ea1d52e0f73446c70ad440cbd", urls = [ - # cockroachdb/rules_go as of 442d7c1f3bfe86f75f1bff048f3f4016021a3371 + # cockroachdb/rules_go as of 1ce9e349278b6e9ea1d52e0f73446c70ad440cbd # (upstream release-0.29 plus a few patches). - "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-49-g442d7c1.tar.gz", + "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.29.0-0-1ce9e349.tar.gz", ], ) diff --git a/build/teamcity-bazel-support.sh b/build/teamcity-bazel-support.sh index c3c999beda57..dfdec20b6e0d 100644 --- a/build/teamcity-bazel-support.sh +++ b/build/teamcity-bazel-support.sh @@ -33,3 +33,83 @@ run_bazel() { ${vols} \ $BAZEL_IMAGE "$@" } + +# local copy of tc_release_branch from teamcity-support.sh to avoid imports. +_tc_release_branch() { + [[ "$TC_BUILD_BRANCH" == master || "$TC_BUILD_BRANCH" == release-* || "$TC_BUILD_BRANCH" == provisional_* ]] +} + +# process_test_json processes logs and submits failures to GitHub +# Requires GITHUB_API_TOKEN set for the release branches. +# Accepts 5 arguments: +# testfilter: path to the `testfilter` executable, usually +# `$BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter` +# github_post: path to the `github-post` executable, usually +# `$BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post` +# artifacts_dir: usually `/artifacts` +# test_json: path to test's JSON output, usually generated by `rules_go`'s and +# `GO_TEST_JSON_OUTPUT_FILE`. The file is removed after processing. +# create_tarball: whether to create a tarball with full logs. If the test's +# exit code is passed, the tarball is generated on failures. +process_test_json() { + local testfilter=$1 + local github_post=$2 + local artifacts_dir=$3 + local test_json=$4 + local create_tarball=$5 + + # move test.json.txt to the artifacts directory in order to simplify tarball creation + if [ ! -e $artifacts_dir/test.json.txt ]; then + mv -f $test_json $artifacts_dir/test.json.txt + test_json=$artifacts_dir/test.json.txt + fi + + $testfilter -mode=strip < "$test_json" | $testfilter -mode=omit | $testfilter -mode=convert > "$artifacts_dir"/failures.txt + failures_size=$(stat --format=%s "$artifacts_dir"/failures.txt) + if [ $failures_size = 0 ]; then + rm -f "$artifacts_dir"/failures.txt + fi + + if _tc_release_branch; then + if [ -z "${GITHUB_API_TOKEN-}" ]; then + # GITHUB_API_TOKEN must be in the env or github-post will barf if it's + # ever asked to post, so enforce that on all runs. + # The way this env var is made available here is quite tricky. The build + # calling this method is usually a build that is invoked from PRs, so it + # can't have secrets available to it (for the PR could modify + # build/teamcity-* to leak the secret). Instead, we provide the secrets + # to a higher-level job (Publish Bleeding Edge) and use TeamCity magic to + # pass that env var through when it's there. This means we won't have the + # env var on PR builds, but we'll have it for builds that are triggered + # from the release branches. + echo "GITHUB_API_TOKEN must be set" + exit 1 + else + tc_start_block "post issues" + $github_post < "$test_json" + tc_end_block "post issues" + fi + fi + + if [ "$create_tarball" -ne 0 ]; then + # Keep the debug file around for failed builds. Compress it to avoid + # clogging the agents with stuff we'll hopefully rarely ever need to + # look at. + # If the process failed, also save the full human-readable output. This is + # helpful in cases in which tests timed out, where it's difficult to blame + # the failure on any particular test. It's also a good alternative to poking + # around in test.json.txt itself when anything else we don't handle well happens, + # whatever that may be. + $testfilter -mode=convert < "$test_json" > "$artifacts_dir"/full_output.txt + (cd "$artifacts_dir" && tar --strip-components 1 -czf full_output.tgz full_output.txt $(basename $test_json)) + rm -rf "$artifacts_dir"/full_output.txt + fi + + rm -f "$test_json" + + # Some unit tests test automatic ballast creation. These ballasts can be + # larger than the maximum artifact size. Remove any artifacts with the + # EMERGENCY_BALLAST filename. + find "$artifacts_dir" -name "EMERGENCY_BALLAST" -delete +} + diff --git a/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly.sh b/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly.sh index 99da36c4d371..979db09bf1d1 100755 --- a/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly.sh +++ b/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly.sh @@ -8,5 +8,6 @@ source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel tc_start_block "Run SQL Logic Test High VModule" -run_bazel build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILD_BRANCH -e GITHUB_API_TOKEN -e BUILD_VCS_NUMBER -e TC_BUILD_ID -e TC_SERVER_URL" \ + run_bazel build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh tc_end_block "Run SQL Logic Test High VModule" diff --git a/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh b/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh index ff8f3a3aea42..d6f6240e99ab 100755 --- a/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh +++ b/build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh @@ -2,12 +2,32 @@ set -xeuo pipefail -bazel build //pkg/cmd/bazci --config=ci +dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" +source "$dir/teamcity-bazel-support.sh" # For process_test_json + +bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci BAZEL_BIN=$(bazel info bazel-bin --config=ci) +ARTIFACTS_DIR=/artifacts +GO_TEST_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test.json.txt + +exit_status=0 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \ test //pkg/sql/logictest:logictest_test -- \ --test_arg=--vmodule=*=10 \ --test_arg=-show-sql \ --test_filter='^TestLogic$' \ - --test_timeout=7200 + --test_env=GO_TEST_WRAP_TESTV=1 \ + --test_env=GO_TEST_WRAP=1 \ + --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE \ + --test_timeout=7200 \ + || exit_status=$? + +process_test_json \ + $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \ + $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \ + $ARTIFACTS_DIR \ + $GO_TEST_JSON_OUTPUT_FILE \ + $exit_status + +exit $exit_status