bazel, ci: use test2json format for nightly logging

Fixes #71930

Previously, bazel based nightlies didn't report errors as github issues.

This PR uses a custom feature of `rules_go` to save the log output in
`test2json` format. A new shell function is added to post-process the
generated log and ports the `run_json_test` features to submit failures
to GitHub.

Release note: None

WORKSPACE

@@ -13,12 +13,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 # Load go bazel tools. This gives us access to the go bazel SDK/toolchains.
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "f22ee17cb3fca740bca01cf4a6d95f5ccc3a411b77044357edff4f2524965261",
-    strip_prefix = "cockroachdb-rules_go-442d7c1",
+    sha256 = "adf650cf3dfded434e3a46c9063863579cdf8960a6e2f27e3e2537eb83a9413b",
+    strip_prefix = "rules_go-1ce9e349278b6e9ea1d52e0f73446c70ad440cbd",
     urls = [
-        # cockroachdb/rules_go as of 442d7c1f3bfe86f75f1bff048f3f4016021a3371
+        # cockroachdb/rules_go as of 1ce9e349278b6e9ea1d52e0f73446c70ad440cbd
         # (upstream release-0.29 plus a few patches).
-        "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-49-g442d7c1.tar.gz",
+        "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.29.0-0-1ce9e349.tar.gz",
     ],
 )
 

build/teamcity-bazel-support.sh

@@ -33,3 +33,83 @@ run_bazel() {
         ${vols} \
         $BAZEL_IMAGE "$@"
 }
+
+# local copy of tc_release_branch from teamcity-support.sh to avoid imports.
+_tc_release_branch() {
+  [[ "$TC_BUILD_BRANCH" == master || "$TC_BUILD_BRANCH" == release-* || "$TC_BUILD_BRANCH" == provisional_* ]]
+}
+
+# process_test_json processes logs and submits failures to GitHub
+# Requires GITHUB_API_TOKEN set for the release branches.
+# Accepts 5 arguments:
+# testfilter: path to the `testfilter` executable, usually
+#   `$BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter`
+# github_post: path to the `github-post` executable, usually
+#   `$BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post`
+# artifacts_dir: usually `/artifacts`
+# test_json: path to test's JSON output, usually generated by `rules_go`'s and
+#   `GO_TEST_JSON_OUTPUT_FILE`. The file is removed after processing.
+# create_tarball: whether to create a tarball with full logs. If the test's
+#   exit code is passed, the tarball is generated on failures.
+process_test_json() {
+  local testfilter=$1
+  local github_post=$2
+  local artifacts_dir=$3
+  local test_json=$4
+  local create_tarball=$5
+
+  # move test.json.txt to the artifacts directory in order to simplify tarball creation
+  if [ ! -e $artifacts_dir/test.json.txt ]; then
+    mv -f $test_json $artifacts_dir/test.json.txt
+    test_json=$artifacts_dir/test.json.txt
+  fi
+
+  $testfilter -mode=strip < "$test_json" | $testfilter -mode=omit | $testfilter -mode=convert > "$artifacts_dir"/failures.txt
+  failures_size=$(stat --format=%s "$artifacts_dir"/failures.txt)
+  if [ $failures_size = 0 ]; then
+    rm -f "$artifacts_dir"/failures.txt
+  fi
+
+  if _tc_release_branch; then
+    if [ -z "${GITHUB_API_TOKEN-}" ]; then
+      # GITHUB_API_TOKEN must be in the env or github-post will barf if it's
+      # ever asked to post, so enforce that on all runs.
+      # The way this env var is made available here is quite tricky. The build
+      # calling this method is usually a build that is invoked from PRs, so it
+      # can't have secrets available to it (for the PR could modify
+      # build/teamcity-* to leak the secret). Instead, we provide the secrets
+      # to a higher-level job (Publish Bleeding Edge) and use TeamCity magic to
+      # pass that env var through when it's there. This means we won't have the
+      # env var on PR builds, but we'll have it for builds that are triggered
+      # from the release branches.
+      echo "GITHUB_API_TOKEN must be set"
+      exit 1
+    else
+      tc_start_block "post issues"
+      $github_post < "$test_json"
+      tc_end_block "post issues"
+    fi
+  fi
+
+  if [ "$create_tarball" -ne 0 ]; then
+    # Keep the debug file around for failed builds. Compress it to avoid
+    # clogging the agents with stuff we'll hopefully rarely ever need to
+    # look at.
+    # If the process failed, also save the full human-readable output. This is
+    # helpful in cases in which tests timed out, where it's difficult to blame
+    # the failure on any particular test. It's also a good alternative to poking
+    # around in test.json.txt itself when anything else we don't handle well happens,
+    # whatever that may be.
+    $testfilter -mode=convert < "$test_json" > "$artifacts_dir"/full_output.txt
+    (cd "$artifacts_dir" && tar --strip-components 1 -czf full_output.tgz full_output.txt $(basename $test_json))
+    rm -rf "$artifacts_dir"/full_output.txt
+  fi
+
+  rm -f "$test_json"
+
+  # Some unit tests test automatic ballast creation. These ballasts can be
+  # larger than the maximum artifact size. Remove any artifacts with the
+  # EMERGENCY_BALLAST filename.
+  find "$artifacts_dir" -name "EMERGENCY_BALLAST" -delete
+}
+

build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly.sh

@@ -8,5 +8,6 @@ source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
 tc_start_block "Run SQL Logic Test High VModule"
-run_bazel build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILD_BRANCH -e GITHUB_API_TOKEN -e BUILD_VCS_NUMBER -e TC_BUILD_ID -e TC_SERVER_URL" \
+  run_bazel build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh
 tc_end_block "Run SQL Logic Test High VModule"

build/teamcity/cockroach/nightlies/sqllogic_hi_vmodule_nightly_impl.sh

@@ -2,12 +2,32 @@
 
 set -xeuo pipefail
 
-bazel build //pkg/cmd/bazci --config=ci
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"  # For process_test_json
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
 BAZEL_BIN=$(bazel info bazel-bin --config=ci)
 
+ARTIFACTS_DIR=/artifacts
+GO_TEST_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test.json.txt
+
+exit_status=0
 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
     test //pkg/sql/logictest:logictest_test -- \
     --test_arg=--vmodule=*=10 \
     --test_arg=-show-sql \
     --test_filter='^TestLogic$' \
-    --test_timeout=7200
+    --test_env=GO_TEST_WRAP_TESTV=1 \
+    --test_env=GO_TEST_WRAP=1 \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE \
+    --test_timeout=7200 \
+    || exit_status=$?
+
+process_test_json \
+  $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+  $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+  $ARTIFACTS_DIR \
+  $GO_TEST_JSON_OUTPUT_FILE \
+  $exit_status
+
+exit $exit_status