bazel: make go test process timeout before bazel kills it

Previously, bazel will kill the test process if it goes
beyond the timeout duration set for its size. This prevented
us from knowing which tests timed out and also prevented us
from getting their stack traces.

This patch causes the `go test` process to timeout before
bazel kills it to allow us to know which test timed out and
get its stack trace.

Closes #78185

Release justification: Non-production code changes
Release note: None

.github/CODEOWNERS

@@ -245,7 +245,7 @@
 /pkg/cmd/generate-logictest/       @cockroachdb/dev-inf
 /pkg/cmd/generate-metadata-tables/ @cockroachdb/sql-experience
 /pkg/cmd/generate-spatial-ref-sys/ @cockroachdb/geospatial
-/pkg/cmd/generate-test-suites/ @cockroachdb/dev-inf
+/pkg/cmd/generate-bazel-extra/ @cockroachdb/dev-inf
 /pkg/cmd/generate-staticcheck/ @cockroachdb/dev-inf
 /pkg/cmd/geoviz/             @cockroachdb/geospatial
 /pkg/cmd/github-post/        @cockroachdb/test-eng

build/bazelutil/bazel-generate.sh

@@ -76,10 +76,13 @@ else
   bazel run pkg/gen/genbzl --run_under="cd $PWD && " -- --out-dir pkg/gen
 fi
 
-if files_unchanged_from_upstream $(find_relevant ./pkg/cmd/generate-test-suites -name BUILD.bazel -or -name '*.go') $(find_relevant ./pkg -name BUILD.bazel) $(find_relevant ./pkg -name '*.bzl'); then
-  echo "Skipping //pkg/cmd/generate-test-suites (relevant files are unchanged from upstream)."
+if ! (files_unchanged_from_upstream $(find_relevant ./pkg -name BUILD.bazel) $(find_relevant ./pkg/cmd/generate-bazel-extra -name BUILD.bazel -or -name '*.go')); then
+  bazel build @com_github_bazelbuild_buildtools//buildozer:buildozer
+  bazel run //pkg/cmd/generate-bazel-extra --run_under="cd $PWD && " -- -gen_test_suites -gen_tests_timeouts
+elif files_unchanged_from_upstream $(find_relevant ./pkg -name '*.bzl'); then
+  echo "Skipping //pkg/cmd/generate-bazel-extra (relevant files are unchanged from upstream)."
 else
+  echo "Skipping `generate tests timeouts` from //pkg/cmd/generate-bazel-extra (relevant files are unchanged from upstream)."
   bazel build @com_github_bazelbuild_buildtools//buildozer:buildozer
-  CONTENTS=$(bazel run //pkg/cmd/generate-test-suites --run_under="cd $PWD && ")
-  echo "$CONTENTS" > pkg/BUILD.bazel
+  bazel run //pkg/cmd/generate-bazel-extra --run_under="cd $PWD && " -- -gen_test_suites
 fi

build/bazelutil/check.sh

@@ -95,7 +95,7 @@ $GIT_GREP '//go:generate' 'pkg/**/*.go' | grep -v stringer | grep -v 'add-leakte
     exit 1
 done
 
-$GIT_GREP 'broken_in_bazel' pkg | grep BUILD.bazel: | grep -v pkg/BUILD.bazel | grep -v pkg/cli/BUILD.bazel | grep -v generate-test-suites | cut -d: -f1 | while read LINE; do
+$GIT_GREP 'broken_in_bazel' pkg | grep BUILD.bazel: | grep -v pkg/BUILD.bazel | grep -v pkg/cli/BUILD.bazel | grep -v generate-bazel-extra | cut -d: -f1 | while read LINE; do
     if [[ "$EXISTING_BROKEN_TESTS_IN_BAZEL" == *"$LINE"* ]]; then
 	# Grandfathered.
 	continue

pkg/BUILD.bazel

@@ -1,4 +1,4 @@
-# Code generated by generate-test-suites, DO NOT EDIT.
+# Code generated by generate-bazel-extra, DO NOT EDIT.
 # gazelle:proto_strip_import_prefix /pkg
 
 load("//build/bazelutil/unused_checker:unused.bzl", "unused_checker")
@@ -868,6 +868,8 @@ GO_TARGETS = [
     "//pkg/cmd/docs-issue-generation:docs-issue-generation_test",
     "//pkg/cmd/fuzz:fuzz",
     "//pkg/cmd/fuzz:fuzz_lib",
+    "//pkg/cmd/generate-bazel-extra:generate-bazel-extra",
+    "//pkg/cmd/generate-bazel-extra:generate-bazel-extra_lib",
     "//pkg/cmd/generate-binary:generate-binary",
     "//pkg/cmd/generate-binary:generate-binary_lib",
     "//pkg/cmd/generate-distdir:generate-distdir",
@@ -881,8 +883,6 @@ GO_TARGETS = [
     "//pkg/cmd/generate-spatial-ref-sys:generate-spatial-ref-sys_lib",
     "//pkg/cmd/generate-staticcheck:generate-staticcheck",
     "//pkg/cmd/generate-staticcheck:generate-staticcheck_lib",
-    "//pkg/cmd/generate-test-suites:generate-test-suites",
-    "//pkg/cmd/generate-test-suites:generate-test-suites_lib",
     "//pkg/cmd/geoviz:geoviz",
     "//pkg/cmd/geoviz:geoviz_lib",
     "//pkg/cmd/github-post:github-post",
@@ -2253,14 +2253,14 @@ GET_X_DATA_TARGETS = [
     "//pkg/cmd/docgen/extract:get_x_data",
     "//pkg/cmd/docs-issue-generation:get_x_data",
     "//pkg/cmd/fuzz:get_x_data",
+    "//pkg/cmd/generate-bazel-extra:get_x_data",
     "//pkg/cmd/generate-binary:get_x_data",
     "//pkg/cmd/generate-distdir:get_x_data",
     "//pkg/cmd/generate-logictest:get_x_data",
     "//pkg/cmd/generate-metadata-tables:get_x_data",
     "//pkg/cmd/generate-metadata-tables/rdbms:get_x_data",
     "//pkg/cmd/generate-spatial-ref-sys:get_x_data",
     "//pkg/cmd/generate-staticcheck:get_x_data",
-    "//pkg/cmd/generate-test-suites:get_x_data",
     "//pkg/cmd/geoviz:get_x_data",
     "//pkg/cmd/github-post:get_x_data",
     "//pkg/cmd/github-pull-request-make:get_x_data",

pkg/base/BUILD.bazel

@@ -56,6 +56,7 @@ go_test(
         "node_id_test.go",
         "store_spec_test.go",
     ],
+    args = ["-test.timeout=55s"],
     deps = [
         ":base",
         "//pkg/roachpb",

pkg/bench/BUILD.bazel

@@ -29,6 +29,7 @@ go_test(
         "main_test.go",
         "pgbench_test.go",
     ],
+    args = ["-test.timeout=55s"],
     embed = [":bench"],
     deps = [
         "//pkg/base",

pkg/bench/rttanalysis/BUILD.bazel

@@ -48,6 +48,7 @@ go_test(
         "validate_benchmark_data_test.go",
         "virtual_table_bench_test.go",
     ],
+    args = ["-test.timeout=895s"],
     data = glob(["testdata/**"]),
     embed = [":rttanalysis"],
     shard_count = 16,

pkg/bench/tpcc/BUILD.bazel

@@ -18,6 +18,7 @@ go_test(
         "tpcc_bench_options_test.go",
         "tpcc_bench_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":tpcc"],
     deps = [
         "//pkg/base",

pkg/blobs/BUILD.bazel

@@ -37,6 +37,7 @@ go_test(
         "local_storage_test.go",
         "service_test.go",
     ],
+    args = ["-test.timeout=55s"],
     embed = [":blobs"],
     deps = [
         "//pkg/base",

pkg/build/starlarkutil/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
 go_test(
     name = "starlarkutil_test",
     srcs = ["starlarkutil_test.go"],
+    args = ["-test.timeout=295s"],
     embed = [":starlarkutil"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/build/util/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
 go_test(
     name = "util_test",
     srcs = ["util_test.go"],
+    args = ["-test.timeout=295s"],
     embed = [":util"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/ccl/backupccl/BUILD.bazel

@@ -177,6 +177,7 @@ go_test(
         "system_schema_test.go",
         "utils_test.go",
     ],
+    args = ["-test.timeout=3595s"],
     data = glob(["testdata/**"]) + ["//c-deps:libgeos"],
     embed = [":backupccl"],
     shard_count = 16,

pkg/ccl/backupccl/backupdest/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
         "incrementals_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":backupdest"],
     deps = [
         "//pkg/ccl/backupccl/backupbase",

pkg/ccl/backupccl/backupinfo/BUILD.bazel

@@ -52,6 +52,7 @@ go_library(
 go_test(
     name = "backupinfo_test",
     srcs = ["main_test.go"],
+    args = ["-test.timeout=295s"],
     embed = [":backupinfo"],
     deps = [
         "//pkg/ccl/utilccl",

pkg/ccl/backupccl/backuprand/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "backup_rand_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     data = ["//c-deps:libgeos"],
     deps = [
         "//pkg/base",

pkg/ccl/backupccl/backupresolver/BUILD.bazel

@@ -29,6 +29,7 @@ go_test(
         "main_test.go",
         "targets_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":backupresolver"],
     deps = [
         "//pkg/ccl/storageccl",

pkg/ccl/baseccl/BUILD.bazel

@@ -22,6 +22,7 @@ go_test(
     name = "baseccl_test",
     size = "small",
     srcs = ["encryption_spec_test.go"],
+    args = ["-test.timeout=55s"],
     embed = [":baseccl"],
     deps = ["//pkg/util/leaktest"],
 )

pkg/ccl/benchccl/rttanalysisccl/BUILD.bazel

@@ -8,6 +8,7 @@ go_test(
         "bench_test.go",
         "multi_region_bench_test.go",
     ],
+    args = ["-test.timeout=3595s"],
     data = glob(["testdata/**"]),
     shard_count = 16,
     deps = [

pkg/ccl/changefeedccl/BUILD.bazel

@@ -160,6 +160,7 @@ go_test(
         "testfeed_test.go",
         "validations_test.go",
     ],
+    args = ["-test.timeout=3595s"],
     embed = [":changefeedccl"],
     shard_count = 16,
     deps = [

pkg/ccl/changefeedccl/cdceval/BUILD.bazel

@@ -53,6 +53,7 @@ go_test(
         "main_test.go",
         "validation_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":cdceval"],
     deps = [
         "//pkg/base",

pkg/ccl/changefeedccl/cdcevent/BUILD.bazel

@@ -42,6 +42,7 @@ go_test(
         "main_test.go",
         "projection_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":cdcevent"],
     deps = [
         "//pkg/base",

pkg/ccl/changefeedccl/cdctest/BUILD.bazel

@@ -47,6 +47,7 @@ go_test(
         "main_test.go",
         "validator_test.go",
     ],
+    args = ["-test.timeout=55s"],
     embed = [":cdctest"],
     deps = [
         "//pkg/base",

pkg/ccl/changefeedccl/cdcutils/BUILD.bazel

@@ -20,6 +20,7 @@ go_library(
 go_test(
     name = "cdcutils_test",
     srcs = ["throttle_test.go"],
+    args = ["-test.timeout=295s"],
     embed = [":cdcutils"],
     deps = [
         "//pkg/ccl/changefeedccl/changefeedbase",

pkg/ccl/changefeedccl/changefeedbase/BUILD.bazel

@@ -27,6 +27,7 @@ go_library(
 go_test(
     name = "changefeedbase_test",
     srcs = ["options_test.go"],
+    args = ["-test.timeout=295s"],
     embed = [":changefeedbase"],
     deps = [
         "//pkg/util/leaktest",

pkg/ccl/changefeedccl/kvevent/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
         "blocking_buffer_test.go",
         "chunked_event_queue_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":kvevent"],
     deps = [
         "//pkg/jobs/jobspb",

pkg/ccl/changefeedccl/kvfeed/BUILD.bazel

@@ -44,6 +44,7 @@ go_test(
         "main_test.go",
         "scanner_test.go",
     ],
+    args = ["-test.timeout=55s"],
     embed = [":kvfeed"],
     deps = [
         "//pkg/base",

pkg/ccl/changefeedccl/schemafeed/BUILD.bazel

@@ -52,6 +52,7 @@ go_test(
         "table_event_filter_datadriven_test.go",
         "table_event_filter_test.go",
     ],
+    args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
     embed = [":schemafeed"],
     deps = [

pkg/ccl/cliccl/BUILD.bazel

@@ -72,6 +72,7 @@ go_test(
         "debug_backup_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":cliccl"],
     deps = [
         "//pkg/base",

pkg/ccl/cloudccl/amazon/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "main_test.go",
         "s3_connection_test.go",
     ],
+    args = ["-test.timeout=295s"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/ccl/cloudccl/azure/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "azure_connection_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/ccl/cloudccl/cloudprivilege/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "main_test.go",
         "privileges_test.go",
     ],
+    args = ["-test.timeout=295s"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/ccl/cloudccl/externalconn/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "datadriven_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
     deps = [
         "//pkg/base",

pkg/ccl/cloudccl/gcp/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "gcp_connection_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/ccl/importerccl/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "ccl_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     data = [
         "//c-deps:libgeos",
         "//pkg/sql/importer:testdata",

pkg/ccl/jobsccl/jobsprotectedtsccl/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
         "jobs_protected_ts_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     deps = [
         "//pkg/base",
         "//pkg/ccl/kvccl/kvtenantccl",

pkg/ccl/kvccl/kvfollowerreadsccl/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
         "followerreads_test.go",
         "main_test.go",
     ],
+    args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
     embed = [":kvfollowerreadsccl"],
     deps = [

pkg/ccl/kvccl/kvtenantccl/BUILD.bazel

@@ -49,6 +49,7 @@ go_test(
         "tenant_trace_test.go",
         "tenant_upgrade_test.go",
     ],
+    args = ["-test.timeout=295s"],
     embed = [":kvtenantccl"],
     shard_count = 16,
     deps = [

pkg/ccl/logictestccl/tests/3node-tenant-multiregion/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "3node-tenant-multiregion_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/sql/logictest:testdata",  # keep

pkg/ccl/logictestccl/tests/3node-tenant/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "3node-tenant_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep

pkg/ccl/logictestccl/tests/5node/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "5node_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep

pkg/ccl/logictestccl/tests/fakedist-disk/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "fakedist-disk_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep

pkg/ccl/logictestccl/tests/fakedist-vec-off/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "fakedist-vec-off_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep

pkg/ccl/logictestccl/tests/fakedist/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "fakedist_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep

pkg/ccl/logictestccl/tests/local-vec-off/BUILD.bazel

@@ -5,6 +5,7 @@ go_test(
     name = "local-vec-off_test",
     size = "enormous",
     srcs = ["generated_test.go"],
+    args = ["-test.timeout=3595s"],
     data = [
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep