*: customize the timeouts used by unit tests in Bazel Essential CI

We manage unit tests timeouts at two levels:
1. Bazel timeout, by default [60s,300s,900s,3600s] for [small,medium,large,enormous] targets.
2. Go timeout, set to 5 seconds less than the corresponding Bazel timeout [see cockroachdb#86363].

Previously, unit tests used the same timeouts both when running in `Bazel Essential
CI` and elsewhere. As a result, enormous test targets inherited a timeout of 1 hour
from Bazel's default timeout. This is way beyond the expected time needed by any
test target in `Bazel Essential CI`. We can't change enormous targets to large ones for
two reasons:
1. `Enormous` is also used to indicate the resources needed by a test target.
2. `Enormous` test targets may still need the large timeout when running locally.

To make this possible, we needed to support setting an `attr` value to a `select`
using Buildozer. This was done in bazelbuild/buildtools#1153.

This change only affects the timeout of `enormous` test targets. It however makes it
simple to customize the timeout of other test sizes if desired in the future.

Release note: None
Epic: none

.bazelrc

@@ -51,6 +51,8 @@ test:test --test_env=TZ=
 # If those values are updated, the script should be updated accordingly.
 test:race --test_timeout=1200,6000,18000,72000
 
+# CI uses a custom timeout for enormous targets.
+test:use_ci_timeouts --test_timeout=60,300,900,900 --define=use_ci_timeouts=true
 # CI should always run with `--config=ci` or `--config=cinolint`.
 # Prefer the first to the second unless some other job will handle linting the
 # same code you're building.

WORKSPACE

@@ -355,10 +355,10 @@ load(
 go_repository(
     name = "com_github_bazelbuild_buildtools",
     importpath = "github.com/bazelbuild/buildtools",
-    sha256 = "a9ef5103739dfb5ed2a5b47ab1654842a89695812e4af09e57d7015a5caf97e0",
-    strip_prefix = "buildtools",
+    sha256 = "d71a889e3bc50cc8b9d42c859e15a74f7c8d10b6786f8dd82f08f2bf24e5bdc6",
+    strip_prefix = "bazelbuild-buildtools-b182fc4",
     urls = [
-        "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v0.0.0-20200718160251-b1667ff58f71/buildtools-v0.0.0-20200718160251-b1667ff58f71.tar.gz",
+        "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v6.1.2-0-gb182fc4/bazelbuild-buildtools-v6.1.2-0-gb182fc4.tar.gz",
     ],
 )
 

build/bazelutil/distdir_files.bzl

@@ -1171,7 +1171,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/public-bazel-artifacts/go/20230214-214430/go1.19.4.linux-arm64.tar.gz": "6bb5752483c0d145b91199e5cc1352960d926850e75864dea16282337b0d92fe",
     "https://storage.googleapis.com/public-bazel-artifacts/go/20230214-214430/go1.19.4.windows-amd64.tar.gz": "0f37edf2a6663db33c8f67ee36e21a7eb391fbf35d494299f6a81a59e294f4a0",
     "https://storage.googleapis.com/public-bazel-artifacts/go/20230427-165819/go1.19.8fips.linux-amd64.tar.gz": "8170fd871cb61dc771ec1f309451b31a73d5aca3410dfa9d952672ae2be4ac9e",
-    "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v0.0.0-20200718160251-b1667ff58f71/buildtools-v0.0.0-20200718160251-b1667ff58f71.tar.gz": "a9ef5103739dfb5ed2a5b47ab1654842a89695812e4af09e57d7015a5caf97e0",
+    "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v6.1.2-0-gb182fc4/bazelbuild-buildtools-v6.1.2-0-gb182fc4.tar.gz": "d71a889e3bc50cc8b9d42c859e15a74f7c8d10b6786f8dd82f08f2bf24e5bdc6",
     "https://storage.googleapis.com/public-bazel-artifacts/java/railroad/rr-1.63-java8.zip": "d2791cd7a44ea5be862f33f5a9b3d40aaad9858455828ebade7007ad7113fb41",
     "https://storage.googleapis.com/public-bazel-artifacts/js/node/v16.13.0/node-v16.13.0-darwin-arm64.tar.gz": "46d83fc0bd971db5050ef1b15afc44a6665dee40bd6c1cbaec23e1b40fa49e6d",
     "https://storage.googleapis.com/public-bazel-artifacts/js/node/v16.13.0/node-v16.13.0-darwin-x64.tar.gz": "37e09a8cf2352f340d1204c6154058d81362fef4ec488b0197b2ce36b3f0367a",

build/teamcity/cockroach/ci/tests/unit_tests_ccl_impl.sh

@@ -15,6 +15,6 @@ if tc_release_branch; then
   EXTRA_PARAMS=" --flaky_test_attempts=3" 
 fi
 
-$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint -c fastbuild \
+$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint --config=use_ci_timeouts -c fastbuild \
                                   //pkg:ccl_tests \
                                    --profile=/artifacts/profile.gz $EXTRA_PARAMS

build/teamcity/cockroach/ci/tests/unit_tests_impl.sh

@@ -15,6 +15,6 @@ if tc_release_branch; then
   EXTRA_PARAMS=" --flaky_test_attempts=3" 
 fi
 
-$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint -c fastbuild \
+$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint --config=use_ci_timeouts -c fastbuild \
                                   //pkg:small_non_ccl_tests //pkg:medium_non_ccl_tests //pkg:large_non_ccl_tests //pkg:enormous_non_ccl_tests \
                                    --profile=/artifacts/profile.gz $EXTRA_PARAMS

c-deps/BUILD.bazel

@@ -245,14 +245,14 @@ configure_make(
     }),
     lib_source = "@krb5//:all",
     out_static_libs = LIBKRB5_LIBS,
-    postfix_script = ("""mkdir -p libkrb5/lib
+    postfix_script = """mkdir -p libkrb5/lib
 cp lib/libcom_err.a libkrb5/lib
 cp lib/libgssapi_krb5.a libkrb5/lib
 cp lib/libkrb5.a libkrb5/lib
 cp lib/libkrb5support.a libkrb5/lib
 cp lib/libk5crypto.a libkrb5/lib
 mkdir -p libkrb5/include/gssapi
-cp include/gssapi/gssapi.h libkrb5/include/gssapi"""),
+cp include/gssapi/gssapi.h libkrb5/include/gssapi""",
     visibility = ["//visibility:public"],
 )
 

pkg/BUILD.bazel

@@ -3521,3 +3521,10 @@ test_suite(
 )
 
 unused_checker(srcs = GET_X_DATA_TARGETS)
+
+config_setting(
+    name = "use_ci_timeouts",
+    values = {
+        "define": "use_ci_timeouts=true"
+    }
+)

pkg/base/BUILD.bazel

@@ -58,7 +58,10 @@ go_test(
         "node_id_test.go",
         "store_spec_test.go",
     ],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     data = glob(["testdata/**"]),
     deps = [
         ":base",

pkg/bench/BUILD.bazel

@@ -36,7 +36,10 @@ go_test(
         "main_test.go",
         "pgbench_test.go",
     ],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     embed = [":bench"],
     deps = [
         "//pkg/base",

pkg/bench/rttanalysis/BUILD.bazel

@@ -51,7 +51,10 @@ go_test(
         "validate_benchmark_data_test.go",
         "virtual_table_bench_test.go",
     ],
-    args = ["-test.timeout=895s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=895s"],
+        "//conditions:default": ["-test.timeout=895s"],
+    }),
     data = glob(["testdata/**"]),
     embed = [":rttanalysis"],
     shard_count = 16,

pkg/bench/tpcc/BUILD.bazel

@@ -18,7 +18,10 @@ go_test(
         "tpcc_bench_options_test.go",
         "tpcc_bench_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":tpcc"],
     deps = [
         "//pkg/base",

pkg/blobs/BUILD.bazel

@@ -38,7 +38,10 @@ go_test(
         "local_storage_test.go",
         "service_test.go",
     ],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     embed = [":blobs"],
     deps = [
         "//pkg/base",

pkg/build/BUILD.bazel

@@ -49,7 +49,10 @@ go_proto_library(
 go_test(
     name = "build_test",
     srcs = ["info_test.go"],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":build"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/build/starlarkutil/BUILD.bazel

@@ -12,7 +12,10 @@ go_library(
 go_test(
     name = "starlarkutil_test",
     srcs = ["starlarkutil_test.go"],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":starlarkutil"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/build/util/BUILD.bazel

@@ -11,7 +11,10 @@ go_library(
 go_test(
     name = "util_test",
     srcs = ["util_test.go"],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":util"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/ccl/backupccl/BUILD.bazel

@@ -193,7 +193,10 @@ go_test(
         "tenant_backup_nemesis_test.go",
         "utils_test.go",
     ],
-    args = ["-test.timeout=3595s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=895s"],
+        "//conditions:default": ["-test.timeout=3595s"],
+    }),
     data = glob(["testdata/**"]) + ["//c-deps:libgeos"],
     embed = [":backupccl"],
     shard_count = 48,

pkg/ccl/backupccl/backupdest/BUILD.bazel

@@ -41,7 +41,10 @@ go_test(
         "incrementals_test.go",
         "main_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     tags = ["ccl_test"],
     deps = [
         ":backupdest",

pkg/ccl/backupccl/backupinfo/BUILD.bazel

@@ -64,7 +64,10 @@ go_test(
         "main_test.go",
         "manifest_handling_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     tags = ["ccl_test"],
     deps = [
         ":backupinfo",

pkg/ccl/backupccl/backuprand/BUILD.bazel

@@ -7,7 +7,10 @@ go_test(
         "backup_rand_test.go",
         "main_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     data = ["//c-deps:libgeos"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/backupccl/backupresolver/BUILD.bazel

@@ -30,7 +30,10 @@ go_test(
         "main_test.go",
         "targets_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":backupresolver"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/baseccl/BUILD.bazel

@@ -22,7 +22,10 @@ go_test(
     name = "baseccl_test",
     size = "small",
     srcs = ["encryption_spec_test.go"],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     embed = [":baseccl"],
     tags = ["ccl_test"],
     deps = ["//pkg/util/leaktest"],

pkg/ccl/benchccl/rttanalysisccl/BUILD.bazel

@@ -8,7 +8,10 @@ go_test(
         "bench_test.go",
         "multi_region_bench_test.go",
     ],
-    args = ["-test.timeout=3595s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=895s"],
+        "//conditions:default": ["-test.timeout=3595s"],
+    }),
     data = glob(["testdata/**"]),
     shard_count = 16,
     tags = ["ccl_test"],

pkg/ccl/changefeedccl/BUILD.bazel

@@ -203,7 +203,10 @@ go_test(
         "testfeed_test.go",
         "validations_test.go",
     ],
-    args = ["-test.timeout=3595s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=895s"],
+        "//conditions:default": ["-test.timeout=3595s"],
+    }),
     embed = [":changefeedccl"],
     shard_count = 16,
     tags = ["ccl_test"],

pkg/ccl/changefeedccl/cdceval/BUILD.bazel

@@ -62,7 +62,10 @@ go_test(
         "plan_test.go",
         "validation_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":cdceval"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/cdcevent/BUILD.bazel

@@ -50,7 +50,10 @@ go_test(
         "projection_test.go",
         "rowfetcher_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":cdcevent"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/cdctest/BUILD.bazel

@@ -48,7 +48,10 @@ go_test(
         "main_test.go",
         "validator_test.go",
     ],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     embed = [":cdctest"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/cdcutils/BUILD.bazel

@@ -20,7 +20,10 @@ go_library(
 go_test(
     name = "cdcutils_test",
     srcs = ["throttle_test.go"],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":cdcutils"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/changefeedbase/BUILD.bazel

@@ -29,7 +29,10 @@ go_library(
 go_test(
     name = "changefeedbase_test",
     srcs = ["options_test.go"],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":changefeedbase"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/kvevent/BUILD.bazel

@@ -41,7 +41,10 @@ go_test(
         "blocking_buffer_test.go",
         "chunked_event_queue_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":kvevent"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/kvfeed/BUILD.bazel

@@ -48,7 +48,10 @@ go_test(
         "main_test.go",
         "scanner_test.go",
     ],
-    args = ["-test.timeout=55s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=55s"],
+        "//conditions:default": ["-test.timeout=55s"],
+    }),
     embed = [":kvfeed"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/schemafeed/BUILD.bazel

@@ -52,7 +52,10 @@ go_test(
         "table_event_filter_datadriven_test.go",
         "table_event_filter_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     data = glob(["testdata/**"]),
     embed = [":schemafeed"],
     tags = ["ccl_test"],

pkg/ccl/cliccl/BUILD.bazel

@@ -53,7 +53,10 @@ go_test(
         "ear_test.go",
         "main_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     embed = [":cliccl"],
     tags = ["ccl_test"],
     deps = [

pkg/ccl/cloudccl/amazon/BUILD.bazel

@@ -7,7 +7,10 @@ go_test(
         "main_test.go",
         "s3_connection_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     tags = ["ccl_test"],
     deps = [
         "//pkg/base",

pkg/ccl/cloudccl/azure/BUILD.bazel

@@ -7,7 +7,10 @@ go_test(
         "azure_connection_test.go",
         "main_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = select({
+        "//pkg:use_ci_timeouts": ["-test.timeout=295s"],
+        "//conditions:default": ["-test.timeout=295s"],
+    }),
     tags = ["ccl_test"],
     deps = [
         "//pkg/base",