ci: attempt to shard huge test targets more

This code change attempts to shard some test targets that could benefit from more sharding. It also splits unit tests in TeamCity into two runs: ccl unit tests, and non-ccl unit tests. The current build config will be used to run non-ccl unit tests. The new build config will be used for ccl tests (those under `pkg/ccl`). This cuts unit tests wall time by half while keeping machine time almost the same. Release note: None Epic: none
cockroachdb · Mar 22, 2023 · 69921eb · 69921eb
1 parent 78bb769
commit 69921eb
Show file tree

Hide file tree

Showing 106 changed files with 228 additions and 47 deletions.
diff --git a/build/teamcity/cockroach/ci/tests/unit_tests_ccl.sh b/build/teamcity/cockroach/ci/tests/unit_tests_ccl.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+tc_start_block "Run unit tests"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILD_BRANCH -e GITHUB_API_TOKEN -e BUILD_VCS_NUMBER -e TC_BUILD_ID -e TC_SERVER_URL -e TC_BUILDTYPE_ID -e GITHUB_REPO" run_bazel build/teamcity/cockroach/ci/tests/unit_tests_ccl_impl.sh
+tc_end_block "Run unit tests"
diff --git a/build/teamcity/cockroach/ci/tests/unit_tests_ccl_impl.sh b/build/teamcity/cockroach/ci/tests/unit_tests_ccl_impl.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+
+source "$dir/teamcity-support.sh"  # for 'tc_release_branch'
+
+bazel build //pkg/cmd/bazci --config=ci
+
+EXTRA_PARAMS=""
+
+if tc_release_branch; then
+  # enable up to 2 retries (3 attempts, worst-case) per test executable to report flakes but only on release branches (i.e., not staging)
+  EXTRA_PARAMS=" --flaky_test_attempts=3" 
+fi
+
+$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint -c fastbuild \
+                                  //pkg:ccl_tests \
+                                   --profile=/artifacts/profile.gz $EXTRA_PARAMS
diff --git a/build/teamcity/cockroach/ci/tests/unit_tests_impl.sh b/build/teamcity/cockroach/ci/tests/unit_tests_impl.sh
@@ -16,5 +16,5 @@ if tc_release_branch; then
 fi
 
 $(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci -- test --config=cinolint -c fastbuild \
-                                  //pkg:small_tests //pkg:medium_tests //pkg:large_tests //pkg:enormous_tests \
+                                  //pkg:small_non_ccl_tests //pkg:medium_non_ccl_tests //pkg:large_non_ccl_tests //pkg:enormous_non_ccl_tests \
                                    --profile=/artifacts/profile.gz $EXTRA_PARAMS
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
@@ -3378,9 +3378,20 @@ test_suite(
 )
 
 test_suite(
-    name = "small_tests",
+    name = "ccl_tests",
     tags = [
         "-broken_in_bazel",
+        "ccl_test",
+        "-integration",
+    ],
+    tests = ALL_TESTS,
+)
+
+test_suite(
+    name = "small_non_ccl_tests",
+    tags = [
+        "-broken_in_bazel",
+        "-ccl_test",
         "-flaky",
         "-integration",
         "small",
@@ -3389,9 +3400,10 @@ test_suite(
 )
 
 test_suite(
-    name = "medium_tests",
+    name = "medium_non_ccl_tests",
     tags = [
         "-broken_in_bazel",
+        "-ccl_test",
         "-flaky",
         "-integration",
         "medium",
@@ -3400,9 +3412,10 @@ test_suite(
 )
 
 test_suite(
-    name = "large_tests",
+    name = "large_non_ccl_tests",
     tags = [
         "-broken_in_bazel",
+        "-ccl_test",
         "-flaky",
         "-integration",
         "large",
@@ -3411,9 +3424,10 @@ test_suite(
 )
 
 test_suite(
-    name = "enormous_tests",
+    name = "enormous_non_ccl_tests",
     tags = [
         "-broken_in_bazel",
+        "-ccl_test",
         "-flaky",
         "-integration",
         "enormous",

diff --git a/pkg/ccl/backupccl/BUILD.bazel b/pkg/ccl/backupccl/BUILD.bazel
@@ -195,7 +195,8 @@ go_test(
     args = ["-test.timeout=3595s"],
     data = glob(["testdata/**"]) + ["//c-deps:libgeos"],
     embed = [":backupccl"],
-    shard_count = 16,
+    shard_count = 48,
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/blobs",

diff --git a/pkg/ccl/backupccl/backupdest/BUILD.bazel b/pkg/ccl/backupccl/backupdest/BUILD.bazel
@@ -42,6 +42,7 @@ go_test(
         "main_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         ":backupdest",
         "//pkg/ccl",

diff --git a/pkg/ccl/backupccl/backupinfo/BUILD.bazel b/pkg/ccl/backupccl/backupinfo/BUILD.bazel
@@ -65,6 +65,7 @@ go_test(
         "manifest_handling_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         ":backupinfo",
         "//pkg/base",

diff --git a/pkg/ccl/backupccl/backuprand/BUILD.bazel b/pkg/ccl/backupccl/backuprand/BUILD.bazel
@@ -9,6 +9,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     data = ["//c-deps:libgeos"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/backupccl/backupresolver/BUILD.bazel b/pkg/ccl/backupccl/backupresolver/BUILD.bazel
@@ -32,6 +32,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     embed = [":backupresolver"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/ccl",
         "//pkg/ccl/storageccl",

diff --git a/pkg/ccl/baseccl/BUILD.bazel b/pkg/ccl/baseccl/BUILD.bazel
@@ -24,6 +24,7 @@ go_test(
     srcs = ["encryption_spec_test.go"],
     args = ["-test.timeout=55s"],
     embed = [":baseccl"],
+    tags = ["ccl_test"],
     deps = ["//pkg/util/leaktest"],
 )
 

diff --git a/pkg/ccl/benchccl/rttanalysisccl/BUILD.bazel b/pkg/ccl/benchccl/rttanalysisccl/BUILD.bazel
@@ -11,6 +11,7 @@ go_test(
     args = ["-test.timeout=3595s"],
     data = glob(["testdata/**"]),
     shard_count = 16,
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/bench/rttanalysis",

diff --git a/pkg/ccl/changefeedccl/BUILD.bazel b/pkg/ccl/changefeedccl/BUILD.bazel
@@ -195,6 +195,7 @@ go_test(
     args = ["-test.timeout=3595s"],
     embed = [":changefeedccl"],
     shard_count = 16,
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/blobs",

diff --git a/pkg/ccl/changefeedccl/cdceval/BUILD.bazel b/pkg/ccl/changefeedccl/cdceval/BUILD.bazel
@@ -63,6 +63,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     embed = [":cdceval"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl/changefeedccl/cdcevent",

diff --git a/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel b/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel
@@ -52,6 +52,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     embed = [":cdcevent"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl/changefeedccl/cdctest",

diff --git a/pkg/ccl/changefeedccl/cdctest/BUILD.bazel b/pkg/ccl/changefeedccl/cdctest/BUILD.bazel
@@ -50,6 +50,7 @@ go_test(
     ],
     args = ["-test.timeout=55s"],
     embed = [":cdctest"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/changefeedccl/cdcutils/BUILD.bazel b/pkg/ccl/changefeedccl/cdcutils/BUILD.bazel
@@ -22,6 +22,7 @@ go_test(
     srcs = ["throttle_test.go"],
     args = ["-test.timeout=295s"],
     embed = [":cdcutils"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/ccl/changefeedccl/changefeedbase",
         "//pkg/settings/cluster",

diff --git a/pkg/ccl/changefeedccl/changefeedbase/BUILD.bazel b/pkg/ccl/changefeedccl/changefeedbase/BUILD.bazel
@@ -31,6 +31,7 @@ go_test(
     srcs = ["options_test.go"],
     args = ["-test.timeout=295s"],
     embed = [":changefeedbase"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/util/leaktest",
         "//pkg/util/log",

diff --git a/pkg/ccl/changefeedccl/kvevent/BUILD.bazel b/pkg/ccl/changefeedccl/kvevent/BUILD.bazel
@@ -43,6 +43,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     embed = [":kvevent"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/jobs/jobspb",
         "//pkg/keys",

diff --git a/pkg/ccl/changefeedccl/kvfeed/BUILD.bazel b/pkg/ccl/changefeedccl/kvfeed/BUILD.bazel
@@ -49,6 +49,7 @@ go_test(
     ],
     args = ["-test.timeout=55s"],
     embed = [":kvfeed"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/changefeedccl/schemafeed/BUILD.bazel b/pkg/ccl/changefeedccl/schemafeed/BUILD.bazel
@@ -55,6 +55,7 @@ go_test(
     args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
     embed = [":schemafeed"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/cliccl/BUILD.bazel b/pkg/ccl/cliccl/BUILD.bazel
@@ -55,6 +55,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     embed = [":cliccl"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/build",
         "//pkg/ccl",

diff --git a/pkg/ccl/cloudccl/amazon/BUILD.bazel b/pkg/ccl/cloudccl/amazon/BUILD.bazel
@@ -8,6 +8,7 @@ go_test(
         "s3_connection_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/cloudccl/azure/BUILD.bazel b/pkg/ccl/cloudccl/azure/BUILD.bazel
@@ -8,6 +8,7 @@ go_test(
         "main_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/cloudccl/cloudprivilege/BUILD.bazel b/pkg/ccl/cloudccl/cloudprivilege/BUILD.bazel
@@ -8,6 +8,7 @@ go_test(
         "privileges_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/cloudccl/externalconn/BUILD.bazel b/pkg/ccl/cloudccl/externalconn/BUILD.bazel
@@ -9,6 +9,7 @@ go_test(
     ],
     args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl/backupccl",

diff --git a/pkg/ccl/cloudccl/gcp/BUILD.bazel b/pkg/ccl/cloudccl/gcp/BUILD.bazel
@@ -8,6 +8,7 @@ go_test(
         "main_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/importerccl/BUILD.bazel b/pkg/ccl/importerccl/BUILD.bazel
@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",
         "//pkg/sql/importer:testdata",
     ],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/build/bazel",

diff --git a/pkg/ccl/jobsccl/jobsprotectedtsccl/BUILD.bazel b/pkg/ccl/jobsccl/jobsprotectedtsccl/BUILD.bazel
@@ -8,6 +8,7 @@ go_test(
         "main_test.go",
     ],
     args = ["-test.timeout=295s"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/jwtauthccl/BUILD.bazel b/pkg/ccl/jwtauthccl/BUILD.bazel
@@ -36,6 +36,7 @@ go_test(
     ],
     args = ["-test.timeout=55s"],
     embed = [":jwtauthccl"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/kvccl/kvfollowerreadsccl/BUILD.bazel b/pkg/ccl/kvccl/kvfollowerreadsccl/BUILD.bazel
@@ -45,6 +45,7 @@ go_test(
     args = ["-test.timeout=295s"],
     data = glob(["testdata/**"]),
     embed = [":kvfollowerreadsccl"],
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/kvccl/kvtenantccl/BUILD.bazel b/pkg/ccl/kvccl/kvtenantccl/BUILD.bazel
@@ -59,6 +59,7 @@ go_test(
     args = ["-test.timeout=895s"],
     embed = [":kvtenantccl"],
     shard_count = 16,
+    tags = ["ccl_test"],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/3node-tenant-multiregion/BUILD.bazel b/pkg/ccl/logictestccl/tests/3node-tenant-multiregion/BUILD.bazel
@@ -12,7 +12,10 @@ go_test(
         "//pkg/sql/opt/exec/execbuilder:testdata",  # keep
     ],
     shard_count = 4,
-    tags = ["cpu:2"],
+    tags = [
+        "ccl_test",
+        "cpu:2",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/3node-tenant/BUILD.bazel b/pkg/ccl/logictestccl/tests/3node-tenant/BUILD.bazel
@@ -12,8 +12,11 @@ go_test(
         "//pkg/sql/logictest:testdata",  # keep
         "//pkg/sql/opt/exec/execbuilder:testdata",  # keep
     ],
-    shard_count = 16,
-    tags = ["cpu:2"],
+    shard_count = 48,
+    tags = [
+        "ccl_test",
+        "cpu:2",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/5node/BUILD.bazel b/pkg/ccl/logictestccl/tests/5node/BUILD.bazel
@@ -11,7 +11,10 @@ go_test(
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
     shard_count = 5,
-    tags = ["cpu:3"],
+    tags = [
+        "ccl_test",
+        "cpu:3",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/fakedist-disk/BUILD.bazel b/pkg/ccl/logictestccl/tests/fakedist-disk/BUILD.bazel
@@ -11,7 +11,10 @@ go_test(
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
     shard_count = 5,
-    tags = ["cpu:2"],
+    tags = [
+        "ccl_test",
+        "cpu:2",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/fakedist-vec-off/BUILD.bazel b/pkg/ccl/logictestccl/tests/fakedist-vec-off/BUILD.bazel
@@ -11,7 +11,10 @@ go_test(
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
     shard_count = 5,
-    tags = ["cpu:2"],
+    tags = [
+        "ccl_test",
+        "cpu:2",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",

diff --git a/pkg/ccl/logictestccl/tests/fakedist/BUILD.bazel b/pkg/ccl/logictestccl/tests/fakedist/BUILD.bazel
@@ -11,7 +11,10 @@ go_test(
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
     shard_count = 6,
-    tags = ["cpu:2"],
+    tags = [
+        "ccl_test",
+        "cpu:2",
+    ],
     deps = [
         "//pkg/build/bazel",
         "//pkg/ccl",