build: add basic infrastructure for remote execution with EngFlow

Add the `--config engflow` which sets some appropriate configurations
for building against our engflow cluster, and set some other metadata.
Also bump some test sizes and shard counts to get everything working.

Epic: CRDB-8308
Release note: None

.bazelrc

@@ -53,14 +53,15 @@ test:race --test_timeout=1200,6000,18000,72000
 # CI uses a custom timeout for enormous targets.
 test:use_ci_timeouts --test_timeout=60,300,900,900 --define=use_ci_timeouts=true
 # CI should always run with `--config=ci`.
-build:ci --lintonbuild
+build:cibase --lintonbuild
 # Set `-test.v` in Go tests.
 # Ref: https://github.com/bazelbuild/rules_go/pull/2456
-test:ci --test_env=GO_TEST_WRAP_TESTV=1
+test:cibase --test_env=GO_TEST_WRAP_TESTV=1
 # Dump all output for failed tests to the build log.
-test:ci --test_output=errors
+test:cibase --test_output=errors
 # Put all tmp artifacts in /artifacts/tmp.
-build:ci --test_tmpdir=/artifacts/tmp
+test:ci --test_tmpdir=/artifacts/tmp
+build:ci --config=cibase
 
 build:cross --stamp
 
@@ -111,6 +112,23 @@ build:macos --host_action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/b
 # more precise --workspace_status_command option.
 build:simplestamp --stamp --workspace_status_command=./build/bazelutil/stamp.sh
 
+build:engflow --define=EXECUTOR=remote
+build:engflow --disk_cache=
+build:engflow --experimental_inmemory_dotd_files
+build:engflow --experimental_inmemory_jdeps_files
+build:engflow --incompatible_strict_action_env=true
+build:engflow --remote_timeout=600
+build:engflow --nolegacy_important_outputs
+build:engflow --grpc_keepalive_time=30s
+build:engflow --experimental_remote_cache_compression=true
+build:engflow --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build:engflow --remote_cache=grpcs://tanzanite.cluster.engflow.com
+build:engflow --remote_executor=grpcs://tanzanite.cluster.engflow.com
+build:engflow --bes_backend=grpcs://tanzanite.cluster.engflow.com
+build:engflow --bes_results_url=https://tanzanite.cluster.engflow.com/invocation/
+build:engflow --extra_execution_platforms=//build/toolchains:cross_linux
+test:engflow --test_env=REMOTE_EXEC=1
+
 try-import %workspace%/.bazelrc.user
 
 # vi: ft=sh

build/toolchains/BUILD.bazel

@@ -31,6 +31,11 @@ platform(
         "@platforms//os:linux",
         "@platforms//cpu:x86_64",
     ],
+    exec_properties = {
+        "container-image": "docker://cockroachdb/bazel@sha256:2fdff6a4a2c66bf01d6ad4c6973eb4b38acdb812d07a431a0471f2b6449bb653",
+        "dockerReuse": "True",
+        "Pool": "default",
+    },
 )
 
 toolchain(
@@ -131,6 +136,11 @@ platform(
         "@platforms//os:linux",
         "@platforms//cpu:arm64",
     ],
+    exec_properties = {
+        "container-image": "docker://cockroachdb/bazel@sha256:6acc131994de3e9adcdf05ddd0956a047cb7a7e07a939a76fa9cad11af3a1a8a",
+        "dockerReuse": "True",
+        "Pool": "default",
+    },
 )
 
 platform(

pkg/acceptance/BUILD.bazel

@@ -53,6 +53,7 @@ go_test(
         "//pkg/cli:interactive_tests",
     ],
     embed = [":acceptance"],
+    exec_properties = {"Pool": "large"},
     gotags = ["acceptance"],
     shard_count = 16,
     tags = ["integration"],

pkg/ccl/backupccl/BUILD.bazel

@@ -205,6 +205,7 @@ go_test(
     }),
     data = glob(["testdata/**"]) + ["//c-deps:libgeos"],
     embed = [":backupccl"],
+    exec_properties = {"Pool": "large"},
     shard_count = 48,
     tags = ["ccl_test"],
     deps = [

pkg/ccl/benchccl/rttanalysisccl/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//conditions:default": ["-test.timeout=3595s"],
     }),
     data = glob(["testdata/**"]),
+    exec_properties = {"Pool": "large"},
     shard_count = 16,
     tags = ["ccl_test"],
     deps = [

pkg/ccl/changefeedccl/BUILD.bazel

@@ -206,6 +206,7 @@ go_test(
         "//conditions:default": ["-test.timeout=3595s"],
     }),
     embed = [":changefeedccl"],
+    exec_properties = {"Pool": "large"},
     shard_count = 16,
     tags = ["ccl_test"],
     deps = [

pkg/ccl/kvccl/kvtenantccl/upgradeinterlockccl/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//build/toolchains:use_ci_timeouts": ["-test.timeout=895s"],
         "//conditions:default": ["-test.timeout=3595s"],
     }),
+    exec_properties = {"Pool": "large"},
     shard_count = 14,
     tags = ["ccl_test"],
     deps = [

pkg/ccl/logictestccl/tests/3node-tenant-multiregion/BUILD.bazel

@@ -14,6 +14,7 @@ go_test(
         "//pkg/sql/logictest:testdata",  # keep
         "//pkg/sql/opt/exec/execbuilder:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/3node-tenant/BUILD.bazel

@@ -14,6 +14,7 @@ go_test(
         "//pkg/sql/logictest:testdata",  # keep
         "//pkg/sql/opt/exec/execbuilder:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 48,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/5node/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/fakedist-disk/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/fakedist-vec-off/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/fakedist/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 6,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/local-legacy-schema-changer/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/local-mixed-22.2-23.1/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 6,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/local-vec-off/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 5,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/local/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 19,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-15node-5region-3azs/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 4,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-3node-3superlongregions/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 1,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-no-los/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 19,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-tenant/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 15,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-vec-off/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 8,
     tags = [
         "ccl_test",

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 26,
     tags = [
         "ccl_test",

pkg/ccl/multiregionccl/BUILD.bazel

@@ -44,6 +44,7 @@ go_test(
     }),
     data = glob(["testdata/**"]),
     embed = [":multiregionccl"],
+    exec_properties = {"Pool": "large"},
     shard_count = 16,
     tags = ["ccl_test"],
     deps = [

pkg/ccl/schemachangerccl/BUILD.bazel

@@ -31,6 +31,7 @@ go_test(
         "//pkg/sql/schemachanger:end_to_end_testdata",
     ],
     embed = [":schemachangerccl"],
+    exec_properties = {"Pool": "large"},
     shard_count = 48,
     tags = [
         "ccl_test",

pkg/ccl/serverccl/BUILD.bazel

@@ -47,6 +47,7 @@ go_test(
     }),
     data = glob(["testdata/**"]),
     embed = [":serverccl"],
+    exec_properties = {"Pool": "large"},
     tags = ["ccl_test"],
     deps = [
         "//pkg/base",

pkg/ccl/sqlitelogictestccl/tests/3node-tenant/BUILD.bazel

@@ -9,6 +9,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "@com_github_cockroachdb_sqllogictest//:testfiles",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 48,
     tags = [
         "ccl_test",

pkg/ccl/sqlproxyccl/BUILD.bazel

@@ -59,7 +59,7 @@ go_library(
 
 go_test(
     name = "sqlproxyccl_test",
-    size = "medium",
+    size = "large",
     srcs = [
         "authentication_test.go",
         "backend_dialer_test.go",
@@ -73,9 +73,10 @@ go_test(
         "proxy_handler_test.go",
         "server_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = ["-test.timeout=895s"],
     data = glob(["testdata/**"]),
     embed = [":sqlproxyccl"],
+    shard_count = 8,
     tags = ["ccl_test"],
     deps = [
         "//pkg/base",

pkg/ccl/streamingccl/streamclient/BUILD.bazel

@@ -50,7 +50,10 @@ go_test(
     ],
     args = ["-test.timeout=55s"],
     embed = [":streamclient"],
-    tags = ["ccl_test"],
+    tags = [
+        "ccl_test",
+        "no-remote-exec",
+    ],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/ccl/streamingccl/streamingest/BUILD.bazel

@@ -97,7 +97,10 @@ go_test(
     data = glob(["testdata/**"]),
     embed = [":streamingest"],
     shard_count = 16,
-    tags = ["ccl_test"],
+    tags = [
+        "ccl_test",
+        "no-remote-exec",
+    ],
     deps = [
         "//pkg/base",
         "//pkg/ccl",

pkg/cmd/generate-bazel-extra/main.go

@@ -252,13 +252,13 @@ func excludeReallyEnormousTargets(targets []string) []string {
 		// Answer the following questions before adding a test target to this list:
 		//  1. Does this target run in Bazel Essential CI? If it does and you need
 		//     timeout to be > 1 hour then you need to talk to dev-inf. This is not
-		//	   expected.
+		//     expected.
 		//  2. Are you increasing the timeout for stress-testing purposes in CI? Make
-		// 	   your change in `pkg/cmd/teamcity-trigger` by updating `customTimeouts`.
-		//	3. You should only add a test target here if it's for stand-alone testing.
-		//	   For example: `/pkg/sql/sqlitelogictest` is only tested in a nightly in
-		//	   `build/teamcity/cockroach/nightlies/sqlite_logic_test_impl.sh`. If this is
-		//	   the case, you should tag your test as `integration`.
+		//     your change in `pkg/cmd/teamcity-trigger` by updating `customTimeouts`.
+		//  3. You should only add a test target here if it's for stand-alone testing.
+		//     For example: `/pkg/sql/sqlitelogictest` is only tested in a nightly in
+		//     `build/teamcity/cockroach/nightlies/sqlite_logic_test_impl.sh`. If this is
+		//     the case, you should tag your test as `integration`.
 		//  4. If you are not sure, please ask the dev-inf team for help.
 		for _, toExclude := range []string{
 			"//pkg/ccl/sqlitelogictestccl",
@@ -288,6 +288,7 @@ func generateTestsTimeouts() {
 	}
 	for size, defaultTimeout := range testSizeToDefaultTimeout {
 		if size == "enormous" {
+			runBuildozer(append([]string{`dict_set exec_properties Pool:large`}, targets[size]...))
 			// Exclude really enormous targets since they have a custom timeout that
 			// exceeds the default 1h.
 			targets[size] = excludeReallyEnormousTargets(targets[size])

pkg/cmd/generate-logictest/templates.go

@@ -268,6 +268,7 @@ go_test(
         "//pkg/sql/logictest:testdata",  # keep{{ end }}{{ if .ExecBuildLogicTest }}
         "//pkg/sql/opt/exec/execbuilder:testdata",  # keep{{ end }}
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = {{ if gt .TestCount 48 }}48{{ else }}{{ .TestCount }}{{end}},
     tags = [{{ if .Ccl }}
         "ccl_test",{{ end }}

pkg/compose/BUILD.bazel

@@ -20,6 +20,7 @@ go_test(
         "//pkg/compose:compare/docker-compose.yml",
     ],
     embed = [":compose"],
+    exec_properties = {"Pool": "large"},
     gotags = ["compose"],
     tags = ["integration"],
     deps = [

pkg/kv/kvserver/BUILD.bazel

@@ -360,6 +360,7 @@ go_test(
     }),
     data = glob(["testdata/**"]),
     embed = [":kvserver"],
+    exec_properties = {"Pool": "large"},
     shard_count = 48,
     tags = ["cpu:2"],
     deps = [

pkg/server/BUILD.bazel

@@ -471,6 +471,7 @@ go_test(
     }),
     data = glob(["testdata/**"]),
     embed = [":server"],
+    exec_properties = {"Pool": "large"},
     shard_count = 16,
     deps = [
         "//pkg/base",

pkg/sql/BUILD.bazel

@@ -734,7 +734,9 @@ go_test(
         "//pkg/sql/vtable:pg_catalog.go",
     ],
     embed = [":sql"],
+    exec_properties = {"Pool": "large"},
     shard_count = 16,
+    tags = ["no-remote-exec"],
     deps = [
         "//pkg/base",
         "//pkg/build/bazel",

pkg/sql/gcjob_test/BUILD.bazel

@@ -8,6 +8,7 @@ go_test(
         "main_test.go",
     ],
     args = ["-test.timeout=295s"],
+    shard_count = 2,
     deps = [
         "//pkg/base",
         "//pkg/clusterversion",

pkg/sql/logictest/tests/5node-disk/BUILD.bazel

@@ -12,6 +12,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/sql/logictest:testdata",  # keep
     ],
+    exec_properties = {"Pool": "large"},
     shard_count = 14,
     tags = [
         "cpu:3",