Merge #71770 #72574 #73055 #73080

71770: bazel: run acceptance tests under Bazel r=rickystewart a=rail This patch makes the acceptance test work under Bazel. * Add `AbsCertsDir()` in order to keep track of certificate path for cases when tests change the working directory. * docker-compose tests to use interpolation and environment variables in order to override `CERTS_DIR` and `COCKROACH_BINARY`. * Add `copyRunfiles()` in order to copy Bazel-generated symlinked runfiles as regular files to make them available in docker mounted volumes. Related: #71932, #71930 Fixes: #59446 Release note: None 72574: ci: add bazel roachtest gce teamcity job r=rail a=rickystewart Release note: None 73055: ui: prevent undefined axis label on custom chart r=zachlite a=zachlite Release note (bug fix): Y-axis labels on custom charts no longer display 'undefined'. @thtruo, this fix addresses the issue as described in #72115. Now, when the user selects a new unit from the dropdown, the user will experience up to a 10 second delay before the axis label refreshes. @nathanstilwell and I investigated this last week, and we'd need more time to think of a solution that doesn't create long term maintenance headaches. The benefit of this PR as it stands is that the Y-axis label is noticeably less broken. FYI. 73080: bazel: don't shard `kvserver` test r=rail a=rickystewart The `exclusive` tag here prevents the shards from running concurrently. See #65407, #65582. Release note: None Co-authored-by: Rail Aliiev <[email protected]> Co-authored-by: Ricky Stewart <[email protected]> Co-authored-by: Zach Lite <[email protected]>
cockroachdb · Nov 23, 2021 · 2ac3cf4 · 2ac3cf4
5 parents 8bbcc17 + ccf0dd9 + 56260ba + ac5dfab + b0f95d2
commit 2ac3cf4
Show file tree

Hide file tree

Showing 26 changed files with 367 additions and 121 deletions.
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -129,6 +129,8 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
 # gazelle:exclude pkg/util/buildutil/crdb_test_dyn.go
 # gazelle:exclude pkg/util/buildutil/crdb_test_off.go
 # gazelle:exclude pkg/util/buildutil/crdb_test_on.go
+# gazelle:exclude pkg/acceptance/compose/gss/psql/*
+# gazelle:exclude pkg/acceptance/test_main.go
 #
 # Generally useful references:
 #

diff --git a/build/bazelbuilder/Dockerfile b/build/bazelbuilder/Dockerfile
@@ -19,6 +19,7 @@ RUN apt-get update \
     make \
     netbase \
     openjdk-8-jre \
+    openssh-client \
     patchelf \
     unzip \
  && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \
@@ -49,6 +50,12 @@ RUN apt-get update && \
     cd .. && \
     rm -rf git-2.29.2.zip git-2.29.2
 
+RUN curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+ && echo 'deb https://packages.cloud.google.com/apt cloud-sdk main' | tee /etc/apt/sources.list.d/gcloud.list \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    google-cloud-sdk
+
 RUN apt-get purge -y \
     apt-transport-https \
     flex \

diff --git a/build/bazelutil/check.sh b/build/bazelutil/check.sh
@@ -83,7 +83,7 @@ git grep '//go:generate' -- './*.go' | grep -v stringer | grep -v 'add-leaktest\
     exit 1
 done
 
-git grep 'broken_in_bazel' pkg | grep BUILD.bazel: | grep -v pkg/BUILD.bazel | grep -v generate-test-suites | cut -d: -f1 | while read LINE; do
+git grep 'broken_in_bazel' pkg | grep BUILD.bazel: | grep -v pkg/BUILD.bazel | grep -v pkg/cli/BUILD.bazel | grep -v generate-test-suites | cut -d: -f1 | while read LINE; do
     if [[ "$EXISTING_BROKEN_TESTS_IN_BAZEL" == *"$LINE"* ]]; then
 	# Grandfathered.
 	continue

diff --git a/build/teamcity-bazel-support.sh b/build/teamcity-bazel-support.sh
@@ -1,10 +1,11 @@
 # FYI: You can run `./dev builder` to run this Docker image. :)
 # `dev` depends on this variable! Don't change the name or format unless you
 # also update `dev` accordingly.
-BAZEL_IMAGE=cockroachdb/bazel:20211008-130456
+BAZEL_IMAGE=cockroachdb/bazel:20211109-174337
 
 # Call `run_bazel $NAME_OF_SCRIPT` to start an appropriately-configured Docker
 # container with the `cockroachdb/bazel` image running the given script.
+# BAZEL_SUPPORT_EXTRA_DOCKER_ARGS will be passed on to `docker run` unchanged.
 run_bazel() {
     if [ -z "${root:-}" ]
     then
@@ -28,6 +29,7 @@ run_bazel() {
     docker run -i ${tty-} --rm --init \
         -u "$(id -u):$(id -g)" \
         --workdir="/go/src/github.com/cockroachdb/cockroach" \
+	${BAZEL_SUPPORT_EXTRA_DOCKER_ARGS:+$BAZEL_SUPPORT_EXTRA_DOCKER_ARGS} \
         ${vols} \
         $BAZEL_IMAGE "$@"
 }
diff --git a/build/teamcity-nightly-roachtest.sh b/build/teamcity-nightly-roachtest.sh
@@ -23,84 +23,8 @@ export PATH=$PATH:$(GOFLAGS=; go env GOPATH)/bin
 build/builder/mkrelease.sh amd64-linux-gnu build bin/workload bin/roachtest bin/roachprod \
   > "${artifacts}/build.txt" 2>&1 || (cat "${artifacts}/build.txt"; false)
 
-# Set up Google credentials. Note that we need this for all clouds since we upload
-# perf artifacts to Google Storage at the end.
-if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then
-  echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json
-  gcloud auth activate-service-account --key-file=creds.json
-  export ROACHPROD_USER=teamcity
-else
-  echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2
-  echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2
-fi
-
-# Early bind the stats dir. Roachtest invocations can take ages, and we want the
-# date at the time of the start of the run (which identifies the version of the
-# code run best).
-stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}"
-
-# Set up a function we'll invoke at the end.
-function upload_stats {
- if tc_release_branch; then
-      bucket="cockroach-nightly-${CLOUD}"
-      if [[ "${CLOUD}" == "gce" ]]; then
-          # GCE, having been there first, gets an exemption.
-          bucket="cockroach-nightly"
-      fi
-
-      remote_artifacts_dir="artifacts-${TC_BUILD_BRANCH}"
-      if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then
-        # The master branch is special, as roachperf hard-codes
-        # the location.
-        remote_artifacts_dir="artifacts"
-      fi
-
-      # The stats.json files need some path translation:
-      #     ${artifacts}/path/to/test/stats.json
-      # to
-      #     gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json
-      #
-      # `find` below will expand "{}" as ./path/to/test/stats.json. We need
-      # to bend over backwards to remove the `./` prefix or gsutil will have
-      # a `.` folder in ${stats_dir}, which we don't want.
-      (cd "${artifacts}" && \
-        while IFS= read -r f; do
-          if [[ -n "${f}" ]]; then
-            gsutil cp "${f}" "gs://${bucket}/${remote_artifacts_dir}/${stats_dir}/${f}"
-          fi
-        done <<< "$(find . -name stats.json | sed 's/^\.\///')")
-  fi
-}
-
-# Upload any stats.json we can find, no matter what happens.
-trap upload_stats EXIT
-
-# Set up the parameters for the roachtest invocation.
-PARALLELISM=16
-CPUQUOTA=1024
-ZONES=""
-TESTS=""
-case "${CLOUD}" in
-  gce)
-    # We specify --zones below so that nodes are created in us-central1-b by
-    # default. This reserves us-east1-b (the roachprod default zone) for use by
-    # manually created clusters.
-    ZONES="us-central1-b,us-west1-b,europe-west2-b"
-    ;;
-  aws)
-    PARALLELISM=3
-    CPUQUOTA=384
-    if [ -z "${TESTS}" ]; then
-      # NB: anchor ycsb to beginning of line to avoid matching `zfs/ycsb/*` which
-      # isn't supported on AWS at time of writing.
-      TESTS="kv(0|95)|^ycsb|tpcc/(headroom/n4cpu16)|tpccbench/(nodes=3/cpu=16)|scbench/randomload/(nodes=3/ops=2000/conc=1)|backup/(KMS/n3cpu4)"
-    fi
-    ;;
-  *)
-    echo "unknown cloud ${CLOUD}"
-    exit 1
-    ;;
-esac
+# Set up GCE authentication, artifact upload logic, and the PARALLELISM/CPUQUOTA/TESTS env variables.
+source $root/build/teamcity/util/roachtest_util.sh
 
 build/teamcity-roachtest-invoke.sh \
   --cloud="${CLOUD}" \

diff --git a/build/teamcity/cockroach/ci/tests/acceptance.sh b/build/teamcity/cockroach/ci/tests/acceptance.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+source "$dir/teamcity-support.sh"
+
+tc_prepare
+
+export ARTIFACTSDIR=$PWD/artifacts/acceptance
+mkdir -p "$ARTIFACTSDIR"
+
+tc_start_block "Run acceptance tests"
+bazel run \
+  //pkg/acceptance:acceptance_test \
+  --config=crosslinux --config=test \
+  --test_arg=-l="$ARTIFACTSDIR" \
+  --test_env=TZ=America/New_York \
+  --test_timeout=1800
+tc_end_block "Run acceptance tests"
diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_TAG -e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILD_BRANCH -e TC_BUILD_ID" \
+			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"
+
+if [[ ! -f ~/.ssh/id_rsa.pub ]]; then
+  ssh-keygen -q -C "roachtest-nightly-bazel $(date)" -N "" -f ~/.ssh/id_rsa
+fi
+
+bazel build --config crosslinux --config ci --config with_ui -c opt \
+      //pkg/cmd/cockroach //pkg/cmd/workload //pkg/cmd/roachtest \
+      //pkg/cmd/roachprod //c-deps:libgeos
+BAZEL_BIN=$(bazel info bazel-bin --config crosslinux --config ci --config with_ui -c opt)
+# Move this stuff to bin for simplicity.
+mkdir -p bin
+chmod o+rwx bin
+cp $BAZEL_BIN/pkg/cmd/cockroach/cockroach_/cockroach bin
+cp $BAZEL_BIN/pkg/cmd/roachprod/roachprod_/roachprod bin
+cp $BAZEL_BIN/pkg/cmd/roachtest/roachtest_/roachtest bin
+cp $BAZEL_BIN/pkg/cmd/workload/workload_/workload    bin
+chmod a+w bin/cockroach bin/roachprod bin/roachtest bin/workload
+# Stage the geos libs in the appropriate spot.
+mkdir -p lib.docker_amd64
+chmod o+rwx lib.docker_amd64
+cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos.so   lib.docker_amd64
+cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos_c.so lib.docker_amd64
+chmod a+w lib.docker_amd64/libgeos.so lib.docker_amd64/libgeos_c.so
+
+artifacts=/artifacts
+source $root/build/teamcity/util/roachtest_util.sh
+
+build/teamcity-roachtest-invoke.sh \
+  --cloud="${CLOUD}" \
+  --count="${COUNT-1}" \
+  --parallelism="${PARALLELISM}" \
+  --cpu-quota="${CPUQUOTA}" \
+  --cluster-id="${TC_BUILD_ID}" \
+  --build-tag="${BUILD_TAG}" \
+  --cockroach="${PWD}/bin/cockroach" \
+  --artifacts=/artifacts \
+  --artifacts-literal="${LITERAL_ARTIFACTS_DIR:-}" \
+  --slack-token="${SLACK_TOKEN}" \
+  "${TESTS}"
diff --git a/build/teamcity/util/roachtest_util.sh b/build/teamcity/util/roachtest_util.sh
@@ -0,0 +1,75 @@
+# Common logic used by the nightly roachtest scripts (Bazel and non-Bazel).
+
+# Set up Google credentials. Note that we need this for all clouds since we upload
+# perf artifacts to Google Storage at the end.
+if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then
+  echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json
+  gcloud auth activate-service-account --key-file=creds.json
+  export ROACHPROD_USER=teamcity
+else
+  echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2
+  echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2
+fi
+
+# Early bind the stats dir. Roachtest invocations can take ages, and we want the
+# date at the time of the start of the run (which identifies the version of the
+# code run best).
+stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}"
+
+# Set up a function we'll invoke at the end.
+function upload_stats {
+ if tc_release_branch; then
+      bucket="cockroach-nightly-${CLOUD}"
+      if [[ "${CLOUD}" == "gce" ]]; then
+          # GCE, having been there first, gets an exemption.
+          bucket="cockroach-nightly"
+      fi
+
+      remote_artifacts_dir="artifacts-${TC_BUILD_BRANCH}"
+      if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then
+        # The master branch is special, as roachperf hard-codes
+        # the location.
+        remote_artifacts_dir="artifacts"
+      fi
+
+      # The stats.json files need some path translation:
+      #     ${artifacts}/path/to/test/stats.json
+      # to
+      #     gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json
+      #
+      # `find` below will expand "{}" as ./path/to/test/stats.json. We need
+      # to bend over backwards to remove the `./` prefix or gsutil will have
+      # a `.` folder in ${stats_dir}, which we don't want.
+      (cd "${artifacts}" && \
+        while IFS= read -r f; do
+          if [[ -n "${f}" ]]; then
+            gsutil cp "${f}" "gs://${bucket}/${remote_artifacts_dir}/${stats_dir}/${f}"
+          fi
+        done <<< "$(find . -name stats.json | sed 's/^\.\///')")
+  fi
+}
+
+# Upload any stats.json we can find, no matter what happens.
+trap upload_stats EXIT
+
+# Set up the parameters for the roachtest invocation.
+PARALLELISM=16
+CPUQUOTA=1024
+TESTS=""
+case "${CLOUD}" in
+  gce)
+    ;;
+  aws)
+    PARALLELISM=3
+    CPUQUOTA=384
+    if [ -z "${TESTS}" ]; then
+      # NB: anchor ycsb to beginning of line to avoid matching `zfs/ycsb/*` which
+      # isn't supported on AWS at time of writing.
+      TESTS="kv(0|95)|^ycsb|tpcc/(headroom/n4cpu16)|tpccbench/(nodes=3/cpu=16)|scbench/randomload/(nodes=3/ops=2000/conc=1)|backup/(KMS/n3cpu4)"
+    fi
+    ;;
+  *)
+    echo "unknown cloud ${CLOUD}"
+    exit 1
+    ;;
+esac
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
@@ -2,7 +2,6 @@
 # gazelle:proto_strip_import_prefix /pkg
 
 ALL_TESTS = [
-    "//pkg/acceptance:acceptance_test",
     "//pkg/base:base_test",
     "//pkg/bench/rttanalysis:rttanalysis_test",
     "//pkg/bench:bench_test",

diff --git a/pkg/acceptance/BUILD.bazel b/pkg/acceptance/BUILD.bazel
@@ -4,7 +4,7 @@ go_library(
     name = "acceptance",
     srcs = [
         "flags.go",
-        "test_main.go",
+        "test_acceptance.go",  # keep
         "util_cluster.go",
         "util_docker.go",
     ],
@@ -13,9 +13,15 @@ go_library(
     deps = [
         "//pkg/acceptance/cluster",
         "//pkg/base",
+        "//pkg/build/bazel",
         "//pkg/security",
+        "//pkg/security/securitytest",  #keep
+        "//pkg/server",  # keep
         "//pkg/testutils",
+        "//pkg/testutils/serverutils",  # keep
+        "//pkg/testutils/testcluster",  # keep
         "//pkg/util/log",
+        "//pkg/util/randutil",  # keep
         "//pkg/util/stop",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_containerd_containerd//platforms",
@@ -34,11 +40,24 @@ go_test(
         "debug_remote_test.go",
         "main_test.go",
     ],
-    data = glob(["testdata/**"]),
+    args = [
+        "-e",
+        "/cockroach/cockroach",
+        "-b",
+        "../../../../../../cmd/cockroach/cockroach_/cockroach",
+    ],
+    data = glob([
+        "testdata/**",
+        "compose/**",
+    ]) + [
+        "//pkg/cli:interactive_tests",
+        "//pkg/cmd/cockroach:cockroach",
+    ],
     embed = [":acceptance"],
-    tags = ["broken_in_bazel"],
+    gotags = ["acceptance"],
     deps = [
         "//pkg/acceptance/cluster",
+        "//pkg/build/bazel",
         "//pkg/security",
         "//pkg/testutils/skip",
         "//pkg/util/log",

diff --git a/pkg/acceptance/cli_test.go b/pkg/acceptance/cli_test.go
@@ -18,14 +18,12 @@ import (
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/acceptance/cluster"
+	"github.com/cockroachdb/cockroach/pkg/build/bazel"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 )
 
-const testGlob = "../cli/interactive_tests/test*.tcl"
-const containerPath = "/go/src/github.com/cockroachdb/cockroach/cli/interactive_tests"
-
 var cmdBase = []string{
 	"/usr/bin/env",
 	"COCKROACH_SKIP_UPDATE_CHECK=1",
@@ -46,6 +44,7 @@ func TestDockerCLI(t *testing.T) {
 		skip.IgnoreLintf(t, `TODO(dt): No binary in one-shot container, see #6086: %s`, err)
 	}
 
+	testGlob := "../cli/interactive_tests/test*.tcl"
 	paths, err := filepath.Glob(testGlob)
 	if err != nil {
 		t.Fatal(err)
@@ -54,6 +53,10 @@ func TestDockerCLI(t *testing.T) {
 		t.Fatalf("no testfiles found (%v)", testGlob)
 	}
 
+	containerPath := "/go/src/github.com/cockroachdb/cockroach/cli/interactive_tests"
+	if bazel.BuiltWithBazel() {
+		containerPath = "/mnt/interactive_tests"
+	}
 	for _, p := range paths {
 		testFile := filepath.Base(p)
 		testPath := filepath.Join(containerPath, testFile)