-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #112562 from RaduBerinde/backport23.1-111382-111410
release-23.1: backport changes to roachtest build scripts
- Loading branch information
Showing
5 changed files
with
214 additions
and
64 deletions.
There are no files selected for viewing
12 changes: 12 additions & 0 deletions
12
build/teamcity/cockroach/coverage/roachtest_nightly_gce.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -exuo pipefail | ||
|
|
||
| dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" | ||
|
|
||
| source "$dir/teamcity-support.sh" # For $root | ||
| source "$dir/teamcity-bazel-support.sh" # For run_bazel | ||
|
|
||
| CLOUD=gce | ||
| BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY" \ | ||
| run_bazel build/teamcity/cockroach/coverage/roachtest_nightly_gce_impl.sh |
39 changes: 39 additions & 0 deletions
39
build/teamcity/cockroach/coverage/roachtest_nightly_gce_impl.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -exuo pipefail | ||
|
|
||
| dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" | ||
|
|
||
| source "$dir/teamcity-support.sh" | ||
|
|
||
| if [[ ! -f ~/.ssh/id_rsa.pub ]]; then | ||
| ssh-keygen -q -C "roachtest-nightly-bazel $(date)" -N "" -f ~/.ssh/id_rsa | ||
| fi | ||
|
|
||
| # We will only spin up AMD64 clusters. Note that the TeamCity runner (this host) | ||
| # doesn't need to be AMD64; the roachtest binary is always built for the host | ||
| # architecture. | ||
| $root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh --with-code-coverage amd64 | ||
|
|
||
| echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json | ||
| gcloud auth activate-service-account --key-file=creds.json | ||
| export ROACHPROD_USER=teamcity | ||
|
|
||
| # See build/teamcity/util/roachtest_util.sh. | ||
| PARALLELISM=16 | ||
| CPUQUOTA=1024 | ||
| FILTER="tag:aws tag:default" | ||
|
|
||
| build/teamcity-roachtest-invoke.sh \ | ||
| --metamorphic-encryption-probability=0.5 \ | ||
| --select-probability="${SELECT_PROBABILITY:-1.0}" \ | ||
| --cloud="${CLOUD}" \ | ||
| --count="${COUNT-1}" \ | ||
| --parallelism="${PARALLELISM}" \ | ||
| --cpu-quota="${CPUQUOTA}" \ | ||
| --cluster-id="${TC_BUILD_ID}" \ | ||
| --artifacts=/artifacts \ | ||
| --artifacts-literal="${LITERAL_ARTIFACTS_DIR:-}" \ | ||
| --slack-token="${SLACK_TOKEN:-}" \ | ||
| --go-cover \ | ||
| ${TESTS:-} ${FILTER} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # arch_to_config returns the bazel config for the given arch. | ||
| function arch_to_config() { | ||
| case "$1" in | ||
| amd64) | ||
| echo "crosslinux" | ||
| ;; | ||
| arm64) | ||
| echo "crosslinuxarm" | ||
| ;; | ||
| amd64-fips) | ||
| echo "crosslinuxfips" | ||
| ;; | ||
| *) | ||
| echo "Error: invalid arch '$1'" >&2 | ||
| exit 1 | ||
| ;; | ||
| esac | ||
| } | ||
|
|
||
| # get_host_arch determines the current host's CPU architecture. | ||
| function get_host_arch() { | ||
| if [[ "$(uname -m)" =~ (arm64|aarch64)$ ]]; then | ||
| echo "arm64" | ||
| else | ||
| echo "amd64" | ||
| fi | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
build/teamcity/cockroach/nightlies/roachtest_compile_component.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| source "$(dirname $0)/roachtest_arch_util.sh" | ||
|
|
||
| if [ "$#" -eq 0 ]; then | ||
| echo "Builds components necessary for roachtests and stages them in bin/ and/or lib/." | ||
| echo "" | ||
| echo "Usage: $(basename $0) [--with-coverage] <os/arch/component>" | ||
| echo " where os is one of: linux" | ||
| echo " arch is one of: amd64, arm64, amd64-fips" | ||
| echo " component is one of: cockroach, cockroach-ea, workload, libgeos, roachtest" | ||
| echo " --with-coverage enables go code coverage instrumentation (only applies to cockroach binaries)" | ||
| exit 1 | ||
| fi | ||
|
|
||
| crdb_extra_flags="" | ||
| target="" | ||
|
|
||
| for arg in "$@"; do | ||
| case "$arg" in | ||
| --with-code-coverage) | ||
| crdb_extra_flags="--collect_code_coverage --bazel_code_coverage" | ||
| ;; | ||
| *) | ||
| if [ -n "$target" ]; then | ||
| echo "Error: too many arguments" | ||
| exit 1 | ||
| fi | ||
| target=$arg | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| # Split $target into $os/$arch/$component. | ||
| os=${target%%/*} | ||
| arch_and_comp=${target#*/} | ||
| arch=${arch_and_comp%%/*} | ||
| component=${arch_and_comp#*/} | ||
|
|
||
| if [ -z $os -o -z $arch -o -z $component ]; then | ||
| echo "Invalid target: $target; syntax is os/arch/component" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ "$os" != linux ]; then | ||
| echo "Invalid os; only linux supported" | ||
| exit 1 | ||
| fi | ||
|
|
||
| config=$(arch_to_config $arch) | ||
|
|
||
| # Array of arguments to be passed to bazel for the component. | ||
| bazel_args=() | ||
| # Array of build artifacts. Each item has format "src:dest"; src is relative to | ||
| # the bazel-bin directory, dst is relative to cwd. | ||
| artifacts=() | ||
|
|
||
| case "$component" in | ||
| cockroach) | ||
| # Cockroach binary. | ||
| bazel_args=(--config force_build_cdeps //pkg/cmd/cockroach $crdb_extra_flags) | ||
| artifacts=("pkg/cmd/cockroach/cockroach_/cockroach:bin/cockroach.$os-$arch") | ||
| ;; | ||
| cockroach-ea) | ||
| # Cockroach-short with enabled assertions (EA). | ||
| bazel_args=(--config force_build_cdeps //pkg/cmd/cockroach-short --crdb_test $crdb_extra_flags) | ||
| artifacts=("pkg/cmd/cockroach-short/cockroach-short_/cockroach-short:bin/cockroach-ea.$os-$arch") | ||
| ;; | ||
| workload) | ||
| # Workload binary. | ||
| bazel_args=(--config force_build_cdeps //pkg/cmd/workload) | ||
| artifacts=("pkg/cmd/workload/workload_/workload:bin/workload.$os-$arch") | ||
| ;; | ||
| libgeos) | ||
| # Geos libs. | ||
| bazel_args=(--config force_build_cdeps //c-deps:libgeos) | ||
| artifacts=( | ||
| "c-deps/libgeos_foreign/lib/libgeos.so:lib/libgeos.$os-$arch.so" | ||
| "c-deps/libgeos_foreign/lib/libgeos_c.so:lib/libgeos_c.$os-$arch.so" | ||
| ) | ||
| ;; | ||
| roachtest) | ||
| # Roachtest binary. | ||
| bazel_args=(//pkg/cmd/roachtest) | ||
| artifacts=("pkg/cmd/roachtest/roachtest_/roachtest:bin/roachtest.$os-$arch") | ||
| ;; | ||
| *) | ||
| echo "Unknown component '$component'" | ||
| exit 1 | ||
| ;; | ||
| esac | ||
|
|
||
| echo "Building $os/$arch/$component..." | ||
|
|
||
| bazel build --config $config --config ci -c opt "${bazel_args[@]}" | ||
| BAZEL_BIN=$(bazel info bazel-bin --config $config --config ci -c opt) | ||
| for artifact in "${artifacts[@]}"; do | ||
| src=${artifact%%:*} | ||
| dst=${artifact#*:} | ||
| cp "$BAZEL_BIN/$src" "$dst" | ||
| # Make files writable to simplify cleanup and copying (e.g., scp retry). | ||
| chmod a+w "$dst" | ||
| done |