From 8219a7fffa32765081f5b90b7d7aa96c90d6d815 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Fri, 7 Feb 2020 12:41:23 +0100 Subject: [PATCH] roachtest: check in CI configs The configurations for our nightly CI runs are maintained ad-hoc in TeamCity. In particular, there's a separate shell script for AWS and one for GCE. This is error-prone and hard to maintain. One argument we've had in the past for keeping it on TC is that it's easier to make tweaks, but I'd posit that we can just paste the script we're tweaking into CI while tweaking (ideally in a scratch copy of the job), or we can find ways to run the nightly off a custom branch (which is difficult right now due to the Publish Bleeding Edge dependency). The shell scripts were copied without modifications. Obviously they should be consolidated as a follow-up step. Release note: None --- build/teamcity-nightly-roachtest-aws.sh | 77 ++++++++++++++++++++++++ build/teamcity-nightly-roachtest-gce.sh | 80 +++++++++++++++++++++++++ build/teamcity-nightly-roachtest.sh | 9 +++ 3 files changed, 166 insertions(+) create mode 100755 build/teamcity-nightly-roachtest-aws.sh create mode 100755 build/teamcity-nightly-roachtest-gce.sh create mode 100755 build/teamcity-nightly-roachtest.sh diff --git a/build/teamcity-nightly-roachtest-aws.sh b/build/teamcity-nightly-roachtest-aws.sh new file mode 100755 index 000000000000..2a0f1f8c010d --- /dev/null +++ b/build/teamcity-nightly-roachtest-aws.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# TODO(peter,benesch): This script contains a ton of duplication with Roachtest Nightly - GCE + +# Note that when this script is called, the cockroach binary to be tested +# already exists in the current directory. + +if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then + echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json + gcloud auth activate-service-account --key-file=creds.json + export ROACHPROD_USER=teamcity +else + echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2 + echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2 +fi + +set -x + +if [[ ! -f ~/.ssh/id_rsa.pub ]]; then + ssh-keygen -q -N "" -f ~/.ssh/id_rsa +fi + +export PATH=$PATH:$(go env GOPATH)/bin + +build_tag=$(git describe --abbrev=0 --tags --match=v[0-9]*) +git checkout master +git pull origin master + +git rev-parse HEAD +make bin/workload bin/roachtest bin/roachprod + +# release-2.0 names the cockroach binary differently. +if [[ -f cockroach-linux-2.6.32-gnu-amd64 ]]; then + mv cockroach-linux-2.6.32-gnu-amd64 cockroach.linux-2.6.32-gnu-amd64 +fi + +chmod +x cockroach.linux-2.6.32-gnu-amd64 + +# Artifacts dir needs to match up with TC's (without any extra components). +artifacts=$PWD/artifacts +mkdir -p "$artifacts" + +# NB: Teamcity has a 1080 minute timeout that, when reached, +# kills the process without a stack trace (probably SIGKILL). +# We'd love to see a stack trace though, so after 1000 minutes, +# kill with SIGINT which will allow roachtest to fail tests and +# cleanup. +# +# NB(2): We specify --zones below so that nodes are created in us-central1-b +# by default. This reserves us-east1-b (the roachprod default zone) for use +# by manually created clusters. +exit_status=0 +if ! timeout -s INT $((1000*60)) bin/roachtest run \ + --build-tag "${build_tag}" \ + --slack-token "${SLACK_TOKEN}" \ + --cluster-id "${TC_BUILD_ID}" \ + --cloud "aws" \ + --cockroach "$PWD/cockroach.linux-2.6.32-gnu-amd64" \ + --roachprod "$PWD/bin/roachprod" \ + --workload "$PWD/bin/workload" \ + --artifacts "$artifacts" \ + --parallelism 3 \ + --teamcity \ + --cpu-quota=384 \ + "kv(0|95)|ycsb|tpcc/(headroom/n4cpu16)|tpccbench/(nodes=3/cpu=16)"; then + exit_status=$? +fi + +# Upload any stats.json files to the cockroach-nightly bucket on master. +if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then + for file in $(find ${artifacts#${PWD}/} -name stats.json); do + gsutil cp ${file} gs://cockroach-nightly-aws/${file} + done +fi + +exit "$exit_status" diff --git a/build/teamcity-nightly-roachtest-gce.sh b/build/teamcity-nightly-roachtest-gce.sh new file mode 100755 index 000000000000..d190ac496bca --- /dev/null +++ b/build/teamcity-nightly-roachtest-gce.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Note that when this script is called, the cockroach binary to be tested +# already exist in the current directory. + +if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then + echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json + gcloud auth activate-service-account --key-file=creds.json + export ROACHPROD_USER=teamcity +else + echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2 + echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2 +fi + +set -x + +if [[ ! -f ~/.ssh/id_rsa.pub ]]; then + ssh-keygen -q -N "" -f ~/.ssh/id_rsa +fi + + +# The artifacts dir should match up with that supplied by TC. +artifacts=$PWD/artifacts +mkdir -p "$artifacts" +chmod o+rwx "${artifacts}" + +export PATH=$PATH:$(go env GOPATH)/bin + +build_tag=$(git describe --abbrev=0 --tags --match=v[0-9]*) +git checkout master +git pull origin master + +git rev-parse HEAD +make bin/workload bin/roachtest bin/roachprod > "${artifacts}/build.txt" 2>&1 || cat "${artifacts}/build.txt" + +# release-2.0 names the cockroach binary differently. +if [[ -f cockroach-linux-2.6.32-gnu-amd64 ]]; then + mv cockroach-linux-2.6.32-gnu-amd64 cockroach.linux-2.6.32-gnu-amd64 +fi + +chmod +x cockroach.linux-2.6.32-gnu-amd64 + + +# NB: Teamcity has a 1080 minute timeout that, when reached, +# kills the process without a stack trace (probably SIGKILL). +# We'd love to see a stack trace though, so after 1000 minutes, +# kill with SIGINT which will allow roachtest to fail tests and +# cleanup. +# +# NB(2): We specify --zones below so that nodes are created in us-central1-b +# by default. This reserves us-east1-b (the roachprod default zone) for use +# by manually created clusters. +exit_status=0 +if ! timeout -s INT $((1000*60)) bin/roachtest run \ + --count="${COUNT-1}" \ + --debug="${DEBUG-false}" \ + --build-tag="${build_tag}" \ + --slack-token="${SLACK_TOKEN}" \ + --cluster-id="${TC_BUILD_ID}" \ + --zones="us-central1-b,us-west1-b,europe-west2-b" \ + --cockroach="$PWD/cockroach.linux-2.6.32-gnu-amd64" \ + --roachprod="$PWD/bin/roachprod" \ + --workload="$PWD/bin/workload" \ + --artifacts="$artifacts" \ + --parallelism=16 \ + --cpu-quota=1024 \ + --teamcity=true \ + "${TESTS}"; then + exit_status=$? +fi + +# Upload any stats.json files to the cockroach-nightly bucket. +if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then + for file in $(find ${artifacts#${PWD}/} -name stats.json); do + gsutil cp ${file} gs://cockroach-nightly/${file} + done +fi + +exit "$exit_status" diff --git a/build/teamcity-nightly-roachtest.sh b/build/teamcity-nightly-roachtest.sh new file mode 100755 index 000000000000..d97d44bc466b --- /dev/null +++ b/build/teamcity-nightly-roachtest.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Entry point for the nightly roachtests. These are run from CI and require +# appropriate secrets for the ${CLOUD} parameter (along with other things, +# apologies, you're going to have to dig around for them below or even better +# yet, look at the job). + +set -euo pipefail + +"build/teamcity-nightly-roachtest-${CLOUD}.sh"