From dcbff22067b27e2908c46bbb4932b9ca7fdec7ef Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Mon, 18 Nov 2024 18:39:00 +0530 Subject: [PATCH] build/teamcity: add changes to enable openmetrics in nightly roachtests There was a regression in https://github.com/cockroachdb/cockroach/pull/135239. The change was reverted and this PR aims to fix the regressions Epic: https://cockroachlabs.atlassian.net/browse/CRDB-41852 Release note: None --- .../nightlies/roachtest_nightly_aws.sh | 2 +- .../nightlies/roachtest_nightly_azure.sh | 2 +- .../nightlies/roachtest_nightly_gce.sh | 2 +- .../nightlies/roachtest_nightly_impl.sh | 1 + .../cockroach/nightlies/roachtest_weekly.sh | 2 +- .../nightlies/roachtest_weekly_aws.sh | 2 +- .../nightlies/roachtest_weekly_impl.sh | 3 +- build/teamcity/util/roachtest_util.sh | 98 +++++++++++-------- 8 files changed, 67 insertions(+), 45 deletions(-) diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh index 6d8235585a1d..b5228c329c4b 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh index 86b93565390b..00b3844cd031 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh index 6f7b9e97d8ea..001531b16062 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh index ed58b84a0bf7..930c1d07a18b 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh @@ -84,4 +84,5 @@ build/teamcity-roachtest-invoke.sh \ --suite nightly \ --selective-tests="${SELECTIVE_TESTS:-true}" \ --side-eye-token="${SIDE_EYE_API_TOKEN}" \ + --export-openmetrics="${EXPORT_OPENMETRICS:-false}" \ "${TESTS}" diff --git a/build/teamcity/cockroach/nightlies/roachtest_weekly.sh b/build/teamcity/cockroach/nightlies/roachtest_weekly.sh index daca6f27ac43..af064a4c9a77 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_weekly.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_weekly.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_weekly_aws.sh b/build/teamcity/cockroach/nightlies/roachtest_weekly_aws.sh index ed1e8e1d2241..bcf75f01107a 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_weekly_aws.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_weekly_aws.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh b/build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh index 1b4faac6acb6..d00b943a0a25 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh @@ -47,4 +47,5 @@ timeout -s INT $((7800*60)) build/teamcity-roachtest-invoke.sh \ --metamorphic-arm64-probability="${ARM_PROBABILITY:-0.5}" \ --use-spot="${USE_SPOT:-auto}" \ --slack-token="${SLACK_TOKEN}" \ - --side-eye-token="${SIDE_EYE_API_TOKEN}" + --side-eye-token="${SIDE_EYE_API_TOKEN}" \ + --export-openmetrics="${EXPORT_OPENMETRICS:-false}" diff --git a/build/teamcity/util/roachtest_util.sh b/build/teamcity/util/roachtest_util.sh index 3f9468ca991e..81f1e87a0581 100644 --- a/build/teamcity/util/roachtest_util.sh +++ b/build/teamcity/util/roachtest_util.sh @@ -23,50 +23,70 @@ source $root/build/teamcity/util/roachtest_arch_util.sh # date at the time of the start of the run (which identifies the version of the # code run best). stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}" +stats_file_name="stats.json" + +# Provide a default value for EXPORT_OPENMETRICS if it is not set +EXPORT_OPENMETRICS="${EXPORT_OPENMETRICS:-false}" + +if [[ "${EXPORT_OPENMETRICS}" == "true" ]]; then + stats_file_name="stats.om" +fi # Set up a function we'll invoke at the end. function upload_stats { if tc_release_branch; then - bucket="${ROACHTEST_BUCKET:-cockroach-nightly-${CLOUD}}" - if [[ "${CLOUD}" == "gce" ]]; then - # GCE, having been there first, gets an exemption. - bucket="cockroach-nightly" - fi + bucket="${ROACHTEST_BUCKET:-cockroach-nightly-${CLOUD}}" + if [[ "${EXPORT_OPENMETRICS}" == "true" ]]; then - branch=$(tc_build_branch) - remote_artifacts_dir="artifacts-${branch}" - if [[ "${branch}" == "master" ]]; then - # The master branch is special, as roachperf hard-codes - # the location. - remote_artifacts_dir="artifacts" - fi - # TODO: FIPS_ENABLED is deprecated, use roachtest --metamorphic-fips-probability, instead. - # In FIPS-mode, keep artifacts separate by using the 'fips' suffix. - if [[ ${FIPS_ENABLED:-0} == 1 ]]; then - remote_artifacts_dir="${remote_artifacts_dir}-fips" - fi - - # The stats.json files need some path translation: - # ${artifacts}/path/to/test/stats.json - # to - # gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json - # - # `find` below will expand "{}" as ./path/to/test/stats.json. We need - # to bend over backwards to remove the `./` prefix or gsutil will have - # a `.` folder in ${stats_dir}, which we don't want. - (cd "${artifacts}" && \ - while IFS= read -r f; do - if [[ -n "${f}" ]]; then - artifacts_dir="${remote_artifacts_dir}" - # If 'cpu_arch=xxx' is encoded in the path, use it as suffix to separate artifacts by cpu_arch. - if [[ "${f}" == *"/cpu_arch=arm64/"* ]]; then - artifacts_dir="${artifacts_dir}-arm64" - elif [[ "${f}" == *"/cpu_arch=fips/"* ]]; then - artifacts_dir="${artifacts_dir}-fips" - fi - gsutil cp "${f}" "gs://${bucket}/${artifacts_dir}/${stats_dir}/${f}" + # TODO(sambhav-jain-16): Change the bucket after new buckets are created + bucket="${ROACHTEST_BUCKET:-cockroach-testeng-metrics/omloader/incoming/${CLOUD}}" + fi + + if [[ "${CLOUD}" == "gce" && "${EXPORT_OPENMETRICS}" == "false" ]]; then + # GCE, having been there first, gets an exemption. + bucket="cockroach-nightly" + fi + + branch=$(tc_build_branch) + remote_artifacts_dir="artifacts-${branch}" + if [[ "${branch}" == "master" ]]; then + # The master branch is special, as roachperf hard-codes + # the location. + remote_artifacts_dir="artifacts" + fi + # TODO: FIPS_ENABLED is deprecated, use roachtest --metamorphic-fips-probability, instead. + # In FIPS-mode, keep artifacts separate by using the 'fips' suffix. + if [[ ${FIPS_ENABLED:-0} == 1 ]]; then + remote_artifacts_dir="${remote_artifacts_dir}-fips" + fi + + # If using openmetrics, activate new service account for uploading to openmetrics bucket + if [[ "${EXPORT_OPENMETRICS}" == "true" && "$ROACHPERF_OPENMETRICS_CREDENTIALS" ]]; then + echo "$ROACHPERF_OPENMETRICS_CREDENTIALS" > roachperf.json + gcloud auth activate-service-account --key-file=roachperf.json + fi + + # The ${stats_file_name} files need some path translation: + # ${artifacts}/path/to/test/${stats_file_name} + # to + # gs://${bucket}/artifacts/${stats_dir}/path/to/test/${stats_file_name} + # + # `find` below will expand "{}" as ./path/to/test/${stats_file_name}. We need + # to bend over backwards to remove the `./` prefix or gsutil will have + # a `.` folder in ${stats_dir}, which we don't want. + (cd "${artifacts}" && \ + while IFS= read -r f; do + if [[ -n "${f}" ]]; then + artifacts_dir="${remote_artifacts_dir}" + # If 'cpu_arch=xxx' is encoded in the path, use it as suffix to separate artifacts by cpu_arch. + if [[ "${f}" == *"/cpu_arch=arm64/"* ]]; then + artifacts_dir="${artifacts_dir}-arm64" + elif [[ "${f}" == *"/cpu_arch=fips/"* ]]; then + artifacts_dir="${artifacts_dir}-fips" fi - done <<< "$(find . -name stats.json | sed 's/^\.\///')") + gsutil cp "${f}" "gs://${bucket}/${artifacts_dir}/${stats_dir}/${f}" + fi + done <<< "$(find . -name ${stats_file_name} | sed 's/^\.\///')") fi } @@ -92,7 +112,7 @@ function upload_all { upload_binaries } -# Upload any stats.json we can find, and some binaries, no matter what happens. +# Upload any ${stats_file_name} we can find, and some binaries, no matter what happens. trap upload_all EXIT # Set up the parameters for the roachtest invocation.