From d1b95ced5669acd686b974e341c9e678c301499f Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Thu, 18 May 2023 10:21:41 -0400 Subject: [PATCH] ci: separate location for custom builds Previously, we published custom builds to the same bucket and docker repository as the release-* and master builds. This PR uses separate GCS bucket and GAR repository for builds other than on master and release-* branches. It also removes unused dockerhub login call. Additionally, customized builds do not push the tag to the `release-staging` repo and do not consume release verification testing resources. Fixes: DEVINF-590 Epic: None Release note: None --- .../make-and-publish-build-artifacts.sh | 47 +++++++---- .../process/make-and-publish-build-tagging.sh | 77 ++++++++++++------- scripts/tag-custom-build.sh | 12 ++- 3 files changed, 92 insertions(+), 44 deletions(-) diff --git a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh index b350f13ba269..d37b895a1def 100755 --- a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh +++ b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh @@ -12,16 +12,26 @@ build_name=$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev # On no match, `grep -Eo` returns 1. `|| echo""` makes the script not error. release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")" -is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")" +is_customized_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")" +is_release_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^(release-[0-9][0-9]\.[0-9](\.0)?)$|master$" || echo "")" if [[ -z "${DRY_RUN}" ]] ; then - gcs_bucket="cockroach-builds-artifacts-prod" - google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS - gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach" - # Used for docker login for gcloud - gcr_hostname="us-docker.pkg.dev" - # export the variable to avoid shell escaping - export gcs_credentials="$GCS_CREDENTIALS_PROD" + if [[ -z "${is_release_build}" ]] ; then + # export the variable to avoid shell escaping + export gcs_credentials="$GOOGLE_CREDENTIALS_CUSTOMIZED" + google_credentials=$GOOGLE_CREDENTIALS_CUSTOMIZED + gcs_bucket="cockroach-customized-builds-artifacts-prod" + gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb-customized/cockroach-customized" + gcr_hostname="us-docker.pkg.dev" + else + gcs_bucket="cockroach-builds-artifacts-prod" + google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS + gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach" + # Used for docker login for gcloud + gcr_hostname="us-docker.pkg.dev" + # export the variable to avoid shell escaping + export gcs_credentials="$GCS_CREDENTIALS_PROD" + fi else gcs_bucket="cockroach-builds-artifacts-dryrun" google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS" @@ -31,15 +41,15 @@ else # export the variable to avoid shell escaping export gcs_credentials="$GCS_CREDENTIALS_DEV" fi -download_prefix="https://storage.googleapis.com/$gcs_bucket" cat << EOF - build_name: $build_name - release_branch: $release_branch - is_custom_build: $is_custom_build - gcs_bucket: $gcs_bucket - gcr_repository: $gcr_repository + build_name: $build_name + release_branch: $release_branch + is_customized_build: $is_customized_build + gcs_bucket: $gcs_bucket + gcr_repository: $gcr_repository + is_release_build: $is_release_build EOF tc_end_block "Variable Setup" @@ -64,7 +74,6 @@ EOF tc_end_block "Compile and publish artifacts" tc_start_block "Make and push multiarch docker images" -configure_docker_creds docker_login_with_google gcr_tag="${gcr_repository}:${build_name}" @@ -125,5 +134,13 @@ cat << EOF Build ID: ${build_name} +The binaries are available at: + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.linux-amd64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.linux-arm64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.darwin-10.9-amd64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.windows-6.2-amd64.zip + +Pull the docker image by: + docker pull $gcr_repository:$build_name EOF diff --git a/build/teamcity/internal/release/process/make-and-publish-build-tagging.sh b/build/teamcity/internal/release/process/make-and-publish-build-tagging.sh index 7fcceba22366..1eebb10a6eea 100755 --- a/build/teamcity/internal/release/process/make-and-publish-build-tagging.sh +++ b/build/teamcity/internal/release/process/make-and-publish-build-tagging.sh @@ -12,14 +12,23 @@ build_name=$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev # On no match, `grep -Eo` returns 1. `|| echo""` makes the script not error. release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")" -is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")" +is_release_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^(release-[0-9][0-9]\.[0-9](\.0)?)$" || echo "")" +is_customized_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")" +github_ssh_key="${GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY}" if [[ -z "${DRY_RUN}" ]] ; then - google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS - gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach" - # Used for docker login for gcloud - gcr_hostname="us-docker.pkg.dev" + if [[ -z "${is_customized_build}" ]] ; then + google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS + gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach" + # Used for docker login for gcloud + gcr_hostname="us-docker.pkg.dev" + gcs_bucket="cockroach-builds-artifacts-prod" + else + gcs_bucket="cockroach-customized-builds-artifacts-prod" + gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb-customized/cockroach-customized" + fi else + gcs_bucket="cockroach-builds-artifacts-dryrun" google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS" gcr_repository="us.gcr.io/cockroach-release/cockroach-test" build_name="${build_name}.dryrun" @@ -28,35 +37,39 @@ fi cat << EOF - build_name: $build_name - release_branch: $release_branch - is_custom_build: $is_custom_build - gcr_repository: $gcr_repository + build_name: $build_name + release_branch: $release_branch + is_customized_build: $is_customized_build + gcr_repository: $gcr_repository EOF tc_end_block "Variable Setup" - -tc_start_block "Tag the release" -git tag "${build_name}" -tc_end_block "Tag the release" - -tc_start_block "Push release tag to github.com/cockroachlabs/release-staging" -github_ssh_key="${GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY}" configure_git_ssh_key -git_wrapped push ssh://git@github.com/cockroachlabs/release-staging.git "${build_name}" -tc_end_block "Push release tag to github.com/cockroachlabs/release-staging" - -tc_start_block "Tag docker image as latest-build" -# Only tag the image as "latest-vX.Y-build" if the tag is on a release branch -# (or master for the alphas for the next major release). -if [[ -n "${release_branch}" ]] ; then - log_into_gcloud - gcloud container images add-tag "${gcr_repository}:${build_name}" "${gcr_repository}:latest-${release_branch}-build" -fi +if [[ -z "${is_customized_build}" ]] ; then + tc_start_block "Tag the release" + git tag "${build_name}" + tc_end_block "Tag the release" + + tc_start_block "Push release tag to github.com/cockroachlabs/release-staging" + git_wrapped push ssh://git@github.com/cockroachlabs/release-staging.git "${build_name}" + tc_end_block "Push release tag to github.com/cockroachlabs/release-staging" + + + tc_start_block "Tag docker image as latest-build" + # Only tag the image as "latest-vX.Y-build" if the tag is on a release branch + # (or master for the alphas for the next major release). + if [[ -n "${is_release_build}" ]] ; then + log_into_gcloud + gcloud container images add-tag "${gcr_repository}:${build_name}" "${gcr_repository}:latest-${release_branch}-build" + fi + if [[ "$TC_BUILD_BRANCH" == "master" ]]; then + log_into_gcloud + gcloud container images add-tag "${gcr_repository}:${build_name}" "${gcr_repository}:latest-master-build" + fi tc_end_block "Tag docker image as latest-build" - +fi # Make finding the tag name easy. cat << EOF @@ -64,11 +77,19 @@ cat << EOF Build ID: ${build_name} +The binaries are available at: + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.linux-amd64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.linux-arm64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.darwin-10.9-amd64.tgz + https://storage.googleapis.com/$gcs_bucket/cockroach-$build_name.windows-6.2-amd64.zip + +Pull the docker image by: + docker pull $gcr_repository:$build_name EOF -if [[ -n "${is_custom_build}" ]] ; then +if [[ -n "${is_customized_build}" ]] ; then tc_start_block "Delete custombuild tag" git_wrapped push ssh://git@github.com/cockroachdb/cockroach.git --delete "${TC_BUILD_BRANCH}" tc_end_block "Delete custombuild tag" diff --git a/scripts/tag-custom-build.sh b/scripts/tag-custom-build.sh index 43e8792db413..c886c8b8bd27 100755 --- a/scripts/tag-custom-build.sh +++ b/scripts/tag-custom-build.sh @@ -50,7 +50,7 @@ git tag "$TAG" "$SHA" git push git@github.com:cockroachdb/cockroach.git "$TAG" TAG_URL="https://github.com/cockroachdb/cockroach/releases/tag/${TAG}" -TEAMCITY_URL="https://teamcity.cockroachdb.com/buildConfiguration/Internal_Release_Process_TestingMakeAndPublishBuildV221?mode=builds&branch=${TAG}" +TEAMCITY_URL="https://teamcity.cockroachdb.com/buildConfiguration/Internal_Cockroach_Release_Customized_MakeAndPublishCustomizedBuildArtifacts?mode=builds&branch=${TAG}" if [ "$(command -v open)" ] ; then open "$TEAMCITY_URL" open "$TAG_URL" @@ -75,4 +75,14 @@ Here is where the build run should show up in TeamCity for the tag: Tag name: $TAG Build ID: $ID + +The binaries will be available at: + https://storage.googleapis.com/cockroach-customized-builds-artifacts-prod/cockroach-$ID.linux-amd64.tgz + https://storage.googleapis.com/cockroach-customized-builds-artifacts-prod/cockroach-$ID.linux-arm64.tgz + https://storage.googleapis.com/cockroach-customized-builds-artifacts-prod/cockroach-$ID.darwin-10.9-amd64.tgz + https://storage.googleapis.com/cockroach-customized-builds-artifacts-prod/cockroach-$ID.windows-6.2-amd64.zip + +Pull the docker image by: + docker pull us-docker.pkg.dev/cockroach-cloud-images/cockroachdb-customized/cockroach-customized:$ID + EOF