Skip to content

Commit

Permalink
ci: upload binaries to GCS
Browse files Browse the repository at this point in the history
Previously, by default CI uploaded the cockroachdb binaries to S3 only.

This commit adds all necessary variables and CLI arguments to upload the
binaries to GCS as well.

Fixes:
* Removed the unset `BUCKET` environment variable to simplify the logic.
* Close the GCS object at the end in order to write the binaries for
  reals.
* Remove unnecessary double slash and reflect the change in the tests.

Release note: None
  • Loading branch information
rail committed Aug 10, 2022
1 parent 6e0b2f2 commit e91584a
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 58 deletions.
13 changes: 11 additions & 2 deletions build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,17 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh"
source "$dir/teamcity-bazel-support.sh"

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH" run_bazel << 'EOF'
# s3 pushes to the "cockroach" bucket. There is no test/dev bucket fir this build type.
gcs_bucket="cockroach-edge-artifacts-prod"
# export the variable to avoid shell escaping
export gcs_credentials="$GCS_CREDENTIALS_PROD"

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
bazel build --config ci //pkg/cmd/publish-artifacts
BAZEL_BIN=$(bazel info bazel-bin --config ci)
$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts
export google_credentials="$gcs_credentials"
source "build/teamcity-support.sh" # For log_into_gcloud
log_into_gcloud
export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket"
EOF
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,23 @@ release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")"
is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")"

if [[ -z "${DRY_RUN}" ]] ; then
bucket="${BUCKET-cockroach-builds}"
bucket="cockroach-builds"
gcs_bucket="cockroach-builds-artifacts-prod"
google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS
gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach"
# Used for docker login for gcloud
gcr_hostname="us-docker.pkg.dev"
# export the variable to avoid shell escaping
export gcs_credentials="$GCS_CREDENTIALS_PROD"
else
bucket="${BUCKET:-cockroach-builds-test}"
bucket="cockroach-builds-test"
gcs_bucket="cockroach-builds-artifacts-dryrun"
google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
build_name="${build_name}.dryrun"
gcr_hostname="us.gcr.io"
# export the variable to avoid shell escaping
export gcs_credentials="$GCS_CREDENTIALS_DEV"
fi

cat << EOF
Expand All @@ -34,6 +40,7 @@ cat << EOF
release_branch: $release_branch
is_custom_build: $is_custom_build
bucket: $bucket
gcs_bucket: $gcs_bucket
gcr_repository: $gcr_repository
EOF
Expand All @@ -47,10 +54,14 @@ git tag "${build_name}"
tc_end_block "Tag the release"

tc_start_block "Compile and publish S3 artifacts"
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
bazel build --config ci //pkg/cmd/publish-provisional-artifacts
BAZEL_BIN=$(bazel info bazel-bin --config ci)
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
export google_credentials="$gcs_credentials"
source "build/teamcity-support.sh" # For log_into_gcloud
log_into_gcloud
export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
EOF
tc_end_block "Compile and publish S3 artifacts"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,11 @@ fi
release_branch=$(echo ${build_name} | grep -E -o '^v[0-9]+\.[0-9]+')

if [[ -z "${DRY_RUN}" ]] ; then
bucket="${BUCKET:-binaries.cockroachdb.com}"
bucket="binaries.cockroachdb.com"
gcs_bucket="cockroach-release-artifacts-prod"
google_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS"
# export the variable to avoid shell escaping
export gcs_credentials="$GCS_CREDENTIALS_PROD"
if [[ -z "${PRE_RELEASE}" ]] ; then
dockerhub_repository="docker.io/cockroachdb/cockroach"
else
Expand All @@ -37,8 +40,11 @@ if [[ -z "${DRY_RUN}" ]] ; then
s3_download_hostname="${bucket}"
git_repo_for_tag="cockroachdb/cockroach"
else
bucket="${BUCKET:-cockroach-builds-test}"
bucket="cockroach-builds-test"
gcs_bucket="cockroach-release-artifacts-dryrun"
google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
# export the variable to avoid shell escaping
export gcs_credentials="$GCS_CREDENTIALS_DEV"
dockerhub_repository="docker.io/cockroachdb/cockroach-misc"
gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
gcr_hostname="us.gcr.io"
Expand Down Expand Up @@ -77,10 +83,14 @@ tc_end_block "Tag the release"
tc_start_block "Make and publish release S3 artifacts"
# Using publish-provisional-artifacts here is funky. We're directly publishing
# the official binaries, not provisional ones. Legacy naming. To clean up...
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
bazel build --config ci //pkg/cmd/publish-provisional-artifacts
BAZEL_BIN=$(bazel info bazel-bin --config ci)
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
export google_credentials="$gcs_credentials"
source "build/teamcity-support.sh" # For log_into_gcloud
log_into_gcloud
export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
EOF
tc_end_block "Make and publish release S3 artifacts"

Expand Down Expand Up @@ -127,10 +137,10 @@ tc_start_block "Publish S3 binaries and archive as latest"
# Only push the "latest" for our most recent release branch.
# https://github.com/cockroachdb/cockroach/issues/41067
if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket -e gcs_credentials -e gcs_bucket" run_bazel << 'EOF'
bazel build --config ci //pkg/cmd/publish-provisional-artifacts
BAZEL_BIN=$(bazel info bazel-bin --config ci)
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket"
$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
EOF

else
Expand Down
34 changes: 17 additions & 17 deletions pkg/cmd/publish-artifacts/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,87 +170,87 @@ func TestPublish(t *testing.T) {
"env=[] args=bazel info bazel-bin -c opt --config=crosslinux --config=ci",
},
expectedPuts: []string{
"s3://cockroach//cockroach/cockroach.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/cockroach.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.linux-gnu-amd64.1234567890abcdef",
"s3://cockroach//cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef",
"s3://cockroach//cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so",
"s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so",
"s3://cockroach//cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
"s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.LATEST/no-cache REDIRECT /cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef",
"s3://cockroach//cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
"s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef",
"s3://cockroach//cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
"s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so",
"s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
"s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so",
"s3://cockroach//cockroach/cockroach.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/cockroach.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.darwin-amd64.1234567890abcdef",
"s3://cockroach//cockroach/cockroach-sql.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach-sql.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/cockroach-sql.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.darwin-amd64.1234567890abcdef",
"s3://cockroach//cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/lib/libgeos.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib",
"s3://cockroach//cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib",
"s3://cockroach//cockroach/cockroach.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/cockroach.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.windows-amd64.1234567890abcdef.exe",
"s3://cockroach//cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/cockroach-sql.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe",
"s3://cockroach//cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/lib/libgeos.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll",
"s3://cockroach//cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " +
"s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " +
"//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " +
"-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll",
"s3://cockroach//cockroach/workload.1234567890abcdef CONTENTS env=[] args=bazel build //pkg/cmd/workload -c opt --config=crosslinux --config=ci",
"s3://cockroach/cockroach/workload.1234567890abcdef CONTENTS env=[] args=bazel build //pkg/cmd/workload -c opt --config=crosslinux --config=ci",
"s3://cockroach/cockroach/workload.LATEST/no-cache REDIRECT /cockroach/workload.1234567890abcdef",
},
},
Expand Down
Loading

0 comments on commit e91584a

Please sign in to comment.