From e91584a036fb35ad2f359ad67c24290181dfcd68 Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Tue, 12 Jul 2022 12:51:23 -0400 Subject: [PATCH] ci: upload binaries to GCS Previously, by default CI uploaded the cockroachdb binaries to S3 only. This commit adds all necessary variables and CLI arguments to upload the binaries to GCS as well. Fixes: * Removed the unset `BUCKET` environment variable to simplify the logic. * Close the GCS object at the end in order to write the binaries for reals. * Remove unnecessary double slash and reflect the change in the tests. Release note: None --- .../post-merge/publish-bleeding-edge.sh | 13 +++++-- .../make-and-publish-build-artifacts.sh | 19 ++++++++--- .../process/publish-cockroach-release.sh | 22 ++++++++---- pkg/cmd/publish-artifacts/main_test.go | 34 +++++++++---------- .../main_test.go | 32 ++++++++--------- pkg/release/gcs.go | 20 ++++++----- pkg/release/upload.go | 11 +++--- 7 files changed, 93 insertions(+), 58 deletions(-) diff --git a/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh b/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh index 1278f0dfe15a..8315eaf5eebf 100755 --- a/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh +++ b/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh @@ -9,8 +9,17 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" source "$dir/teamcity-bazel-support.sh" -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH" run_bazel << 'EOF' +# s3 pushes to the "cockroach" bucket. There is no test/dev bucket fir this build type. +gcs_bucket="cockroach-edge-artifacts-prod" +# export the variable to avoid shell escaping +export gcs_credentials="$GCS_CREDENTIALS_PROD" + +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF' bazel build --config ci //pkg/cmd/publish-artifacts BAZEL_BIN=$(bazel info bazel-bin --config ci) -$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts +export google_credentials="$gcs_credentials" +source "build/teamcity-support.sh" # For log_into_gcloud +log_into_gcloud +export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json" +$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket" EOF diff --git a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh index 52d07fd9c563..29ffeed9e577 100755 --- a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh +++ b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh @@ -15,17 +15,23 @@ release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")" is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")" if [[ -z "${DRY_RUN}" ]] ; then - bucket="${BUCKET-cockroach-builds}" + bucket="cockroach-builds" + gcs_bucket="cockroach-builds-artifacts-prod" google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach" # Used for docker login for gcloud gcr_hostname="us-docker.pkg.dev" + # export the variable to avoid shell escaping + export gcs_credentials="$GCS_CREDENTIALS_PROD" else - bucket="${BUCKET:-cockroach-builds-test}" + bucket="cockroach-builds-test" + gcs_bucket="cockroach-builds-artifacts-dryrun" google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS" gcr_repository="us.gcr.io/cockroach-release/cockroach-test" build_name="${build_name}.dryrun" gcr_hostname="us.gcr.io" + # export the variable to avoid shell escaping + export gcs_credentials="$GCS_CREDENTIALS_DEV" fi cat << EOF @@ -34,6 +40,7 @@ cat << EOF release_branch: $release_branch is_custom_build: $is_custom_build bucket: $bucket + gcs_bucket: $gcs_bucket gcr_repository: $gcr_repository EOF @@ -47,10 +54,14 @@ git tag "${build_name}" tc_end_block "Tag the release" tc_start_block "Compile and publish S3 artifacts" -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF' +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF' bazel build --config ci //pkg/cmd/publish-provisional-artifacts BAZEL_BIN=$(bazel info bazel-bin --config ci) -$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" +export google_credentials="$gcs_credentials" +source "build/teamcity-support.sh" # For log_into_gcloud +log_into_gcloud +export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json" +$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket" EOF tc_end_block "Compile and publish S3 artifacts" diff --git a/build/teamcity/internal/release/process/publish-cockroach-release.sh b/build/teamcity/internal/release/process/publish-cockroach-release.sh index 21d0ce068a46..a7c8305d2cff 100755 --- a/build/teamcity/internal/release/process/publish-cockroach-release.sh +++ b/build/teamcity/internal/release/process/publish-cockroach-release.sh @@ -24,8 +24,11 @@ fi release_branch=$(echo ${build_name} | grep -E -o '^v[0-9]+\.[0-9]+') if [[ -z "${DRY_RUN}" ]] ; then - bucket="${BUCKET:-binaries.cockroachdb.com}" + bucket="binaries.cockroachdb.com" + gcs_bucket="cockroach-release-artifacts-prod" google_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS" + # export the variable to avoid shell escaping + export gcs_credentials="$GCS_CREDENTIALS_PROD" if [[ -z "${PRE_RELEASE}" ]] ; then dockerhub_repository="docker.io/cockroachdb/cockroach" else @@ -37,8 +40,11 @@ if [[ -z "${DRY_RUN}" ]] ; then s3_download_hostname="${bucket}" git_repo_for_tag="cockroachdb/cockroach" else - bucket="${BUCKET:-cockroach-builds-test}" + bucket="cockroach-builds-test" + gcs_bucket="cockroach-release-artifacts-dryrun" google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS" + # export the variable to avoid shell escaping + export gcs_credentials="$GCS_CREDENTIALS_DEV" dockerhub_repository="docker.io/cockroachdb/cockroach-misc" gcr_repository="us.gcr.io/cockroach-release/cockroach-test" gcr_hostname="us.gcr.io" @@ -77,10 +83,14 @@ tc_end_block "Tag the release" tc_start_block "Make and publish release S3 artifacts" # Using publish-provisional-artifacts here is funky. We're directly publishing # the official binaries, not provisional ones. Legacy naming. To clean up... -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF' +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF' bazel build --config ci //pkg/cmd/publish-provisional-artifacts BAZEL_BIN=$(bazel info bazel-bin --config ci) -$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" +export google_credentials="$gcs_credentials" +source "build/teamcity-support.sh" # For log_into_gcloud +log_into_gcloud +export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json" +$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket" EOF tc_end_block "Make and publish release S3 artifacts" @@ -127,10 +137,10 @@ tc_start_block "Publish S3 binaries and archive as latest" # Only push the "latest" for our most recent release branch. # https://github.com/cockroachdb/cockroach/issues/41067 if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then - BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF' + BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket -e gcs_credentials -e gcs_bucket" run_bazel << 'EOF' bazel build --config ci //pkg/cmd/publish-provisional-artifacts BAZEL_BIN=$(bazel info bazel-bin --config ci) -$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" +$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" --gcs-bucket="$gcs_bucket" EOF else diff --git a/pkg/cmd/publish-artifacts/main_test.go b/pkg/cmd/publish-artifacts/main_test.go index b396179647fe..bd2a2813dfd2 100644 --- a/pkg/cmd/publish-artifacts/main_test.go +++ b/pkg/cmd/publish-artifacts/main_test.go @@ -170,87 +170,87 @@ func TestPublish(t *testing.T) { "env=[] args=bazel info bazel-bin -c opt --config=crosslinux --config=ci", }, expectedPuts: []string{ - "s3://cockroach//cockroach/cockroach.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/cockroach.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.linux-gnu-amd64.1234567890abcdef", - "s3://cockroach//cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-amd64.1234567890abcdef", - "s3://cockroach//cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-amd64.1234567890abcdef.so", - "s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-amd64.1234567890abcdef.so", - "s3://cockroach//cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.LATEST/no-cache REDIRECT /cockroach/cockroach.linux-gnu-aarch64.1234567890abcdef", - "s3://cockroach//cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-aarch64.1234567890abcdef", - "s3://cockroach//cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-aarch64.1234567890abcdef.so", - "s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-aarch64.1234567890abcdef.so", - "s3://cockroach//cockroach/cockroach.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/cockroach.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.darwin-amd64.1234567890abcdef", - "s3://cockroach//cockroach/cockroach-sql.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach-sql.darwin-amd64.1234567890abcdef CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/cockroach-sql.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.darwin-amd64.1234567890abcdef", - "s3://cockroach//cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/lib/libgeos.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.darwin-amd64.1234567890abcdef.dylib", - "s3://cockroach//cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.darwin-amd64.1234567890abcdef.dylib", - "s3://cockroach//cockroach/cockroach.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/cockroach.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach.windows-amd64.1234567890abcdef.exe", - "s3://cockroach//cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/cockroach-sql.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.windows-amd64.1234567890abcdef.exe", - "s3://cockroach//cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/lib/libgeos.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.windows-amd64.1234567890abcdef.dll", - "s3://cockroach//cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " + + "s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll CONTENTS env=[] args=bazel build " + "//pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' " + "-c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.windows-amd64.1234567890abcdef.dll", - "s3://cockroach//cockroach/workload.1234567890abcdef CONTENTS env=[] args=bazel build //pkg/cmd/workload -c opt --config=crosslinux --config=ci", + "s3://cockroach/cockroach/workload.1234567890abcdef CONTENTS env=[] args=bazel build //pkg/cmd/workload -c opt --config=crosslinux --config=ci", "s3://cockroach/cockroach/workload.LATEST/no-cache REDIRECT /cockroach/workload.1234567890abcdef", }, }, diff --git a/pkg/cmd/publish-provisional-artifacts/main_test.go b/pkg/cmd/publish-provisional-artifacts/main_test.go index bfef66726cfb..966be18427cb 100644 --- a/pkg/cmd/publish-provisional-artifacts/main_test.go +++ b/pkg/cmd/publish-provisional-artifacts/main_test.go @@ -219,68 +219,68 @@ func TestProvisional(t *testing.T) { }, expectedGets: nil, expectedPuts: []string{ - "s3://cockroach//cockroach/cockroach.linux-gnu-amd64.00SHA00 " + + "s3://cockroach/cockroach/cockroach.linux-gnu-amd64.00SHA00 " + "CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp." + "sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/cockroach.linux-gnu-amd64.LATEST/no-cache " + "REDIRECT /cockroach/cockroach.linux-gnu-amd64.00SHA00", - "s3://cockroach//cockroach/cockroach-sql.linux-gnu-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", + "s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-amd64.00SHA00", - "s3://cockroach//cockroach/lib/libgeos.linux-gnu-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.00SHA00." + "so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-amd64.00SHA00.so", - "s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00." + "so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase", "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00.so", - "s3://cockroach//cockroach/cockroach.darwin-amd64.00SHA00 " + + "s3://cockroach/cockroach/cockroach.darwin-amd64.00SHA00 " + "CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/cockroach.darwin-amd64.LATEST/no-cache " + "REDIRECT /cockroach/cockroach.darwin-amd64.00SHA00", - "s3://cockroach//cockroach/cockroach-sql.darwin-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", + "s3://cockroach/cockroach/cockroach-sql.darwin-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/cockroach-sql.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql." + "darwin-amd64.00SHA00", - "s3://cockroach//cockroach/lib/libgeos.darwin-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos.darwin-amd64.00SHA00." + "dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/lib/libgeos.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.darwin-amd64.00SHA00.dylib", - "s3://cockroach//cockroach/lib/libgeos_c.darwin-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.00SHA00." + "dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp." + "sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase", "s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.darwin-amd64.00SHA00.dylib", - "s3://cockroach//cockroach/cockroach.windows-amd64.00SHA00.exe " + + "s3://cockroach/cockroach/cockroach.windows-amd64.00SHA00.exe " + "CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp." + "sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/cockroach.windows-amd64.LATEST/no-cache " + "REDIRECT /cockroach/cockroach.windows-amd64.00SHA00.exe", - "s3://cockroach//cockroach/cockroach-sql.windows-amd64.00SHA00.exe CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", + "s3://cockroach/cockroach/cockroach-sql.windows-amd64.00SHA00.exe CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/cockroach-sql.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.windows-amd64.00SHA00.exe", - "s3://cockroach//cockroach/lib/libgeos.windows-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos.windows-amd64.00SHA00." + "dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/lib/libgeos.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.windows-amd64.00SHA00.dll", - "s3://cockroach//cockroach/lib/libgeos_c.windows-amd64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.00SHA00." + "dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase", "s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.windows-amd64.00SHA00.dll", - "s3://cockroach//cockroach/cockroach.linux-gnu-aarch64.00SHA00 " + + "s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.00SHA00 " + "CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp." + "sh aarch64-unknown-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/cockroach.linux-gnu-aarch64.LATEST/no-cache " + "REDIRECT /cockroach/cockroach.linux-gnu-aarch64.00SHA00", - "s3://cockroach//cockroach/cockroach-sql.linux-gnu-aarch64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", + "s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/cockroach-sql.linux-gnu-aarch64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-aarch64.00SHA00", - "s3://cockroach//cockroach/lib/libgeos.linux-gnu-aarch64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.00SHA00." + "so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/lib/libgeos.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-aarch64.00SHA00.so", - "s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-aarch64.00SHA00." + + "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.00SHA00." + "so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " + "'--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase", "s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-aarch64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-aarch64.00SHA00.so", diff --git a/pkg/release/gcs.go b/pkg/release/gcs.go index 6d1c150ce22a..c087d47c5dc8 100644 --- a/pkg/release/gcs.go +++ b/pkg/release/gcs.go @@ -64,25 +64,29 @@ func (p *GCSProvider) PutObject(input *PutObjectInput) error { var body []byte if input.WebsiteRedirectLocation != nil { // Seems like Google storage doesn't support this. Copy the original object. - // copy content - r, err := p.client.Bucket(p.bucket).Object(*input.WebsiteRedirectLocation).NewReader(ctx) + // copy content. Strip the leading slash to normalize the object name. + copyFrom := strings.TrimPrefix(*input.WebsiteRedirectLocation, "/") + r, err := p.client.Bucket(p.bucket).Object(copyFrom).NewReader(ctx) if err != nil { - return fmt.Errorf("cannot read %s: %w", *input.WebsiteRedirectLocation, err) + return fmt.Errorf("cannot read %s: %w", copyFrom, err) } body, err = ioutil.ReadAll(r) if err != nil { - return fmt.Errorf("cannot download %s: %w", *input.WebsiteRedirectLocation, err) + return fmt.Errorf("cannot download %s: %w", copyFrom, err) } } else { var err error body, err = ioutil.ReadAll(input.Body) if err != nil { - return fmt.Errorf("cannot download %s: %w", *input.WebsiteRedirectLocation, err) + return fmt.Errorf("cannot read content: %w", err) } } - w := obj.NewWriter(ctx) - if _, err := w.Write(body); err != nil { - return fmt.Errorf("error in GCS upload %s: %w", *input.Key, err) + wc := obj.NewWriter(ctx) + if _, err := wc.Write(body); err != nil { + return fmt.Errorf("error writing to GCS object %s: %w", *input.Key, err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("error closing GCS object %s: %w", *input.Key, err) } attrs := storage.ObjectAttrsToUpdate{} updateAttrs := false diff --git a/pkg/release/upload.go b/pkg/release/upload.go index 923a41969ec1..f665efd487e1 100644 --- a/pkg/release/upload.go +++ b/pkg/release/upload.go @@ -206,9 +206,7 @@ func PutNonRelease(svc ObjectPutGetter, o PutNonReleaseOptions) { _ = fileToUpload.Close() }() - // NB: The leading slash is required to make redirects work - // correctly since we reuse this key as the redirect location. - versionKey := fmt.Sprintf("/%s/%s", nonReleasePrefix, f.FilePath) + versionKey := fmt.Sprintf("%s/%s", nonReleasePrefix, f.FilePath) log.Printf("Uploading to %s", svc.URL(versionKey)) if err := svc.PutObject(&PutObjectInput{ ContentDisposition: &disposition, @@ -223,12 +221,15 @@ func PutNonRelease(svc ObjectPutGetter, o PutNonReleaseOptions) { latestSuffix = "LATEST" } latestKey := fmt.Sprintf("%s/%s.%s", nonReleasePrefix, f.RedirectPathPrefix, latestSuffix) + // NB: The leading slash is required to make redirects work + // correctly since we reuse this key as the redirect location. + target := "/" + versionKey if err := svc.PutObject(&PutObjectInput{ CacheControl: &NoCache, Key: &latestKey, - WebsiteRedirectLocation: &versionKey, + WebsiteRedirectLocation: &target, }); err != nil { - log.Fatalf("failed adding a redirect to %s: %s", versionKey, err) + log.Fatalf("failed adding a redirect to %s: %s", target, err) } } }