ci,release: backport upload to GCS

Previously, we started uploading build and release artifacts to GCS in parallel with S3 on the master branch (22.2). This PR backports the functionality to the 22.1 branch. Related: #83138 Related: #84597 Release note: None
cockroachdb · Aug 23, 2022 · b5d145e · b5d145e
1 parent 7fb3ee0
commit b5d145e
Show file tree

Hide file tree

Showing 16 changed files with 1,335 additions and 1,034 deletions.
diff --git a/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh b/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh
@@ -9,8 +9,17 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"
 source "$dir/teamcity-bazel-support.sh"
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH" run_bazel << 'EOF'
+# s3 pushes to the "cockroach" bucket. There is no test/dev bucket fir this build type.
+gcs_bucket="cockroach-edge-artifacts-prod"
+# export the variable to avoid shell escaping
+export gcs_credentials="$GCS_CREDENTIALS_PROD"
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
-$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts
+export google_credentials="$gcs_credentials"
+source "build/teamcity-support.sh"  # For log_into_gcloud
+log_into_gcloud
+export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
+$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket"
 EOF
diff --git a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh
@@ -15,17 +15,23 @@ release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")"
 is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")"
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="${BUCKET-cockroach-builds}"
+  bucket="cockroach-builds"
+  gcs_bucket="cockroach-builds-artifacts-prod"
   google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS
   gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach"
   # Used for docker login for gcloud
   gcr_hostname="us-docker.pkg.dev"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_PROD"
 else
-  bucket="${BUCKET:-cockroach-builds-test}"
+  bucket="cockroach-builds-test"
+  gcs_bucket="cockroach-builds-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
   gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
   build_name="${build_name}.dryrun"
   gcr_hostname="us.gcr.io"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_DEV"
 fi
 
 cat << EOF
@@ -34,6 +40,7 @@ cat << EOF
   release_branch:  $release_branch
   is_custom_build: $is_custom_build
   bucket:          $bucket
+  gcs_bucket:      $gcs_bucket
   gcr_repository:  $gcr_repository
 
 EOF
@@ -47,10 +54,14 @@ git tag "${build_name}"
 tc_end_block "Tag the release"
 
 tc_start_block "Compile and publish S3 artifacts"
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
+export google_credentials="$gcs_credentials"
+source "build/teamcity-support.sh"  # For log_into_gcloud
+log_into_gcloud
+export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
 EOF
 tc_end_block "Compile and publish S3 artifacts"
 

diff --git a/build/teamcity/internal/release/process/publish-cockroach-release.sh b/build/teamcity/internal/release/process/publish-cockroach-release.sh
@@ -24,8 +24,11 @@ fi
 release_branch=$(echo ${build_name} | grep -E -o '^v[0-9]+\.[0-9]+')
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="${BUCKET:-binaries.cockroachdb.com}"
+  bucket="binaries.cockroachdb.com"
+  gcs_bucket="cockroach-release-artifacts-prod"
   google_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_PROD"
   if [[ -z "${PRE_RELEASE}" ]] ; then
     dockerhub_repository="docker.io/cockroachdb/cockroach"
   else
@@ -37,8 +40,11 @@ if [[ -z "${DRY_RUN}" ]] ; then
   s3_download_hostname="${bucket}"
   git_repo_for_tag="cockroachdb/cockroach"
 else
-  bucket="${BUCKET:-cockroach-builds-test}"
+  bucket="cockroach-builds-test"
+  gcs_bucket="cockroach-release-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_DEV"
   dockerhub_repository="docker.io/cockroachdb/cockroach-misc"
   gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
   gcr_hostname="us.gcr.io"
@@ -77,10 +83,14 @@ tc_end_block "Tag the release"
 tc_start_block "Make and publish release S3 artifacts"
 # Using publish-provisional-artifacts here is funky. We're directly publishing
 # the official binaries, not provisional ones. Legacy naming. To clean up...
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
+export google_credentials="$gcs_credentials"
+source "build/teamcity-support.sh"  # For log_into_gcloud
+log_into_gcloud
+export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
 EOF
 tc_end_block "Make and publish release S3 artifacts"
 
@@ -127,10 +137,10 @@ tc_start_block "Publish S3 binaries and archive as latest"
 # Only push the "latest" for our most recent release branch.
 # https://github.com/cockroachdb/cockroach/issues/41067
 if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then
-    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket -e gcs_credentials -e gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
 EOF
 
 else
@@ -153,7 +163,7 @@ tc_start_block "Tag docker image as latest"
 # latest unstable release
 # https://github.com/cockroachdb/cockroach/issues/41067
 # https://github.com/cockroachdb/cockroach/issues/48309
-if [[ -n "${PUBLISH_LATEST}" ]]; then
+if [[ -n "${PUBLISH_LATEST}" || -n "${PRE_RELEASE}" ]]; then
   docker push "${dockerhub_repository}:latest"
 else
   echo "The ${dockerhub_repository}:latest docker image tag was _not_ pushed."
@@ -170,7 +180,7 @@ images=(
 if [[ -z "$PRE_RELEASE" ]]; then
   images+=("${dockerhub_repository}:latest-${release_branch}")
 fi
-if [[ -n "${PUBLISH_LATEST}" ]]; then
+if [[ -n "${PUBLISH_LATEST}" || -n "${PRE_RELEASE}" ]]; then
   images+=("${dockerhub_repository}:latest")
 fi
 

diff --git a/pkg/cmd/publish-artifacts/BUILD.bazel b/pkg/cmd/publish-artifacts/BUILD.bazel
@@ -7,9 +7,6 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//pkg/release",
-        "@com_github_aws_aws_sdk_go//aws",
-        "@com_github_aws_aws_sdk_go//aws/session",
-        "@com_github_aws_aws_sdk_go//service/s3",
         "@com_github_kr_pretty//:pretty",
     ],
 )
@@ -23,14 +20,12 @@ go_binary(
 go_test(
     name = "publish-artifacts_test",
     size = "small",
-    srcs = [
-        "main_test.go",
-        "slow_test.go",
-    ],
+    srcs = ["main_test.go"],
     embed = [":publish-artifacts_lib"],
     deps = [
-        "//pkg/testutils/skip",
-        "@com_github_aws_aws_sdk_go//service/s3",
+        "//pkg/release",
+        "//pkg/testutils",
+        "@com_github_alessio_shellescape//:shellescape",
         "@com_github_stretchr_testify//require",
     ],
 )
diff --git a/pkg/cmd/publish-artifacts/main.go b/pkg/cmd/publish-artifacts/main.go
@@ -18,9 +18,6 @@ import (
 	"os/exec"
 	"path/filepath"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/cockroachdb/cockroach/pkg/release"
 	"github.com/kr/pretty"
 )
@@ -31,24 +28,9 @@ const (
 	teamcityBuildBranchKey = "TC_BUILD_BRANCH"
 )
 
-type s3putter interface {
-	PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error)
-}
-
-// Overridden in testing.
-var testableS3 = func() (s3putter, error) {
-	sess, err := session.NewSession(&aws.Config{
-		Region: aws.String("us-east-1"),
-	})
-	if err != nil {
-		return nil, err
-	}
-	return s3.New(sess), nil
-}
-
-var destBucket = flag.String("bucket", "", "override default bucket")
-
 func main() {
+	var destBucket = flag.String("bucket", "cockroach", "override default bucket")
+	var gcsBucket = flag.String("gcs-bucket", "", "override default bucket")
 	flag.Parse()
 
 	if _, ok := os.LookupEnv(awsAccessKeyIDKey); !ok {
@@ -81,73 +63,109 @@ func main() {
 	}
 	versionStr := string(bytes.TrimSpace(out))
 
-	svc, err := testableS3()
+	var providers []release.ObjectPutGetter
+	s3, err := release.NewS3("us-east-1", *destBucket)
 	if err != nil {
 		log.Fatalf("Creating AWS S3 session: %s", err)
 	}
-
-	var bucketName string
-	if len(*destBucket) > 0 {
-		bucketName = *destBucket
-	} else {
-		bucketName = "cockroach"
+	providers = append(providers, s3)
+
+	if *gcsBucket != "" {
+		if _, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); !ok {
+			log.Fatal("GOOGLE_APPLICATION_CREDENTIALS environment variable is not set")
+		}
+		gcs, err := release.NewGCS(*gcsBucket)
+		if err != nil {
+			log.Fatalf("Creating GCS session: %s", err)
+		}
+		providers = append(providers, gcs)
 	}
-	log.Printf("Using S3 bucket: %s", bucketName)
 
-	releaseVersionStrs := []string{versionStr}
+	run(providers, runFlags{
+		pkgDir: pkg,
+		branch: branch,
+		sha:    versionStr,
+	}, release.ExecFn{})
+}
+
+type runFlags struct {
+	branch string
+	sha    string
+	pkgDir string
+}
 
+func run(providers []release.ObjectPutGetter, flags runFlags, execFn release.ExecFn) {
 	for _, platform := range []release.Platform{release.PlatformLinux, release.PlatformMacOS, release.PlatformWindows} {
 		var o opts
 		o.Platform = platform
-		o.ReleaseVersionStrs = releaseVersionStrs
-		o.PkgDir = pkg
-		o.Branch = branch
-		o.VersionStr = versionStr
-		o.BucketName = bucketName
-		o.Branch = branch
-		o.AbsolutePath = filepath.Join(pkg, "cockroach"+release.SuffixFromPlatform(platform))
+		o.ReleaseVersions = []string{flags.sha}
+		o.PkgDir = flags.pkgDir
+		o.Branch = flags.branch
+		o.VersionStr = flags.sha
+		o.AbsolutePath = filepath.Join(flags.pkgDir, "cockroach"+release.SuffixFromPlatform(platform))
+		o.CockroachSQLAbsolutePath = filepath.Join(flags.pkgDir, "cockroach-sql"+release.SuffixFromPlatform(platform))
 
 		log.Printf("building %s", pretty.Sprint(o))
-
-		buildOneCockroach(svc, o)
+		buildOneCockroach(providers, o, execFn)
 	}
+	// We build workload only for Linux.
+	var o opts
+	o.Platform = release.PlatformLinux
+	o.PkgDir = flags.pkgDir
+	o.Branch = flags.branch
+	o.VersionStr = flags.sha
+	buildAndPublishWorkload(providers, o, execFn)
 }
 
-func buildOneCockroach(svc s3putter, o opts) {
+func buildOneCockroach(providers []release.ObjectPutGetter, o opts, execFn release.ExecFn) {
 	log.Printf("building cockroach %s", pretty.Sprint(o))
-	defer func() {
-		log.Printf("done building cockroach: %s", pretty.Sprint(o))
-	}()
-
-	if err := release.MakeRelease(o.Platform, release.BuildOptions{}, o.PkgDir); err != nil {
+	if err := release.MakeRelease(o.Platform, release.BuildOptions{ExecFn: execFn}, o.PkgDir); err != nil {
 		log.Fatal(err)
 	}
-
-	putNonRelease(svc, o, release.MakeCRDBLibraryNonReleaseFiles(o.PkgDir, o.Platform, o.VersionStr)...)
+	for _, provider := range providers {
+		release.PutNonRelease(
+			provider,
+			release.PutNonReleaseOptions{
+				Branch: o.Branch,
+				Files: append(
+					[]release.NonReleaseFile{
+						release.MakeCRDBBinaryNonReleaseFile(o.AbsolutePath, o.VersionStr),
+						release.MakeCRDBBinaryNonReleaseFile(o.CockroachSQLAbsolutePath, o.VersionStr),
+					},
+					release.MakeCRDBLibraryNonReleaseFiles(o.PkgDir, o.Platform, o.VersionStr)...,
+				),
+			},
+		)
+	}
+	log.Printf("done building cockroach: %s", pretty.Sprint(o))
 }
 
-type opts struct {
-	VersionStr         string
-	Branch             string
-	ReleaseVersionStrs []string
-
-	Platform release.Platform
-
-	BucketName   string
-	AbsolutePath string
-	PkgDir       string
+func buildAndPublishWorkload(providers []release.ObjectPutGetter, o opts, execFn release.ExecFn) {
+	log.Printf("building workload %s", pretty.Sprint(o))
+	if err := release.MakeWorkload(release.BuildOptions{ExecFn: execFn}, o.PkgDir); err != nil {
+		log.Fatal(err)
+	}
+	o.AbsolutePath = filepath.Join(o.PkgDir, "bin", "workload")
+	for _, provider := range providers {
+		release.PutNonRelease(
+			provider,
+			release.PutNonReleaseOptions{
+				Branch: o.Branch,
+				Files: []release.NonReleaseFile{
+					release.MakeCRDBBinaryNonReleaseFile(o.AbsolutePath, o.VersionStr),
+				},
+			},
+		)
+	}
+	log.Printf("done building workload: %s", pretty.Sprint(o))
 }
 
-func putNonRelease(svc s3putter, o opts, additionalNonReleaseFiles ...release.NonReleaseFile) {
-	release.PutNonRelease(
-		svc,
-		release.PutNonReleaseOptions{
-			Branch:     o.Branch,
-			BucketName: o.BucketName,
-			Files: append(
-				[]release.NonReleaseFile{release.MakeCRDBBinaryNonReleaseFile(o.AbsolutePath, o.VersionStr)},
-				additionalNonReleaseFiles...,
-			),
-		},
-	)
+type opts struct {
+	VersionStr               string
+	Branch                   string
+	ReleaseVersions          []string
+	Platform                 release.Platform
+	AbsolutePath             string
+	CockroachSQLAbsolutePath string
+	PkgDir                   string
 }