Add arm64 support to docker image creating tasks

This changes the x86_64 image in nightly and final build steps to be prefixed
with "amd64-", and adds an arm64 image which is prefixed with "arm64-".
The unprefixed image, which was previously the x86_64 image, is now a
multi-architecture image which points to the two prefixed images.

This change also splits build_docker_image into build_docker_image_arm64
and build_docker_image_x86_64. The Build and Test Docker TeamCity builds
have been split accordingly, into arm64 and x86_64 variants for each.
These builds do not build/test the combined image, only the individual
platforms' images.

Part of CRDB-3143

Release note (general change): Cockroach Docker images are now multi
architecture manifests supporting the x86_64 (amd64) and arm64
architectures.

build/release/teamcity-publish-redhat-release.sh

@@ -47,23 +47,34 @@ tc_start_block "Configure docker"
 docker_login_with_redhat
 tc_end_block "Configure docker"
 
-tc_start_block "Rebuild docker image"
-sed \
-  -e "s,@repository@,${dockerhub_repository},g" \
-  -e "s,@tag@,${build_name},g" \
-  build/deploy-redhat/Dockerfile.in > build/deploy-redhat/Dockerfile
+tc_start_block "Rebuild per-arch docker images"
+declare -a combined_image_args
+for docker_arch in "${docker_archs[@]}"; do
+  cp --recursive build/deploy-redhat "build/deploy-redhat-${docker_arch}"
 
-cat build/deploy-redhat/Dockerfile
+  sed \
+    -e "s,@repository@,${dockerhub_repository},g" \
+    -e "s,@tag@,${docker_arch}-${build_name},g" \
+    "build/deploy-redhat-${docker_arch}/Dockerfile.in" > "build/deploy-redhat-${docker_arch}/Dockerfile"
 
-docker build --no-cache \
-  --label release=$rhel_release \
-  --tag=${rhel_repository}:${build_name} \
-  build/deploy-redhat
-tc_end_block "Rebuild docker image"
+  cat build/deploy-redhat-${docker_arch}/Dockerfile
 
-tc_start_block "Push RedHat docker image"
-retry docker push "${rhel_repository}:${build_name}"
-tc_end_block "Push RedHat docker image"
+  build_docker_tag="${rhel_repository}:${docker_arch}-${build_name}"
+  docker build --no-cache \
+    --label=release=$rhel_release \
+    --platform="linux/${docker_arch}" \
+    --tag="${build_docker_tag}" \
+    --pull \
+    "build/deploy-redhat-${docker_arch}"
+
+  combined_image_args=( "${docker_tags[@]}" --amend "${build_docker_tag}" )
+done
+tc_end_block "Rebuild per-arch docker images"
+
+tc_start_block "Push multiarch RedHat docker image"
+docker manifest create "${rhel_repository}:${build_name}" "${combined_image_args[@]}"
+retry docker manifest push "${rhel_repository}:${build_name}"
+tc_end_block "Push multiarch RedHat docker image"
 
 tc_start_block "Run preflight"
 mkdir -p artifacts

build/release/teamcity-support.sh

@@ -4,6 +4,18 @@
 root="$(dirname $(dirname $(cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )))"
 source "$root/build/teamcity-common-support.sh"
 
+declare -a platform_names=( "amd64::amd64" "aarch64::arm64" )
+declare -a tarball_archs=( "${platform_names[@]%%::*}" ) 
+declare -a docker_archs=( "${platform_names[@]##*::}" )
+
+function tarball_arch_from_platform_name() {
+  echo "${1%%::*}"
+}
+
+function docker_arch_from_platform_name() {
+  echo "${1##*::}"
+}
+
 remove_files_on_exit() {
   rm -rf ~/.docker
   common_support_remove_files_on_exit

build/teamcity/cockroach/ci/builds/build_docker_image.sh

@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -euxo pipefail
+
+# The first and only parameter is the name of the architecture the image is being built for.
+# This should be the format Docker expects for the platform flag minus linux/ - i.e., either
+# amd64 or arm64. Old TC configs will run this file directly and not supply a platform, so
+# default to amd64.
+build_arch=${1:-amd64}
 
 dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
 source "$dir/teamcity-support.sh"  # For $root
@@ -11,7 +17,7 @@ chmod o+rwx "${artifacts}"
 
 tc_start_block "Copy cockroach binary and dependency files to build/deploy"
 
-# Get the cockroach binary from Build (Linux x86_64)
+# Get the cockroach binary from Build (Linux ${arch})
 # Artifacts rules:
 # bazel-bin/pkg/cmd/cockroach/cockroach_/cockroach=>upstream_artifacts
 # bazel-bin/c-deps/libgeos/lib/libgeos.so=>upstream_artifacts
@@ -33,11 +39,19 @@ docker_image_tar_name="cockroach-docker-image.tar"
 
 docker_tag="cockroachdb/cockroach-ci"
 
+# We have to always pull here because this runner may have been used to build
+# a different architecture's docker image. If that's the case, the cache will
+# return the cached version of the UBI base image (which will be for the wrong
+# architecture), then build will use it and fail because it's for the wrong
+# architecture. The cache is really stupid, in other words.
+
 docker build \
   --no-cache \
+  --platform=linux/${build_arch} \
   --tag="$docker_tag" \
   --memory 30g \
   --memory-swap -1 \
+  --pull \
   build/deploy
 
 docker save "$docker_tag" | gzip > "${artifacts}/${docker_image_tar_name}".gz

build/teamcity/cockroach/ci/builds/build_docker_image_arm64.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source "$(dirname "${0}")/build_docker_image.sh" arm64

build/teamcity/cockroach/ci/builds/build_docker_image_x86_64.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source "$(dirname "${0}")/build_docker_image.sh" amd64

build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -euo pipefail
+set -euxo pipefail
 
 dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
 source "$dir/release/teamcity-support.sh"
@@ -65,20 +65,52 @@ $BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/
 EOF
 tc_end_block "Compile and publish S3 artifacts"
 
-tc_start_block "Make and push docker image"
+tc_start_block "Make and push multiarch docker images"
 configure_docker_creds
 docker_login_with_google
 
-# TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
-# NB: tar usually stops reading as soon as it sees an empty block but that makes
-# curl unhappy, so passing `i` will cause it to read to the end.
-curl -f -s -S -o- "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.linux-amd64.tgz" | tar ixfz - --strip-components 1
-cp cockroach lib/libgeos.so lib/libgeos_c.so build/deploy
-cp -r licenses build/deploy/
+gcr_tag="${gcr_repository}:${build_name}"
+declare -a docker_manifest_amends
+
+for platform_name in "${platform_names[@]}"; do
+  tarball_arch="$(tarball_arch_from_platform_name "$platform_name")"
+  docker_arch="$(docker_arch_from_platform_name "$platform_name")"
+  linux_platform=linux
+  if [[ $tarball_arch == "aarch64" ]]; then
+    linux_platform=linux-3.7.10-gnu
+  fi
+  # TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
+  # NB: tar usually stops reading as soon as it sees an empty block but that makes
+  # curl unhappy, so passing `--ignore-zeros` will cause it to read to the end.
+  cp --recursive "build/deploy" "build/deploy-${docker_arch}"
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --output /dev/stdout \
+    --url "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.${linux_platform}-${tarball_arch}.tgz" \
+    | tar \
+    --directory="build/deploy-${docker_arch}" \
+    --extract \
+    --file=/dev/stdin \
+    --ungzip \
+    --ignore-zeros \
+    --strip-components=1
+  cp --recursive licenses "build/deploy-${docker_arch}"
+  # Move the libs where Dockerfile expects them to be
+  mv build/deploy-${docker_arch}/lib/* build/deploy-${docker_arch}/
+  rmdir build/deploy-${docker_arch}/lib
+
+  build_docker_tag="${gcr_repository}:${docker_arch}-${build_name}"
+  docker build --no-cache --pull --platform "linux/${docker_arch}" --tag="${build_docker_tag}" "build/deploy-${docker_arch}"
+  docker_manifest_amends+=( "--amend" "${build_docker_tag}" )
+done
+
+docker manifest create "${gcr_tag}" "${docker_manifest_amends[@]}"
+docker manifest push "${gcr_tag}"
+tc_end_block "Make and push multiarch docker images"
+
 
-docker build --no-cache --tag="${gcr_repository}:${build_name}" build/deploy
-docker push "${gcr_repository}:${build_name}"
-tc_end_block "Make and push docker image"
 
 # Make finding the tag name easy.
 cat << EOF

build/teamcity/internal/release/process/publish-cockroach-release.sh

@@ -95,26 +95,69 @@ EOF
 tc_end_block "Make and publish release S3 artifacts"
 
 
-tc_start_block "Make and push docker images"
+tc_start_block "Make and push multiarch docker images"
 configure_docker_creds
 docker_login_with_google
 docker_login
 
-# TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem?
-curl -f -s -S -o- "https://${s3_download_hostname}/cockroach-${build_name}.linux-amd64.tgz" | tar ixfz - --strip-components 1
-cp cockroach lib/libgeos.so lib/libgeos_c.so build/deploy
-cp -r licenses build/deploy/
+declare -a gcr_amends
+declare -a dockerhub_amends
 
-docker build \
-  --label version=$version \
-  --no-cache \
-  --tag=${dockerhub_repository}:{"$build_name",latest,latest-"${release_branch}"} \
-  --tag=${gcr_repository}:${build_name} \
-  build/deploy
+for platform_name in "${platform_names[@]}"; do
+  tarball_arch="$(tarball_arch_from_platform_name "$platform_name")"
+  docker_arch="$(docker_arch_from_platform_name "$platform_name")"
+  linux_platform=linux
+  if [[ $tarball_arch == "aarch64" ]]; then
+    linux_platform=linux-3.7.10-gnu
+  fi
+  # TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
+  # NB: tar usually stops reading as soon as it sees an empty block but that makes
+  # curl unhappy, so passing `--ignore-zeros` will cause it to read to the end.
+  cp --recursive "build/deploy" "build/deploy-${docker_arch}"
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --output /dev/stdout \
+    --url "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.${linux_platform}-${tarball_arch}.tgz" \
+    | tar \
+    --directory="build/deploy-${docker_arch}" \
+    --extract \
+    --file=/dev/stdin \
+    --ungzip \
+    --ignore-zeros \
+    --strip-components=1
+  cp --recursive licenses "build/deploy-${docker_arch}"
+  # Move the libs where Dockerfile expects them to be
+  mv build/deploy-${docker_arch}/lib/* build/deploy-${docker_arch}/
+  rmdir build/deploy-${docker_arch}/lib
+
+  dockerhub_arch_tag="${dockerhub_repository}:${docker_arch}-${build_name}"
+  gcr_arch_tag="${gcr_repository}:${docker_arch}-${build_name}"
+  dockerhub_amends+=("--amend" "$dockerhub_arch_tag")
+  gcr_amends+=("--amend" "$gcr_arch_tag")
+
+  # Tag the arch specific images with only one tag per repository. The manifests will reference the tags.
+  docker build \
+    --label version="$version" \
+    --no-cache \
+    --pull \
+    --platform="linux/${docker_arch}" \
+    --tag="${dockerhub_arch_tag}" \
+    --tag="${gcr_arch_tag}" \
+    "build/deploy-${docker_arch}"
+done
 
-docker push "${dockerhub_repository}:${build_name}"
-docker push "${gcr_repository}:${build_name}"
-tc_end_block "Make and push docker images"
+gcr_tag="${gcr_repository}:${build_name}"
+dockerhub_tag="${dockerhub_repository}:${build_name}"
+docker manifest create "${gcr_tag}" "${gcr_amends[@]}"
+docker manifest create "${dockerhub_tag}" "${dockerhub_amends[@]}"
+docker manifest push "${gcr_tag}"
+docker manifest push "${dockerhub_tag}"
+
+docker manifest create "${dockerhub_repository}:latest" "${dockerhub_amends[@]}"
+docker manifest create "${dockerhub_repository}:latest-${release_branch}" "${dockerhub_amends[@]}"
+tc_end_block "Make and push multiarch docker images"
 
 
 tc_start_block "Push release tag to GitHub"
@@ -152,31 +195,31 @@ tc_end_block "Publish S3 binaries and archive as latest"
 
 tc_start_block "Tag docker image as latest-RELEASE_BRANCH"
 if [[ -z "$PRE_RELEASE" ]]; then
-  docker push "${dockerhub_repository}:latest-${release_branch}"
+  docker manifest push "${dockerhub_repository}:latest-${release_branch}"
 else
-  echo "The ${dockerhub_repository}:latest-${release_branch} docker image tag was _not_ pushed."
+  echo "The ${dockerhub_repository}:latest-${release_branch} docker image tags were _not_ pushed."
 fi
-tc_end_block "Tag docker image as latest-RELEASE_BRANCH"
+tc_end_block "Tag docker images as latest-RELEASE_BRANCH"
 
 
-tc_start_block "Tag docker image as latest"
+tc_start_block "Tag docker images as latest"
 # Only push the "latest" tag for our most recent release branch and for the
 # latest unstable release
 # https://github.com/cockroachdb/cockroach/issues/41067
 # https://github.com/cockroachdb/cockroach/issues/48309
 if [[ -n "${PUBLISH_LATEST}" || -n "${PRE_RELEASE}" ]]; then
-  docker push "${dockerhub_repository}:latest"
+  docker manifest push "${dockerhub_repository}:latest"
 else
-  echo "The ${dockerhub_repository}:latest docker image tag was _not_ pushed."
+  echo "The ${dockerhub_repository}:latest docker image tags were _not_ pushed."
 fi
-tc_end_block "Tag docker image as latest"
+tc_end_block "Tag docker images as latest"
 
 
 tc_start_block "Verify docker images"
 
 images=(
-  "${dockerhub_repository}:${build_name}"
-  "${gcr_repository}:${build_name}"
+  "${dockerhub_tag}"
+  "${gcr_tag}"
 )
 if [[ -z "$PRE_RELEASE" ]]; then
   images+=("${dockerhub_repository}:latest-${release_branch}")
@@ -188,9 +231,12 @@ fi
 error=0
 
 for img in "${images[@]}"; do
+    # TODO: indent this properly
+for platform_name in "${platform_names[@]}"; do
+  docker_arch="$(docker_arch_from_platform_name "$platform_name")"
   docker rmi "$img"
-  docker pull "$img"
-  output=$(docker run "$img" version)
+  docker pull --platform="linux/${docker_arch}" "$img"
+  output=$(docker run --platform="linux/${docker_arch}" "$img" version)
   build_type=$(grep "^Build Type:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
   sha=$(grep "^Build Commit ID:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
   build_tag=$(grep "^Build Tag:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
@@ -209,12 +255,13 @@ for img in "${images[@]}"; do
     error=1
   fi
 
-  build_tag_output=$(docker run "$img" version --build-tag)
+  build_tag_output=$(docker run --platform="linux/${docker_arch}" "$img" version --build-tag)
   if [ "$build_tag_output" != "$build_name" ]; then
     echo "ERROR: Build tag from 'cockroach version --build-tag' mismatch, expected '$build_name', got '$build_tag_output'"
     error=1
   fi
 done
+done
 
 if [ $error = 1 ]; then
   echo "ERROR: Docker image verification failed, see logs above"