cockroachdb · craig · Apr 5, 2023 · Jan 9, 2023
@@ -23,6 +23,14 @@ docker_login_with_google() {
   echo "${google_credentials}" | docker login -u _json_key --password-stdin "https://${gcr_hostname}"
 }
 
+docker_login_gcr() {
+  local repo=$1
+  local credentials=$2
+  local hostname="${repo%%/*}"
+  # https://cloud.google.com/container-registry/docs/advanced-authentication#json-key
+  echo "${credentials}" | docker login -u _json_key --password-stdin "https://${hostname}"
+}
+
 docker_login() {
   echo "${DOCKER_ACCESS_TOKEN}" | docker login --username "${DOCKER_ID}" --password-stdin
 }
@@ -46,3 +54,54 @@ EOF
 docker_login_with_redhat() {
   echo "${REDHAT_REGISTRY_KEY}" | docker login --username unused --password-stdin $rhel_registry
 }
+
+verify_docker_image(){
+  local img=$1
+  local docker_platform=$2
+  local expected_sha=$3
+  local expected_build_tag=$4
+  local fips_build=$5
+  local error=0
+
+  docker rmi "$img" || true
+  docker pull --platform="$docker_platform" "$img"
+
+  local output=$(docker run --platform="$docker_platform" "$img" version)
+  build_type=$(grep "^Build Type:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
+  sha=$(grep "^Build Commit ID:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
+  build_tag=$(grep "^Build Tag:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
+  go_version=$(grep "^Go Version:" <<< "$output" | cut -d: -f2 | sed 's/ //g')
+
+  # Build Type should always be "release"
+  if [ "$build_type" != "release" ]; then
+    echo "ERROR: Release type mismatch, expected 'release', got '$build_type'"
+    error=1
+  fi
+  if [ "$sha" != "$expected_sha" ]; then
+    echo "ERROR: SHA mismatch, expected '$expected_sha', got '$sha'"
+    error=1
+  fi
+  if [ "$build_tag" != "$expected_build_tag" ]; then
+    echo "ERROR: Build tag mismatch, expected '$expected_build_tag', got '$build_tag'"
+    error=1
+  fi
+
+  build_tag_output=$(docker run --platform="$docker_platform" "$img" version --build-tag)
+  if [ "$build_tag_output" != "$expected_build_tag" ]; then
+    echo "ERROR: Build tag from 'cockroach version --build-tag' mismatch, expected '$expected_build_tag', got '$build_tag_output'"
+    error=1
+  fi
+  if [[ $fips_build == true ]]; then
+    if [[ "$go_version" != *"fips" ]]; then
+      echo "ERROR: Go version '$go_version' does not contain 'fips'"
+      error=1
+    fi
+    openssl_version_output=$(docker run --platform="$docker_platform" "$img" shell -c "openssl version")
+    if [[ $openssl_version_output != *"FIPS"* ]]; then
+      echo "ERROR: openssl version '$openssl_version_output' does not contain 'FIPS'"
+      error=1
+    fi
+  fi
+  return $error
+}
+
@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+source "$dir/teamcity-support.sh"  # For log_into_gcloud
+source "$dir/release/teamcity-support.sh"
+source "$dir/teamcity-bazel-support.sh"  # for run_bazel
+
+tc_start_block "Variable Setup"
+version=$(grep -v "^#" "$dir/../pkg/build/version.txt" | head -n1)
+prerelease=false
+if [[ $version == *"-"* ]]; then
+  # Our pre-release version contains a dash symbol, e.g. v22.2.0-alpha.1
+  prerelease=true
+fi
+
+if ! echo "${version}" | grep -E -o '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[-.0-9A-Za-z]+)?$'; then
+  #                                    ^major           ^minor           ^patch         ^preRelease
+  # Matching the version name regex from within the cockroach code except
+  # for the `metadata` part at the end because Docker tags don't support
+  # `+` in the tag name.
+  # https://github.com/cockroachdb/cockroach/blob/4c6864b44b9044874488cfedee3a31e6b23a6790/pkg/util/version/version.go#L75
+  echo "Invalid version \"${version}\". Must be of the format \"vMAJOR.MINOR.PATCH(-PRERELEASE)?\"."
+  exit 1
+fi
+
+release_branch=$(echo "${version}" | grep -E -o '^v[0-9]+\.[0-9]+')
+
+if [[ -z "${DRY_RUN}" ]] ; then
+  gcs_bucket="cockroach-release-artifacts-prod"
+  gcs_staged_bucket="cockroach-release-artifacts-staged-prod"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_PROD"
+  if [[ $prerelease == false ]] ; then
+    dockerhub_repository="docker.io/cockroachdb/cockroach"
+  else
+    dockerhub_repository="docker.io/cockroachdb/cockroach-unstable"
+  fi
+  gcr_staged_repository="us-docker.pkg.dev/releases-prod/cockroachdb-staged-releases/cockroach"
+  gcr_staged_credentials="$GCS_CREDENTIALS_PROD"
+  gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach"
+  gcr_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS"
+  git_repo_for_tag="cockroachdb/cockroach"
+else
+  gcs_bucket="cockroach-release-artifacts-dryrun"
+  gcs_staged_bucket="cockroach-release-artifacts-staged-dryrun"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_DEV"
+  dockerhub_repository="docker.io/cockroachdb/cockroach-misc"
+  gcr_staged_repository="us-docker.pkg.dev/releases-dev-356314/cockroachdb-staged-releases/cockroach"
+  gcr_staged_credentials="$GCS_CREDENTIALS_DEV"
+  gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
+  gcr_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
+  git_repo_for_tag=""
+fi
+
+tc_end_block "Variable Setup"
+
+
+tc_start_block "Check remote tag and tag"
+if [[ -z "${DRY_RUN}" ]]; then
+  github_ssh_key="${GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY}"
+  configure_git_ssh_key
+  if git_wrapped ls-remote --exit-code --tags "ssh://[email protected]/${git_repo_for_tag}.git" "${version}"; then
+    echo "Tag ${version} already exists"
+    exit 1
+  fi
+  git tag "${version}"
+else
+  echo "Skipping for dry-run"
+fi
+tc_end_block "Check remote tag and tag"
+
+
+tc_start_block "Setup dockerhub credentials"
+configure_docker_creds
+docker_login
+tc_end_block "Setup dockerhub credentials"
+
+
+tc_start_block "Copy binaries"
+export google_credentials="$gcs_credentials"
+log_into_gcloud
+for product in cockroach cockroach-sql; do
+  for platform in linux-amd64 linux-amd64-fips linux-arm64 darwin-10.9-amd64 darwin-11.0-arm64 windows-6.2-amd64; do
+      archive_suffix=tgz
+      if [[ $platform == *"windows"* ]]; then 
+          archive_suffix=zip
+      fi
+      archive="$product-$version.$platform.$archive_suffix"
+      gsutil cp "gs://$gcs_staged_bucket/$archive" "gs://$gcs_bucket/$archive"
+      gsutil cp "gs://$gcs_staged_bucket/$archive.sha256sum" "gs://$gcs_bucket/$archive.sha256sum"
+  done
+done
+tc_end_block "Copy binaries"
+
+
+tc_start_block "Make and push multiarch docker images"
+declare -a gcr_amends
+declare -a dockerhub_amends
+dockerhub_tag="${dockerhub_repository}:${version}"
+gcr_tag="${gcr_repository}:${version}"
+
+for platform_name in amd64 arm64; do
+  dockerhub_arch_tag="${dockerhub_repository}:${platform_name}-${version}"
+  gcr_arch_tag="${gcr_repository}:${platform_name}-${version}"
+  gcr_staged_arch_tag="${gcr_staged_repository}:${platform_name}-${version}"
+  # Update the packages before pushing to the final destination.
+  tmpdir=$(mktemp -d)
+  echo "FROM $gcr_staged_arch_tag" > "$tmpdir/Dockerfile"
+  echo "RUN microdnf -y --best --refresh upgrade && microdnf clean all && rm -rf /var/cache/yum" >> "$tmpdir/Dockerfile"
+  docker_login_gcr "$gcr_staged_repository" "$gcr_staged_credentials"
+  docker build --pull --no-cache --platform "linux/$platform_name" \
+    --tag "$dockerhub_arch_tag" --tag "$gcr_arch_tag" "$tmpdir"
+  docker_login_gcr "$gcr_repository" "$gcr_credentials"
+  docker push "$gcr_arch_tag"
+  docker push "$dockerhub_arch_tag"
+  gcr_amends+=("--amend" "$gcr_arch_tag")
+  dockerhub_amends+=("--amend" "$dockerhub_arch_tag")
+done
+
+docker_login_gcr "$gcr_repository" "$gcr_credentials"
+
+docker manifest create "${gcr_tag}" "${gcr_amends[@]}"
+docker manifest push "${gcr_tag}"
+
+docker manifest create "${dockerhub_tag}" "${dockerhub_amends[@]}"
+docker manifest push "${dockerhub_tag}"
+
+docker manifest create "${gcr_repository}:latest" "${gcr_amends[@]}"
+docker manifest create "${gcr_repository}:latest-${release_branch}" "${gcr_amends[@]}"
+
+docker manifest create "${dockerhub_repository}:latest" "${dockerhub_amends[@]}"
+docker manifest create "${dockerhub_repository}:latest-${release_branch}" "${dockerhub_amends[@]}"
+tc_end_block "Make and push multiarch docker images"
+
+
+tc_start_block "Make and push FIPS docker image"
+gcr_staged_tag_fips="${gcr_staged_repository}:${version}-fips"
+gcr_tag_fips="${gcr_repository}:${version}-fips"
+dockerhub_tag_fips="${dockerhub_repository}:${version}-fips"
+# Update the packages before pushing to the final destination.
+tmpdir=$(mktemp -d)
+echo "FROM $gcr_staged_tag_fips" > "$tmpdir/Dockerfile"
+echo "RUN microdnf -y --best --refresh upgrade && microdnf clean all && rm -rf /var/cache/yum" >> "$tmpdir/Dockerfile"
+docker_login_gcr "$gcr_staged_repository" "$gcr_staged_credentials"
+docker build --pull --no-cache --platform "linux/amd64" \
+  --tag "$dockerhub_tag_fips" --tag "$gcr_tag_fips" "$tmpdir"
+docker_login_gcr "$gcr_repository" "$gcr_credentials"
+docker push "$gcr_tag_fips"
+docker push "$dockerhub_tag_fips"
+tc_end_block "Make and push FIPS docker image"
+
+
+tc_start_block "Push release tag to GitHub"
+if [[ -z "${DRY_RUN}" ]]; then
+  configure_git_ssh_key
+  git_wrapped push "ssh://[email protected]/${git_repo_for_tag}.git" "$version"
+else
+  echo "skipping for dry-run"
+fi
+tc_end_block "Push release tag to GitHub"
+
+
+tc_start_block "Publish binaries and archive as latest"
+# Only push the "latest" for our most recent release branch.
+# https://github.com/cockroachdb/cockroach/issues/41067
+if [[ -n "${PUBLISH_LATEST}" && $prerelease == false ]]; then
+    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$version -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+bazel build --config ci //pkg/cmd/publish-provisional-artifacts
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+export google_credentials="$gcs_credentials"
+source "build/teamcity-support.sh"  # For log_into_gcloud
+log_into_gcloud
+export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release --gcs-bucket="$gcs_bucket"
+EOF
+
+else
+  echo "The latest binaries and archive were _not_ updated."
+fi
+tc_end_block "Publish binaries and archive as latest"
+
+
+tc_start_block "Tag docker image as latest-RELEASE_BRANCH"
+if [[ $prerelease == false ]]; then
+  docker manifest push "${dockerhub_repository}:latest-${release_branch}"
+else
+  echo "The ${dockerhub_repository}:latest-${release_branch} docker image tags were _not_ pushed."
+fi
+tc_end_block "Tag docker images as latest-RELEASE_BRANCH"
+
+
+tc_start_block "Tag docker images as latest"
+# Only push the "latest" tag for our most recent release branch and for the
+# latest unstable release
+# https://github.com/cockroachdb/cockroach/issues/41067
+# https://github.com/cockroachdb/cockroach/issues/48309
+if [[ -n "${PUBLISH_LATEST}" || $prerelease == true ]]; then
+  docker manifest push "${dockerhub_repository}:latest"
+else
+  echo "The ${dockerhub_repository}:latest docker image tags were _not_ pushed."
+fi
+tc_end_block "Tag docker images as latest"
+
+
+tc_start_block "Verify docker images"
+error=0
+
+images=("${dockerhub_tag}" "${gcr_tag}")
+if [[ $prerelease == false ]]; then
+  images+=("${dockerhub_repository}:latest-${release_branch}")
+fi
+if [[ -n "${PUBLISH_LATEST}" || $prerelease == true ]]; then
+  images+=("${dockerhub_repository}:latest")
+fi
+
+for img in "${images[@]}"; do
+  for platform_name in amd64 arm64; do
+    tc_start_block "Verify $img on $platform_name"
+    if ! verify_docker_image "$img" "linux/$platform_name" "$BUILD_VCS_NUMBER" "$version" false; then
+      error=1
+    fi
+    tc_end_block "Verify $img on $platform_name"
+  done
+done
+
+images=("${dockerhub_tag_fips}" "${gcr_tag_fips}")
+for img in "${images[@]}"; do
+  tc_start_block "Verify $img"
+  if ! verify_docker_image "$img" "linux/amd64" "$BUILD_VCS_NUMBER" "$version" true; then
+    error=1
+  fi
+  tc_end_block "Verify $img"
+done
+
+if [ $error = 1 ]; then
+  echo "ERROR: Docker image verification failed, see logs above"
+  exit 1
+fi
+
+tc_end_block "Verify docker images"
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname $(dirname "${0}"))))))"
+source "$dir/teamcity-support.sh"  # For log_into_gcloud
+
+KEYCHAIN_NAME=signing
+KEYCHAIN_PROFILE=notarization
+curr_dir=$(pwd)
+
+remove_files_on_exit() {
+  rm -f "$curr_dir/.google-credentials.json"
+  security lock-keychain "${KEYCHAIN_NAME}"
+}
+trap remove_files_on_exit EXIT
+
+# By default, set dry-run variables
+google_credentials="$GCS_CREDENTIALS_DEV"
+gcs_staged_bucket="cockroach-release-artifacts-staged-dryrun"
+version=$(grep -v "^#" "$dir/../pkg/build/version.txt" | head -n1)
+
+# override dev defaults with production values
+if [[ -z "${DRY_RUN}" ]] ; then
+  echo "Setting production variable values"
+  google_credentials="$GCS_CREDENTIALS_PROD"
+  gcs_staged_bucket="cockroach-release-artifacts-staged-prod"
+fi
+
+# Install gcloud/gsutil
+GOOGLE_SDK_DIR=$(mktemp -d)
+curl https://sdk.cloud.google.com > "$GOOGLE_SDK_DIR/install.sh"
+bash "$GOOGLE_SDK_DIR/install.sh" --disable-prompts --install-dir="$GOOGLE_SDK_DIR"
+export PATH="$GOOGLE_SDK_DIR/google-cloud-sdk/bin":$PATH
+
+log_into_gcloud
+security unlock-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN_NAME}"
+
+mkdir -p artifacts
+cd artifacts
+
+for product in cockroach cockroach-sql; do
+  # TODO: add Intel binaries too.
+  for platform in darwin-11.0-arm64; do
+    base=${product}-${version}.${platform}
+    unsigned_base=${product}-${version}.${platform}.unsigned
+    unsigned_file=${unsigned_base}.tgz
+    target=${base}.tgz
+
+    gsutil cp "gs://$gcs_staged_bucket/$unsigned_file" "$unsigned_file"
+    gsutil cp "gs://$gcs_staged_bucket/$unsigned_file.sha256sum" "$unsigned_file.sha256sum"
+
+    shasum --algorithm 256 --check "$unsigned_file.sha256sum"
+
+    tar -xf "$unsigned_file"
+    mv "$unsigned_base" "$base"
+
+    codesign --timestamp --options=runtime -f --keychain "$KEYCHAIN_NAME" -s "$SIGNING_IDENTITY" \
+      "$base/$product"
+    tar -czf "$target" "$base"
+
+    zip crl.zip "$base/$product"
+    xcrun notarytool submit crl.zip --wait \
+      --team-id "$TEAM_ID" --keychain-profile "$KEYCHAIN_PROFILE" \
+      --apple-id "$APPLE_ID" --verbose \
+      --keychain "${HOME}/Library/Keychains/${KEYCHAIN_NAME}-db"
+
+    rm -rf "$base" "$unsigned_file" "$unsigned_file.sha256sum" crl.zip
+
+    shasum --algorithm 256 "$target" > "$target.sha256sum"
+    gsutil cp "$target" "gs://$gcs_staged_bucket/$target"
+    gsutil cp "$target.sha256sum" "gs://$gcs_staged_bucket/$target.sha256sum"
+
+  done
+done