This repository has been archived by the owner on Aug 2, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #9623 from EOSIO/zach-ecr
Migrate CI from Docker Hub to Amazon ECR
- Loading branch information
Showing
10 changed files
with
243 additions
and
100 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,71 @@ | ||
| #!/bin/bash | ||
| set -eo pipefail | ||
| echo '+++ :evergreen_tree: Configuring Environment' | ||
| REPO='eosio/ci-contracts-builder' | ||
| . ./.cicd/helpers/general.sh | ||
| PREFIX='base-ubuntu-18.04' | ||
| IMAGE="$REPO:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" | ||
| SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') | ||
| SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') | ||
| echo '+++ :arrow_down: Pulling Container' | ||
| echo "Pulling \"$IMAGE\"" | ||
| docker pull "$IMAGE" | ||
| echo '+++ :label: Tagging Container' | ||
| docker tag "$IMAGE" "$REPO:$PREFIX-$SANITIZED_BRANCH" | ||
| echo "Tagged \"$REPO:$PREFIX-$SANITIZED_BRANCH\"." | ||
| [[ -z "$BUILDKITE_TAG" ]] || docker tag "$IMAGE" "$REPO:$PREFIX-$SANITIZED_TAG" && echo "Tagged \"$REPO:$PREFIX-$SANITIZED_TAG\"." | ||
| echo '+++ :arrow_up: Pushing Container' | ||
| docker push "$REPO:$PREFIX-$SANITIZED_BRANCH" | ||
| [[ -z "$BUILDKITE_TAG" ]] || docker push "$REPO:$PREFIX-$SANITIZED_TAG" | ||
| echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' | ||
| echo ${#CONTRACT_REGISTRIES[*]} | ||
| echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' | ||
| echo ${CONTRACT_REGISTRIES[*]} | ||
| # pull | ||
| echo '+++ :arrow_down: Pulling Container(s)' | ||
| for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| echo "Pulling from '$REGISTRY'." | ||
| IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" | ||
| DOCKER_PULL_COMMAND="docker pull '$IMAGE'" | ||
| echo "$ $DOCKER_PULL_COMMAND" | ||
| eval $DOCKER_PULL_COMMAND | ||
| fi | ||
| done | ||
| # tag | ||
| echo '+++ :label: Tagging Container(s)' | ||
| for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| echo "Tagging for registry $REGISTRY." | ||
| IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" | ||
| DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" | ||
| echo "$ $DOCKER_TAG_COMMAND" | ||
| eval $DOCKER_TAG_COMMAND | ||
| if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then | ||
| DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_TAG'" | ||
| echo "$ $DOCKER_TAG_COMMAND" | ||
| eval $DOCKER_TAG_COMMAND | ||
| fi | ||
| fi | ||
| done | ||
| # push | ||
| echo '+++ :arrow_up: Pushing Container(s)' | ||
| for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| echo "Pushing to '$REGISTRY'." | ||
| DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" | ||
| echo "$ $DOCKER_PUSH_COMMAND" | ||
| eval $DOCKER_PUSH_COMMAND | ||
| if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then | ||
| DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_TAG'" | ||
| echo "$ $DOCKER_PUSH_COMMAND" | ||
| eval $DOCKER_PUSH_COMMAND | ||
| fi | ||
| fi | ||
| done | ||
| # cleanup | ||
| echo '+++ :put_litter_in_its_place: Cleaning Up' | ||
| docker rmi "$REPO:$PREFIX-$SANITIZED_BRANCH" | ||
| [[ -z "$BUILDKITE_TAG" || "$SANITIZED_BRANCH" == "$SANITIZED_TAG" ]] || docker rmi "$REPO:$PREFIX-$SANITIZED_TAG" | ||
| docker rmi "$IMAGE" | ||
| for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| echo "Cleaning up from $REGISTRY." | ||
| DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" | ||
| echo "$ $DOCKER_RMI_COMMAND" | ||
| eval $DOCKER_RMI_COMMAND | ||
| if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then | ||
| DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG'" | ||
| echo "$ $DOCKER_RMI_COMMAND" | ||
| eval $DOCKER_RMI_COMMAND | ||
| fi | ||
| DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE'" | ||
| echo "$ $DOCKER_RMI_COMMAND" | ||
| eval $DOCKER_RMI_COMMAND | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,95 @@ | ||
| #!/bin/bash | ||
| set -eo pipefail | ||
| . ./.cicd/helpers/general.sh | ||
| . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile | ||
| # look for Docker image | ||
| echo "+++ :mag_right: Looking for $FULL_TAG" | ||
| ORG_REPO=$(echo $FULL_TAG | cut -d: -f1) | ||
| TAG=$(echo $FULL_TAG | cut -d: -f2) | ||
| EXISTS=$(curl -S -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG") | ||
| . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" | ||
| # search for base image in docker registries | ||
| echo "+++ :mag_right: Looking for $HASHED_IMAGE_TAG" | ||
| EXISTS_ALL='true' | ||
| EXISTS_DOCKER_HUB='false' | ||
| EXISTS_ECR='false' | ||
| for REGISTRY in ${CI_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| MANIFEST_COMMAND="docker manifest inspect '$REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $MANIFEST_COMMAND" | ||
| set +e | ||
| eval $MANIFEST_COMMAND | ||
| MANIFEST_INSPECT_EXIT_STATUS="$?" | ||
| set -eo pipefail | ||
| if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then | ||
| if [[ "$(echo "$REGISTRY" | grep -icP '[.]amazonaws[.]com/')" != '0' ]]; then | ||
| EXISTS_ECR='true' | ||
| elif [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then | ||
| EXISTS_DOCKER_HUB='true' | ||
| fi | ||
| else | ||
| EXISTS_ALL='false' | ||
| fi | ||
| fi | ||
| done | ||
| # copy, if possible, since it is so much faster | ||
| if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_BASE_IMAGE" != 'true' && ! -z "$MIRROR_REGISTRY" ]]; then | ||
| echo 'Attempting copy from Docker Hub to the mirror instead of a new base image build.' | ||
| DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_PULL_COMMAND" | ||
| set +e | ||
| eval $DOCKER_PULL_COMMAND | ||
| DOCKER_PULL_EXIT_STATUS="$?" | ||
| set -eo pipefail | ||
| if [[ "$DOCKER_PULL_EXIT_STATUS" == '0' ]]; then | ||
| echo 'Pull from Docker Hub worked! Pushing to mirror.' | ||
| # tag | ||
| DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_TAG_COMMAND" | ||
| eval $DOCKER_TAG_COMMAND | ||
| # push | ||
| DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_PUSH_COMMAND" | ||
| eval $DOCKER_PUSH_COMMAND | ||
| EXISTS_ALL='true' | ||
| EXISTS_ECR='true' | ||
| else | ||
| echo 'Pull from Docker Hub failed, rebuilding base image from scratch.' | ||
| fi | ||
| fi | ||
| # esplain yerself | ||
| if [[ "$EXISTS_ALL" == 'false' ]]; then | ||
| echo 'Building base image from scratch.' | ||
| elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then | ||
| echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries." | ||
| elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then | ||
| echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries." | ||
| fi | ||
| # build, if neccessary | ||
| if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' || $FORCE_BASE_IMAGE == true ]]; then # if we cannot pull the image, we build and push it first | ||
| docker build --no-cache -t $FULL_TAG -f $CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile . | ||
| if [[ $FORCE_BASE_IMAGE != true ]]; then | ||
| docker push $FULL_TAG | ||
| if [[ "$EXISTS_ALL" == 'false' || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first | ||
| export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." | ||
| echo "$ $DOCKER_BUILD_COMMAND" | ||
| eval $DOCKER_BUILD_COMMAND | ||
| if [[ "$FORCE_BASE_IMAGE" != 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then | ||
| for REGISTRY in ${CI_REGISTRIES[*]}; do | ||
| if [[ ! -z "$REGISTRY" ]]; then | ||
| # tag | ||
| DOCKER_TAG_COMMAND="docker tag 'ci:$HASHED_IMAGE_TAG' '$REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_TAG_COMMAND" | ||
| eval $DOCKER_TAG_COMMAND | ||
| # push | ||
| DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_PUSH_COMMAND" | ||
| eval $DOCKER_PUSH_COMMAND | ||
| # clean up | ||
| if [[ "$FULL_TAG" != "$REGISTRY:$HASHED_IMAGE_TAG" ]]; then | ||
| DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_RMI_COMMAND" | ||
| eval $DOCKER_RMI_COMMAND | ||
| fi | ||
| fi | ||
| done | ||
| DOCKER_RMI_COMMAND="docker rmi 'ci:$HASHED_IMAGE_TAG'" | ||
| echo "$ $DOCKER_RMI_COMMAND" | ||
| eval $DOCKER_RMI_COMMAND | ||
| else | ||
| echo "Base image creation successful. Not pushing...". | ||
| exit 0 | ||
| fi | ||
| else | ||
| echo "$FULL_TAG already exists." | ||
| fi | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,7 @@ | ||
| #!/bin/bash | ||
| set -eo pipefail | ||
| [[ -z $1 ]] && echo "Please provide the file to be hashed" && exit 1 | ||
| FILE_NAME=$(basename $1 | awk '{split($0,a,/\.(d|s)/); print a[1] }') | ||
| export DETERMINED_HASH=$(sha1sum $1 | awk '{ print $1 }') | ||
| [[ -z "$1" ]] && echo 'Please provide the file to be hashed as first argument.' && exit 1 | ||
| FILE_NAME="$(basename "$1" | awk '{split($0,a,/\.(d|s)/); print a[1] }')" | ||
| export DETERMINED_HASH=$(sha1sum "$1" | awk '{ print $1 }') | ||
| export HASHED_IMAGE_TAG="eos-${FILE_NAME}-${DETERMINED_HASH}" | ||
| export FULL_TAG="${IMAGE_NAME:-"eosio/ci"}:$HASHED_IMAGE_TAG" | ||
| export FULL_TAG="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,28 +1,33 @@ | ||
| export ROOT_DIR=$( dirname "${BASH_SOURCE[0]}" )/../.. | ||
| export BUILD_DIR=$ROOT_DIR/build | ||
| export CICD_DIR=$ROOT_DIR/.cicd | ||
| export HELPERS_DIR=$CICD_DIR/helpers | ||
| export BUILD_DIR="$ROOT_DIR/build" | ||
| export CICD_DIR="$ROOT_DIR/.cicd" | ||
| export HELPERS_DIR="$CICD_DIR/helpers" | ||
| export JOBS=${JOBS:-"$(getconf _NPROCESSORS_ONLN)"} | ||
| export MOUNTED_DIR='/workdir' | ||
| export DOCKER_CLI_EXPERIMENTAL='enabled' | ||
| export DOCKERHUB_CI_REGISTRY="docker.io/eosio/ci" | ||
| export DOCKERHUB_CONTRACTS_REGISTRY="docker.io/eosio/ci-contracts-builder" | ||
| export CI_REGISTRIES=("$DOCKERHUB_CI_REGISTRY" "$MIRROR_REGISTRY") | ||
| export CONTRACT_REGISTRIES=("$DOCKERHUB_CONTRACTS_REGISTRY" "$MIRROR_REGISTRY") | ||
|
|
||
| # capitalize each word in a string | ||
| function capitalize() | ||
| { | ||
| if [[ ! $1 =~ 'mac' ]]; then # Don't capitalize mac | ||
| echo $1 | awk '{$1=toupper(substr($1,1,1))substr($1,2)}1' | ||
| if [[ ! "$1" =~ 'mac' ]]; then # Don't capitalize mac | ||
| echo "$1" | awk '{$1=toupper(substr($1,1,1))substr($1,2)}1' | ||
| else | ||
| echo $1 | ||
| echo "$1" | ||
| fi | ||
| } | ||
|
|
||
| # load buildkite intrinsic environment variables for use in docker run | ||
| function buildkite-intrinsics() | ||
| { | ||
| BK_ENV='' | ||
| if [[ -f $BUILDKITE_ENV_FILE ]]; then | ||
| if [[ -f "$BUILDKITE_ENV_FILE" ]]; then | ||
| while read -r var; do | ||
| BK_ENV="$BK_ENV --env ${var%%=*}" | ||
| done < "$BUILDKITE_ENV_FILE" | ||
| fi | ||
| echo "$BK_ENV" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.