From 27a2c75dd05e51b624dfeb29cd2e0dd8b756eba8 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:08:28 -0400 Subject: [PATCH 01/46] Write function to completely sanitize git references for use as docker image tags --- .cicd/helpers/general.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.cicd/helpers/general.sh b/.cicd/helpers/general.sh index 77f836c92c1..4de9bb626b2 100644 --- a/.cicd/helpers/general.sh +++ b/.cicd/helpers/general.sh @@ -31,3 +31,15 @@ function buildkite-intrinsics() fi echo "$BK_ENV" } + + ##### sanitize branch names for use in URIs (docker containers) ##### +# tr '/' '_' # convert forward-slashes '/' to underscores '_' +# sed -E 's/[^-_.a-zA-Z0-9]+/-/g' # convert invalid docker chars to '-' +# sed -E 's/-+/-/g' # replace multiple dashes in a series "----" with a single dash '-' +# sed -E 's/-*_+-*/_/g' # replace dashes '-' and underscores '_' in-series with a single underscore '_' +# sed -E 's/_+/_/g' # replace multiple underscores in a row "___" with a single underscore '_' +# sed -E 's/(^[-_.]+|[-_.]+$)//g' # ensure tags do not begin or end with separator characters [-_.] +function sanitize() +{ + echo "$1" | tr '/' '_' | sed -E 's/[^-_.a-zA-Z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/-*_+-*/_/g' | sed -E 's/_+/_/g' | sed -E 's/(^[-_.]+|[-_.]+$)//g' +} From 469dcc103a57d8e51d52c6b47bdc31bc1fbd2eb0 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:08:46 -0400 Subject: [PATCH 02/46] Use common sanitize() function --- .cicd/create-docker-from-binary.sh | 5 ++--- .cicd/docker-tag.sh | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index aa58f3e1b38..5e79a2f3ed1 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -4,9 +4,8 @@ set -euo pipefail buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . echo ":done: download successful" - -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" echo "$SANITIZED_BRANCH" echo "$SANITIZED_TAG" diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 85142e60b52..8e20e0f2f8f 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -3,8 +3,8 @@ set -eo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' From 76d53d735a05344caed112af7879e0bc32716ce1 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:10:46 -0400 Subject: [PATCH 03/46] Add useful print statements showing how git references are being transformed --- .cicd/create-docker-from-binary.sh | 2 ++ .cicd/docker-tag.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 5e79a2f3ed1..26489a6a7c7 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -5,7 +5,9 @@ set -euo pipefail buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . echo ":done: download successful" SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." echo "$SANITIZED_BRANCH" echo "$SANITIZED_TAG" diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 8e20e0f2f8f..1a5519274f9 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -4,7 +4,9 @@ echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' From 53dc561747f118045a17d300b13b745b4dc50ee6 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:12:25 -0400 Subject: [PATCH 04/46] BUILDKITE_TAG could be non-empty while SANITIZED_TAG *is* empty --- .cicd/docker-tag.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 1a5519274f9..33211ac88d8 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -25,7 +25,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND @@ -40,7 +40,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND @@ -58,7 +58,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND From 19f22c71094b12cdabb1462664f226e0f55b1d67 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:12:58 -0400 Subject: [PATCH 05/46] Remove duplicate print statements --- .cicd/create-docker-from-binary.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 26489a6a7c7..a3edac204a4 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -8,8 +8,6 @@ SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." -echo "$SANITIZED_BRANCH" -echo "$SANITIZED_TAG" # do docker build echo ":docker::build: Building image..." From f73a10529280fb3f2ec4e85e0bc25fe16a5b7e93 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:33:29 -0400 Subject: [PATCH 06/46] Whitespace --- .cicd/create-docker-from-binary.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index a3edac204a4..245e4678557 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -1,25 +1,19 @@ #!/bin/bash - set -euo pipefail - buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . echo ":done: download successful" SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." - # do docker build echo ":docker::build: Building image..." DOCKERHUB_REGISTRY="docker.io/eosio/eosio" - BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest} DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ." echo "$ $DOCKER_BUILD_GEN" eval $DOCKER_BUILD_GEN - #tag and push on each destination AWS & DOCKERHUB - EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" @@ -47,7 +41,6 @@ for REG in ${EOSIO_REGS[@]}; do eval $DOCKER_REM fi done - DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG || :" echo "Clean up base image" -eval $DOCKER_GEN \ No newline at end of file +eval $DOCKER_GEN From e3c6688318749bbf577cfe157def075206448efd Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:40:09 -0400 Subject: [PATCH 07/46] Add log folding groups to docker build step --- .cicd/create-docker-from-binary.sh | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 245e4678557..106cdc00263 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -1,4 +1,5 @@ #!/bin/bash +echo '--- :evergreen_tree: Configuring Environment' set -euo pipefail buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . echo ":done: download successful" @@ -6,14 +7,15 @@ SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." -# do docker build -echo ":docker::build: Building image..." +# docker build +echo "+++ :docker: Build Docker Container" DOCKERHUB_REGISTRY="docker.io/eosio/eosio" BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest} DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ." echo "$ $DOCKER_BUILD_GEN" eval $DOCKER_BUILD_GEN -#tag and push on each destination AWS & DOCKERHUB +# docker tag +echo '--- :label: Tag Container' EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" @@ -21,23 +23,36 @@ for REG in ${EOSIO_REGS[@]}; do echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH" eval $DOCKER_TAG_COMMIT eval $DOCKER_TAG_BRANCH + if [[ ! -z "$SANITIZED_TAG" ]]; then + DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" + echo -e "$ \n Tagging Image: \n$DOCKER_TAG" + eval $DOCKER_TAG + fi +done +# docker push +echo '--- :arrow_up: Push Container' +for REG in ${EOSIO_REGS[@]}; do DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT" DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH" echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH" eval $DOCKER_PUSH_COMMIT eval $DOCKER_PUSH_BRANCH + if [[ ! -z "$SANITIZED_TAG" ]]; then + DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" + eval $DOCKER_PUSH_TAG + fi +done +# docker rmi +echo '--- :put_litter_in_its_place: Cleanup' +for REG in ${EOSIO_REGS[@]}; do CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT || :" CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH || :" echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$" eval $CLEAN_IMAGE_COMMIT eval $CLEAN_IMAGE_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" - DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" DOCKER_REM="docker rmi $REG:$SANITIZED_TAG || :" - echo -e "$ \n Tagging Image: \n$DOCKER_TAG \n Cleaning Up: \n$DOCKER_REM" - eval $DOCKER_TAG - eval $DOCKER_PUSH_TAG + echo -e "$ Cleaning Up: \n$DOCKER_REM" eval $DOCKER_REM fi done From 73749d043b5582033f4250984f426f86688fbb33 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:44:08 -0400 Subject: [PATCH 08/46] More useful print statements --- .cicd/create-docker-from-binary.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 106cdc00263..dd2feecb2bd 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -2,7 +2,6 @@ echo '--- :evergreen_tree: Configuring Environment' set -euo pipefail buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . -echo ":done: download successful" SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" @@ -20,12 +19,13 @@ EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" DOCKER_TAG_BRANCH="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_BRANCH" - echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH" + echo "$ $DOCKER_TAG_COMMIT" eval $DOCKER_TAG_COMMIT + echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" - echo -e "$ \n Tagging Image: \n$DOCKER_TAG" + echo "$ $DOCKER_TAG" eval $DOCKER_TAG fi done @@ -34,11 +34,13 @@ echo '--- :arrow_up: Push Container' for REG in ${EOSIO_REGS[@]}; do DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT" DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH" - echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH" + echo "$ $DOCKER_PUSH_COMMIT" eval $DOCKER_PUSH_COMMIT + echo "$ $DOCKER_PUSH_BRANCH" eval $DOCKER_PUSH_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" + echo "$ $DOCKER_PUSH_TAG" eval $DOCKER_PUSH_TAG fi done @@ -47,15 +49,16 @@ echo '--- :put_litter_in_its_place: Cleanup' for REG in ${EOSIO_REGS[@]}; do CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT || :" CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH || :" - echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$" + echo "$ $CLEAN_IMAGE_COMMIT" eval $CLEAN_IMAGE_COMMIT + echo "$ $CLEAN_IMAGE_BRANCH" eval $CLEAN_IMAGE_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_REM="docker rmi $REG:$SANITIZED_TAG || :" - echo -e "$ Cleaning Up: \n$DOCKER_REM" + echo "$ $DOCKER_REM" eval $DOCKER_REM fi done DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG || :" -echo "Clean up base image" +echo "$ $DOCKER_GEN" eval $DOCKER_GEN From 1b5d3e4c1444e3b2b1374c05dd1768f08b5b21f2 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:49:20 -0400 Subject: [PATCH 09/46] More unique image name --- .cicd/create-docker-from-binary.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index dd2feecb2bd..856647353f5 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -9,22 +9,22 @@ SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" # docker build echo "+++ :docker: Build Docker Container" DOCKERHUB_REGISTRY="docker.io/eosio/eosio" -BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest} -DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ." +IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" +DOCKER_BUILD_GEN="docker build -t '$IMAGE' -f ./docker/dockerfile ." echo "$ $DOCKER_BUILD_GEN" eval $DOCKER_BUILD_GEN # docker tag echo '--- :label: Tag Container' EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do - DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" - DOCKER_TAG_BRANCH="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_BRANCH" + DOCKER_TAG_COMMIT="docker tag '$IMAGE' $REG:$BUILDKITE_COMMIT" + DOCKER_TAG_BRANCH="docker tag '$IMAGE' $REG:$SANITIZED_BRANCH" echo "$ $DOCKER_TAG_COMMIT" eval $DOCKER_TAG_COMMIT echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" + DOCKER_TAG="docker tag '$IMAGE' $REG:$SANITIZED_TAG" echo "$ $DOCKER_TAG" eval $DOCKER_TAG fi @@ -59,6 +59,6 @@ for REG in ${EOSIO_REGS[@]}; do eval $DOCKER_REM fi done -DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG || :" +DOCKER_GEN="docker rmi '$IMAGE' || :" echo "$ $DOCKER_GEN" eval $DOCKER_GEN From 60af763d1b94b4a4a906d758dae4ba0b9e609bdb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:50:45 -0400 Subject: [PATCH 10/46] String quoting --- .cicd/create-docker-from-binary.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 856647353f5..16a8e9b7445 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -8,7 +8,7 @@ SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." # docker build echo "+++ :docker: Build Docker Container" -DOCKERHUB_REGISTRY="docker.io/eosio/eosio" +DOCKERHUB_REGISTRY='docker.io/eosio/eosio' IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" DOCKER_BUILD_GEN="docker build -t '$IMAGE' -f ./docker/dockerfile ." echo "$ $DOCKER_BUILD_GEN" @@ -17,14 +17,14 @@ eval $DOCKER_BUILD_GEN echo '--- :label: Tag Container' EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do - DOCKER_TAG_COMMIT="docker tag '$IMAGE' $REG:$BUILDKITE_COMMIT" - DOCKER_TAG_BRANCH="docker tag '$IMAGE' $REG:$SANITIZED_BRANCH" + DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" + DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMIT" eval $DOCKER_TAG_COMMIT echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_TAG="docker tag '$IMAGE' $REG:$SANITIZED_TAG" + DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_TAG" eval $DOCKER_TAG fi @@ -32,14 +32,14 @@ done # docker push echo '--- :arrow_up: Push Container' for REG in ${EOSIO_REGS[@]}; do - DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT" - DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH" + DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" + DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'" echo "$ $DOCKER_PUSH_COMMIT" eval $DOCKER_PUSH_COMMIT echo "$ $DOCKER_PUSH_BRANCH" eval $DOCKER_PUSH_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" + DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_TAG" eval $DOCKER_PUSH_TAG fi @@ -47,14 +47,14 @@ done # docker rmi echo '--- :put_litter_in_its_place: Cleanup' for REG in ${EOSIO_REGS[@]}; do - CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT || :" - CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH || :" + CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" + CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :" echo "$ $CLEAN_IMAGE_COMMIT" eval $CLEAN_IMAGE_COMMIT echo "$ $CLEAN_IMAGE_BRANCH" eval $CLEAN_IMAGE_BRANCH if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_REM="docker rmi $REG:$SANITIZED_TAG || :" + DOCKER_REM="docker rmi '$REG:$SANITIZED_TAG' || :" echo "$ $DOCKER_REM" eval $DOCKER_REM fi From 1e767545eb93247b164195bb1b84e9a23e09c379 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:52:42 -0400 Subject: [PATCH 11/46] Sort --- .cicd/create-docker-from-binary.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 16a8e9b7445..b525dd77f46 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -17,12 +17,12 @@ eval $DOCKER_BUILD_GEN echo '--- :label: Tag Container' EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") for REG in ${EOSIO_REGS[@]}; do - DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" - echo "$ $DOCKER_TAG_COMMIT" - eval $DOCKER_TAG_COMMIT echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH + DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_TAG_COMMIT" + eval $DOCKER_TAG_COMMIT if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_TAG" @@ -32,12 +32,12 @@ done # docker push echo '--- :arrow_up: Push Container' for REG in ${EOSIO_REGS[@]}; do - DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'" - echo "$ $DOCKER_PUSH_COMMIT" - eval $DOCKER_PUSH_COMMIT echo "$ $DOCKER_PUSH_BRANCH" eval $DOCKER_PUSH_BRANCH + DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_PUSH_COMMIT" + eval $DOCKER_PUSH_COMMIT if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_TAG" @@ -47,12 +47,12 @@ done # docker rmi echo '--- :put_litter_in_its_place: Cleanup' for REG in ${EOSIO_REGS[@]}; do - CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :" - echo "$ $CLEAN_IMAGE_COMMIT" - eval $CLEAN_IMAGE_COMMIT echo "$ $CLEAN_IMAGE_BRANCH" eval $CLEAN_IMAGE_BRANCH + CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" + echo "$ $CLEAN_IMAGE_COMMIT" + eval $CLEAN_IMAGE_COMMIT if [[ ! -z "$SANITIZED_TAG" ]]; then DOCKER_REM="docker rmi '$REG:$SANITIZED_TAG' || :" echo "$ $DOCKER_REM" From ee1eebdcf9d2b8f33bd4cc2f8eb398f461d943d3 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:54:36 -0400 Subject: [PATCH 12/46] Fix bug when sanitized branch and sanitized tag are equivalent --- .cicd/create-docker-from-binary.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index b525dd77f46..9aa176d30b2 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -23,7 +23,7 @@ for REG in ${EOSIO_REGS[@]}; do DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" echo "$ $DOCKER_TAG_COMMIT" eval $DOCKER_TAG_COMMIT - if [[ ! -z "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_TAG" eval $DOCKER_TAG @@ -38,7 +38,7 @@ for REG in ${EOSIO_REGS[@]}; do DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" echo "$ $DOCKER_PUSH_COMMIT" eval $DOCKER_PUSH_COMMIT - if [[ ! -z "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_TAG" eval $DOCKER_PUSH_TAG @@ -53,7 +53,7 @@ for REG in ${EOSIO_REGS[@]}; do CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" echo "$ $CLEAN_IMAGE_COMMIT" eval $CLEAN_IMAGE_COMMIT - if [[ ! -z "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_REM="docker rmi '$REG:$SANITIZED_TAG' || :" echo "$ $DOCKER_REM" eval $DOCKER_REM From 756dd167cb0b2bbad07b3d6987556dd9ce69dc4e Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:56:07 -0400 Subject: [PATCH 13/46] Rename "EOSIO_REGS" to "REGISTRIES" so it can't be confused with $EOSIO_REGISTRY accidentally --- .cicd/create-docker-from-binary.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 9aa176d30b2..ad885f0359a 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -15,8 +15,8 @@ echo "$ $DOCKER_BUILD_GEN" eval $DOCKER_BUILD_GEN # docker tag echo '--- :label: Tag Container' -EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") -for REG in ${EOSIO_REGS[@]}; do +REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") +for REG in ${REGISTRIES[@]}; do DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH @@ -31,7 +31,7 @@ for REG in ${EOSIO_REGS[@]}; do done # docker push echo '--- :arrow_up: Push Container' -for REG in ${EOSIO_REGS[@]}; do +for REG in ${REGISTRIES[@]}; do DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'" echo "$ $DOCKER_PUSH_BRANCH" eval $DOCKER_PUSH_BRANCH @@ -46,7 +46,7 @@ for REG in ${EOSIO_REGS[@]}; do done # docker rmi echo '--- :put_litter_in_its_place: Cleanup' -for REG in ${EOSIO_REGS[@]}; do +for REG in ${REGISTRIES[@]}; do CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :" echo "$ $CLEAN_IMAGE_BRANCH" eval $CLEAN_IMAGE_BRANCH From ad08d27b642b20c4361738e0f6f3835db4e119af Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 18:57:15 -0400 Subject: [PATCH 14/46] Simplify variable names --- .cicd/create-docker-from-binary.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index ad885f0359a..5374f19012a 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -10,9 +10,9 @@ SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" echo "+++ :docker: Build Docker Container" DOCKERHUB_REGISTRY='docker.io/eosio/eosio' IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" -DOCKER_BUILD_GEN="docker build -t '$IMAGE' -f ./docker/dockerfile ." -echo "$ $DOCKER_BUILD_GEN" -eval $DOCKER_BUILD_GEN +DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ." +echo "$ $DOCKER_BUILD" +eval $DOCKER_BUILD # docker tag echo '--- :label: Tag Container' REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") @@ -54,11 +54,11 @@ for REG in ${REGISTRIES[@]}; do echo "$ $CLEAN_IMAGE_COMMIT" eval $CLEAN_IMAGE_COMMIT if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then - DOCKER_REM="docker rmi '$REG:$SANITIZED_TAG' || :" - echo "$ $DOCKER_REM" - eval $DOCKER_REM + DOCKER_RMI="docker rmi '$REG:$SANITIZED_TAG' || :" + echo "$ $DOCKER_RMI" + eval $DOCKER_RMI fi done -DOCKER_GEN="docker rmi '$IMAGE' || :" -echo "$ $DOCKER_GEN" -eval $DOCKER_GEN +DOCKER_RMI="docker rmi '$IMAGE' || :" +echo "$ $DOCKER_RMI" +eval $DOCKER_RMI From b90e9259fbde1864e7a4f365fc383e7861e185d6 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:01:36 -0400 Subject: [PATCH 15/46] Move "See Also" section in CICD README.md to end --- .cicd/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 95bf71a27db..0288911ecb0 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -1,11 +1,6 @@ # eosio The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pipeline for members of the EOSIO organization developing on the EOSIO/eos repository. The [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipeline is also executed regularly. The difference in these two pipelines is whether the compiler and other dependencies are pinned to specific versions. The eosio pipeline uses pinned compilers/dependencies while the eosio-build-unpinned pipeline avoids pinning dependencies as much as possible. -## See Also -- [Buildkite Documentation](https://github.com/EOSIO/devdocs/wiki/Buildkite) (internal) -- [EOSIO Resume from State Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) (internal) -- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) (internal) -
More Info @@ -13,6 +8,7 @@ The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pip 1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) 1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) 1. [Pipelines](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#pipelines) +1. [See Also](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#see-also) ### Variables Most pipelines in the organization have several environment variables that can be used to configure how the pipeline runs. These environment variables can be specified when manually triggering a build via the Buildkite UI. @@ -87,3 +83,8 @@ Pipeline | Details [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds.
+ +## See Also +- [Buildkite Documentation](https://github.com/EOSIO/devdocs/wiki/Buildkite) (internal) +- [EOSIO Resume from State Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) (internal) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) (internal) From 694a3fd2848bc2948cb75c31a504d8ce83739722 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:06:03 -0400 Subject: [PATCH 16/46] Make "Pipelines" section H2 --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 0288911ecb0..b4635b09a1f 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -67,7 +67,7 @@ SKIP_MULTIVERSION_TEST='true' SKIP_SYNC_TESTS='true' ``` -### Pipelines +## Pipelines There are several eosio pipelines that are exist and executed via pull requests, triggered from other builds, or scheduled to run on a regular basis: From e47e462f0fa9b07f3149ff08282ff57e69b9a804 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:10:24 -0400 Subject: [PATCH 17/46] Typo; Concise --- .cicd/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index b4635b09a1f..5aa45719bb3 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -68,8 +68,7 @@ SKIP_SYNC_TESTS='true' ``` ## Pipelines -There are several eosio pipelines that are exist and executed via pull requests, triggered from other builds, or scheduled to run on a regular basis: - +There are several eosio pipelines that exist and are triggered by pull requests, pipelines, or schedules: Pipeline | Details ---|--- From f82cb40ffc855a05fbd7bdbc843b5f388a160921 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:23:10 -0400 Subject: [PATCH 18/46] Write "Configuration" section --- .cicd/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cicd/README.md b/.cicd/README.md index 5aa45719bb3..d0afcc14408 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -5,11 +5,15 @@ The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pip More Info ## Index +1. [Configuration](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#configuration) 1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) 1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) 1. [Pipelines](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#pipelines) 1. [See Also](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#see-also) +## Configuration +Most EOSIO pipelines are run any time you push a commit or tag to an open pull request in [eos](https://github.com/EOSIO/eos), any time you merge a pull request, and nightly. The [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline only runs when you merge a pull request because it takes so long. Long-running tests are also run in the [eosio](https://buildkite.com/EOSIO/eosio) nightly builds, which have `RUN_ALL_TESTS='true'` set. + ### Variables Most pipelines in the organization have several environment variables that can be used to configure how the pipeline runs. These environment variables can be specified when manually triggering a build via the Buildkite UI. From 78541bfac225b37b54e376c6415a96e4b1eec13d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:24:16 -0400 Subject: [PATCH 19/46] Make "Variables" and "Examples" H3 --- .cicd/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index d0afcc14408..4f4a26df2be 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -6,8 +6,8 @@ The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pip ## Index 1. [Configuration](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#configuration) -1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) -1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) + 1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) + 1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) 1. [Pipelines](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#pipelines) 1. [See Also](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#see-also) From f0bdcbcaf66abbfa198200193602ef5b9f5f9102 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:27:40 -0400 Subject: [PATCH 20/46] Less text --- .cicd/README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 4f4a26df2be..77063061a39 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -19,35 +19,35 @@ Most pipelines in the organization have several environment variables that can b Configure which operating systems are built, tested, and packaged: ```bash -SKIP_LINUX='true|false' # true skips all build/test/packaging steps on Linux distros -SKIP_MAC='true|false' # true skips all build/test/packaging steps on Mac hardware -SKIP_AMAZON_LINUX_2='true|false' # true skips all build/test/packaging steps for Amazon Linux 2 -SKIP_CENTOS_7_7='true|false' # true skips all build/test/packaging steps for Centos 7 -SKIP_CENTOS_8='true|false' # true skips all build/test/packaging steps for Centos 8 -SKIP_MACOS_10_14='true|false' # true skips all build/test/packaging steps for MacOS 10.14 -SKIP_MACOS_10_15='true|false' # true skips all build/test/packaging steps for MacOS 10.15 -SKIP_MACOS_11='true|false' # true skips all build/test/packaging steps for MacOS 11 -SKIP_UBUNTU_16_04='true|false' # true skips all build/test/packaging steps for Ubuntu 16.04 -SKIP_UBUNTU_18_04='true|false' # true skips all build/test/packaging steps for Ubuntu 18.04 -SKIP_UBUNTU_20_04='true|false' # true skips all build/test/packaging steps for Ubuntu 20.04 +SKIP_LINUX='true|false' # skip all steps on Linux distros +SKIP_MAC='true|false' # skip all steps on Mac hardware +SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 +SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7 +SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 +SKIP_MACOS_10_14='true|false' # skip all steps for MacOS 10.14 +SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 +SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 +SKIP_UBUNTU_16_04='true|false' # skip all steps for Ubuntu 16.04 +SKIP_UBUNTU_18_04='true|false' # skip all steps for Ubuntu 18.04 +SKIP_UBUNTU_20_04='true|false' # skip all steps for Ubuntu 20.04 ``` Configure which steps are executed for each operating system: ```bash -SKIP_BUILD='true|false' # true skips all build steps for all distros -SKIP_UNIT_TESTS='true|false' # true skips all unit test executions for all distros -SKIP_WASM_SPEC_TESTS='true|false' # true skips all wasm spec test executions for all distros -SKIP_SERIAL_TESTS='true|false' # true skips all integration test executions for all distros -SKIP_LONG_RUNNING_TESTS='true|false' # true skips all long running test executions for all distros -SKIP_MULTIVERSION_TEST='true|false' # true skips all multiversion tests -SKIP_SYNC_TESTS='true|false' # true skips all sync tests -SKIP_PACKAGE_BUILDER='true|false' # true skips all package building steps for all distros +SKIP_BUILD='true|false' # skip all build steps +SKIP_UNIT_TESTS='true|false' # skip all unit tests +SKIP_WASM_SPEC_TESTS='true|false' # skip all wasm spec tests +SKIP_SERIAL_TESTS='true|false' # skip all integration tests +SKIP_LONG_RUNNING_TESTS='true|false' # skip all long running tests +SKIP_MULTIVERSION_TEST='true|false' # skip all multiversion tests +SKIP_SYNC_TESTS='true|false' # skip all sync tests +SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps ``` Configure how the steps are executed: ```bash PINNED='true|false' # controls compiler/dependency pinning -TIMEOUT='##' # controls timeout in minutes for all steps +TIMEOUT='##' # set timeout in minutes for all steps ``` ### Examples From 9c2a62f074a1717530f851d25ab51d4371e6aaad Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:38:10 -0400 Subject: [PATCH 21/46] Break plaform SKIP_ vars into their own section with statement on order of operations --- .cicd/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 77063061a39..77de4354cc4 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -17,10 +17,15 @@ Most EOSIO pipelines are run any time you push a commit or tag to an open pull r ### Variables Most pipelines in the organization have several environment variables that can be used to configure how the pipeline runs. These environment variables can be specified when manually triggering a build via the Buildkite UI. -Configure which operating systems are built, tested, and packaged: +Configure which platforms are run: ```bash SKIP_LINUX='true|false' # skip all steps on Linux distros SKIP_MAC='true|false' # skip all steps on Mac hardware +``` +These will override more specific operating system declarations, and primarily exist to disable one of our two buildfleets should one be sick or the finite macOS agents are congested. + +Configure which operating systems are built, tested, and packaged: +```bash SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7 SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 From 99c72841182795a1e6fd43edf72247a1ff9ff390 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:41:00 -0400 Subject: [PATCH 22/46] CentOS 7.7 --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 77de4354cc4..6e94ef113aa 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -27,7 +27,7 @@ These will override more specific operating system declarations, and primarily e Configure which operating systems are built, tested, and packaged: ```bash SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 -SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7 +SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 SKIP_MACOS_10_14='true|false' # skip all steps for MacOS 10.14 SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 From 1c73a675cf8963ed91f56b25f9564458c7d2bdc5 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:42:03 -0400 Subject: [PATCH 23/46] Moar words --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 6e94ef113aa..a46ec7e0423 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -51,7 +51,7 @@ SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps Configure how the steps are executed: ```bash -PINNED='true|false' # controls compiler/dependency pinning +PINNED='true|false' # use specific versions of dependencies instead of whatever version is provided by default on a given platform TIMEOUT='##' # set timeout in minutes for all steps ``` From d72e5782f1065a4fb343fa17ea0dbf21aa8d8553 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 19:43:35 -0400 Subject: [PATCH 24/46] Alphabetize pipeline table --- .cicd/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index a46ec7e0423..24d3622b7bd 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -82,13 +82,13 @@ There are several eosio pipelines that exist and are triggered by pull requests, Pipeline | Details ---|--- [eosio](https://buildkite.com/EOSIO/eosio) | Primary pipeline for the EOSIO/eos Github repo. It is triggered when a pull request is created. -[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | Pipeline that performs a build without a pinned compiler. It is triggered when a pull request is created. -[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. [eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | Pipeline that ensures all MacOS VM and Docker container builders can be built. It is scheduled for periodic execution. -[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. -[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. +[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. +[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | Pipeline that performs a build without a pinned compiler. It is triggered when a pull request is created. +[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. +[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. From 80cadc63326a6a36eda87c4d37512e5a8e85bb6b Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:39:58 -0400 Subject: [PATCH 25/46] Expand intro paragraph --- .cicd/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 24d3622b7bd..29edc0b4ed9 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -1,5 +1,7 @@ # eosio -The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pipeline for members of the EOSIO organization developing on the EOSIO/eos repository. The [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipeline is also executed regularly. The difference in these two pipelines is whether the compiler and other dependencies are pinned to specific versions. The eosio pipeline uses pinned compilers/dependencies while the eosio-build-unpinned pipeline avoids pinning dependencies as much as possible. +The [eosio](https://buildkite.com/EOSIO/eosio) and [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipelines are the primary pipelines for the [eos](https://github.com/EOSIO/eos) repository, running with specific or default versions of our dependencies, respectively. Both run against every commit to a base branch or pull request, along with the [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) pipeline. + +The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) and [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) pipelines on each build, and the the [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline on merge commits. Each of these pipelines are described in more detail below and in their respective READMEs.
More Info From b1f757bc71e077df61d50cda9cafe9823f5e8942 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:41:27 -0400 Subject: [PATCH 26/46] Add RUN_ALL_TESTS variable --- .cicd/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/README.md b/.cicd/README.md index 29edc0b4ed9..0cea2250378 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -28,6 +28,7 @@ These will override more specific operating system declarations, and primarily e Configure which operating systems are built, tested, and packaged: ```bash +RUN_ALL_TESTS='true' # run all tests in the current build (including LRTs, overridden by SKIP* variables) SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 From 9f79a484071bb178a6662cb6eb3b4647c141249e Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:43:53 -0400 Subject: [PATCH 27/46] Add eosio-code-coverage pipeline --- .cicd/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/README.md b/.cicd/README.md index 0cea2250378..f0958b7010e 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -89,6 +89,7 @@ Pipeline | Details [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | Pipeline that performs a build without a pinned compiler. It is triggered when a pull request is created. +[eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. From b945e5f7ae09ee32add783b3624e6f2d6f1e5b62 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:46:57 -0400 Subject: [PATCH 28/46] Update description for eosio and eosio-build-unpinned --- .cicd/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index f0958b7010e..0fa337af32e 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -84,11 +84,11 @@ There are several eosio pipelines that exist and are triggered by pull requests, Pipeline | Details ---|--- -[eosio](https://buildkite.com/EOSIO/eosio) | Primary pipeline for the EOSIO/eos Github repo. It is triggered when a pull request is created. +[eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit [eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | Pipeline that ensures all MacOS VM and Docker container builders can be built. It is scheduled for periodic execution. [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. -[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | Pipeline that performs a build without a pinned compiler. It is triggered when a pull request is created. +[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. From 1185f77dfff1963d58a565d336dbcf0d04424640 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:51:44 -0400 Subject: [PATCH 29/46] Don't forget nightly builds --- .cicd/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 0fa337af32e..412e59d9270 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -84,11 +84,11 @@ There are several eosio pipelines that exist and are triggered by pull requests, Pipeline | Details ---|--- -[eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit +[eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly [eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | Pipeline that ensures all MacOS VM and Docker container builders can be built. It is scheduled for periodic execution. [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. -[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit +[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. From 6dee9ee6bd7af59fdc0a53277f79b7276e58ccc1 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:52:40 -0400 Subject: [PATCH 30/46] Update eosio-lrt description --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 412e59d9270..71f4ce17869 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -90,7 +90,7 @@ Pipeline | Details [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit -[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. +[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. From 809413a22b9d5e86c11c8144a6074cfcdacad596 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:54:50 -0400 Subject: [PATCH 31/46] Update eosio-resume-from-state documentation --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 71f4ce17869..f3cdb45a732 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -91,7 +91,7 @@ Pipeline | Details [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits -[eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. +[eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds.
From 447d390b1382fe37d88f1ea2ef9b6d4bcd770824 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 20:58:58 -0400 Subject: [PATCH 32/46] Update eosio-sync-from-genesis description --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index f3cdb45a732..7cfe3ce9a0c 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -92,7 +92,7 @@ Pipeline | Details [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) -[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. +[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build From d86eee1a45678b18293d78562995c269a4a5cddb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:01:11 -0400 Subject: [PATCH 33/46] Update eosio-base-images description --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 7cfe3ce9a0c..654962e0a19 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -85,7 +85,7 @@ There are several eosio pipelines that exist and are triggered by pull requests, Pipeline | Details ---|--- [eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly -[eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | Pipeline that ensures all MacOS VM and Docker container builders can be built. It is scheduled for periodic execution. +[eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | pack EOSIO dependencies into docker and Anka base-images nightly [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly From 64454915ed49411a3434b6289060573139f1a3eb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:02:08 -0400 Subject: [PATCH 34/46] Update eosio-big-sur-beta description --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 654962e0a19..56fc09889b6 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -86,7 +86,7 @@ Pipeline | Details ---|--- [eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly [eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | pack EOSIO dependencies into docker and Anka base-images nightly -[eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. +[eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | build and test [eos](https://github.com/EOSIO/eos) on macOS 11 "Big Sur" weekly [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit From e1eb90eb3ae2c8568b6c3839e836d27ae6ceb5aa Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:03:07 -0400 Subject: [PATCH 35/46] Update eosio-build-scripts description --- .cicd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 56fc09889b6..497393764b7 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -87,7 +87,7 @@ Pipeline | Details [eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly [eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | pack EOSIO dependencies into docker and Anka base-images nightly [eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | build and test [eos](https://github.com/EOSIO/eos) on macOS 11 "Big Sur" weekly -[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. +[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | run [eos](https://github.com/EOSIO/eos) build scripts nightly on empty operating systems [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits From d1a620a71706e9831f2f07b98ef81538f62ec913 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:05:20 -0400 Subject: [PATCH 36/46] Update "See Also" section --- .cicd/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 497393764b7..de49dc2a83b 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -97,6 +97,8 @@ Pipeline | Details ## See Also -- [Buildkite Documentation](https://github.com/EOSIO/devdocs/wiki/Buildkite) (internal) -- [EOSIO Resume from State Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) (internal) -- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) (internal) +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel From 84ff9e09a6befceb0f0cf71fab70e5f600647fb3 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:07:30 -0400 Subject: [PATCH 37/46] Only show intro paragraph by default --- .cicd/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index de49dc2a83b..da441f36806 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -4,7 +4,7 @@ The [eosio](https://buildkite.com/EOSIO/eosio) and [eosio-build-unpinned](https: The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) and [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) pipelines on each build, and the the [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline on merge commits. Each of these pipelines are described in more detail below and in their respective READMEs.
-More Info +See More ## Index 1. [Configuration](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#configuration) @@ -94,11 +94,11 @@ Pipeline | Details [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build -
- ## See Also - Buildkite - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel + + From a3a729ae673379f6d607077ed5697acb86732685 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:09:22 -0400 Subject: [PATCH 38/46] Only upload pipeline documentation once; link to GitHub --- .cicd/generate-pipeline.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8ae93751f5c..4ddb5cdad19 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -6,8 +6,16 @@ export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' -# Add build annotation -cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' +# attach pipeline documentation +echo '+++ :md: Attaching Documentation' +export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' +elif [[ "$BUILDKITE" == 'true' ]]; then + printf "Skipping \033]1339;url=$DOCS_URL;content=documentation\a upload for job retry number $RETRY.\n" +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From f9dd93cbd55018a855d24b2c524666244eac8735 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:22:00 -0400 Subject: [PATCH 39/46] Fix print statements since generate-pipeline.sh is piped to buildkite-agent pipeline upload --- .cicd/generate-pipeline.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4ddb5cdad19..436d761fdfa 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -7,14 +7,13 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -echo '+++ :md: Attaching Documentation' export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' elif [[ "$BUILDKITE" == 'true' ]]; then - printf "Skipping \033]1339;url=$DOCS_URL;content=documentation\a upload for job retry number $RETRY.\n" + printf " # Skipping \033]1339;url=$DOCS_URL;content=documentation\a upload for job retry number $RETRY.\n" fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then From 2e2b1dbbb8e1c25c085c7a1d6f522188eb074c87 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:23:42 -0400 Subject: [PATCH 40/46] Add eosio-debug-build pipeline --- .cicd/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/README.md b/.cicd/README.md index da441f36806..b628784778c 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -90,6 +90,7 @@ Pipeline | Details [eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | run [eos](https://github.com/EOSIO/eos) build scripts nightly on empty operating systems [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit +[eosio-debug-build](https://buildkite.com/EOSIO/eosio-debug-build) | perform a debug build for [eos](https://github.com/EOSIO/eos) on every pull request and base branch commit [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build From 53323023b0707fdd5bd26947781b92251af77f95 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 14 May 2021 21:48:45 -0400 Subject: [PATCH 41/46] Maybe fix YAML syntax? --- .cicd/generate-pipeline.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 436d761fdfa..3de8a081e15 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -12,8 +12,6 @@ export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' -elif [[ "$BUILDKITE" == 'true' ]]; then - printf " # Skipping \033]1339;url=$DOCS_URL;content=documentation\a upload for job retry number $RETRY.\n" fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then @@ -118,6 +116,7 @@ oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) # start with a wait step +echo 'steps:' echo ' - wait' echo '' # build steps From ccba0bac00fc8030c55582848017200a51467f50 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 17 May 2021 18:22:34 -0400 Subject: [PATCH 42/46] Try using relative links instead of hard-coding branch names --- .cicd/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index b628784778c..93ad073ae53 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -7,11 +7,11 @@ The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eo See More ## Index -1. [Configuration](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#configuration) - 1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) - 1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) -1. [Pipelines](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#pipelines) -1. [See Also](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#see-also) +1. [Configuration](README.md#configuration) + 1. [Variables](README.md#variables) + 1. [Examples](README.md#examples) +1. [Pipelines](README.md#pipelines) +1. [See Also](README.md#see-also) ## Configuration Most EOSIO pipelines are run any time you push a commit or tag to an open pull request in [eos](https://github.com/EOSIO/eos), any time you merge a pull request, and nightly. The [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline only runs when you merge a pull request because it takes so long. Long-running tests are also run in the [eosio](https://buildkite.com/EOSIO/eosio) nightly builds, which have `RUN_ALL_TESTS='true'` set. From ef27c704d67a8b4154915e26269534e0debb0a79 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 17 May 2021 18:23:02 -0400 Subject: [PATCH 43/46] Don't forget installation-build.sh --- .cicd/installation-build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/installation-build.sh b/.cicd/installation-build.sh index e35bfb350c8..e2036364903 100755 --- a/.cicd/installation-build.sh +++ b/.cicd/installation-build.sh @@ -3,7 +3,8 @@ set -eo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh export ENABLE_INSTALL='true' -export BRANCH=$(echo $BUILDKITE_BRANCH | sed 's.^/..' | sed 's/[:/]/_/g') +export SANITIZED_BRANCH=$(sanitize "$BUILDKITE_BRANCH") +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-ubuntu-18.04" export ARGS="--name ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER --init -v \"\$(pwd):$MOUNTED_DIR\"" BUILD_COMMAND="'$CICD_DIR/build.sh'" @@ -12,7 +13,7 @@ eval $BUILD_COMMAND echo '+++ :arrow_up: Pushing Container' for REGISTRY in "${CONTRACT_REGISTRIES[@]}"; do if [[ ! -z "$REGISTRY" ]]; then - COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE" "$REGISTRY:base-ubuntu-18.04-$BRANCH-$BUILDKITE_COMMIT") + COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE" "$REGISTRY:base-ubuntu-18.04-$SANITIZED_BRANCH-$BUILDKITE_COMMIT") for COMMIT in "${COMMITS[@]}"; do COMMIT_COMMAND="docker commit 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER' '$COMMIT'" echo "$ $COMMIT_COMMAND" From 75db73fc1ba1e0c351e7de654be31c2e1fd1a14f Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 17 May 2021 18:31:28 -0400 Subject: [PATCH 44/46] Better support for arbitrary branch names in submodule-regression-check.sh --- .cicd/submodule-regression-check.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index 35519a8a5c4..be5461e69ca 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -19,15 +19,18 @@ while read -r a b; do done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "getting submodule info for $BASE_BRANCH" -git checkout $BASE_BRANCH 1> /dev/null -git submodule update --init 1> /dev/null +GIT_CHECKOUT="git checkout --recursive '$BASE_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT + while read -r a b; do BASE_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "switching back to $CURRENT_BRANCH..." -echo "git checkout -qf $CURRENT_BRANCH" -git checkout -qf $CURRENT_BRANCH 1> /dev/null +GIT_CHECKOUT="git checkout -qf '$CURRENT_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT for k in "${!BASE_MAP[@]}"; do base_ts=${BASE_MAP[$k]} @@ -37,11 +40,12 @@ for k in "${!BASE_MAP[@]}"; do echo " timestamp on $BASE_BRANCH: $base_ts" if (( $pr_ts < $base_ts)); then echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating the difference between $CURRENT_BRANCH and $BASE_BRANCH to look for $k changing..." - if [[ ! -z $(for c in $(git --no-pager log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then + GIT_LOG="git --no-pager log '$CURRENT_BRANCH' '^$BASE_BRANCH' --pretty=format:\"%H\"" + if [[ ! -z $(for c in $(eval $GIT_LOG); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then echo "ERROR: $k has regressed" exit 1 else echo "$k was not in the diff; no regression detected" fi fi -done \ No newline at end of file +done From 6f4e8f4480a041939cf1605cc1dcec7c78979da8 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 17 May 2021 19:23:15 -0400 Subject: [PATCH 45/46] Fix bug I introduced to submodule regression check --- .cicd/submodule-regression-check.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index be5461e69ca..43e5af69803 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -19,9 +19,12 @@ while read -r a b; do done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "getting submodule info for $BASE_BRANCH" -GIT_CHECKOUT="git checkout --recursive '$BASE_BRANCH' 1> /dev/null" +GIT_CHECKOUT="git checkout '$BASE_BRANCH' 1> /dev/null" echo "$ $GIT_CHECKOUT" eval $GIT_CHECKOUT +GIT_SUBMODULE="git submodule update --init 1> /dev/null" +echo "$ $GIT_SUBMODULE" +eval $GIT_SUBMODULE while read -r a b; do BASE_MAP[$a]=$b From 4b0c57f79ba50770a64a9dda7f735c8d81805510 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 18 May 2021 10:12:53 -0400 Subject: [PATCH 46/46] Bring in lib --- .cicd/create-docker-from-binary.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 5374f19012a..42bbfc47569 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -1,6 +1,7 @@ #!/bin/bash echo '--- :evergreen_tree: Configuring Environment' set -euo pipefail +. ./.cicd/helpers/general.sh buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."