diff --git a/.cicd/README.md b/.cicd/README.md index 95bf71a27db..93ad073ae53 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -1,53 +1,61 @@ # eosio -The [eosio](https://buildkite.com/EOSIO/eosio) pipeline is the primary CI/CD pipeline for members of the EOSIO organization developing on the EOSIO/eos repository. The [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipeline is also executed regularly. The difference in these two pipelines is whether the compiler and other dependencies are pinned to specific versions. The eosio pipeline uses pinned compilers/dependencies while the eosio-build-unpinned pipeline avoids pinning dependencies as much as possible. +The [eosio](https://buildkite.com/EOSIO/eosio) and [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipelines are the primary pipelines for the [eos](https://github.com/EOSIO/eos) repository, running with specific or default versions of our dependencies, respectively. Both run against every commit to a base branch or pull request, along with the [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) pipeline. -## See Also -- [Buildkite Documentation](https://github.com/EOSIO/devdocs/wiki/Buildkite) (internal) -- [EOSIO Resume from State Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) (internal) -- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) (internal) +The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) and [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) pipelines on each build, and the the [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline on merge commits. Each of these pipelines are described in more detail below and in their respective READMEs.
-More Info +See More ## Index -1. [Variables](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#variables) -1. [Examples](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#examples) -1. [Pipelines](https://github.com/EOSIO/eos/blob/develop/.cicd/README.md#pipelines) +1. [Configuration](README.md#configuration) + 1. [Variables](README.md#variables) + 1. [Examples](README.md#examples) +1. [Pipelines](README.md#pipelines) +1. [See Also](README.md#see-also) + +## Configuration +Most EOSIO pipelines are run any time you push a commit or tag to an open pull request in [eos](https://github.com/EOSIO/eos), any time you merge a pull request, and nightly. The [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline only runs when you merge a pull request because it takes so long. Long-running tests are also run in the [eosio](https://buildkite.com/EOSIO/eosio) nightly builds, which have `RUN_ALL_TESTS='true'` set. ### Variables Most pipelines in the organization have several environment variables that can be used to configure how the pipeline runs. These environment variables can be specified when manually triggering a build via the Buildkite UI. +Configure which platforms are run: +```bash +SKIP_LINUX='true|false' # skip all steps on Linux distros +SKIP_MAC='true|false' # skip all steps on Mac hardware +``` +These will override more specific operating system declarations, and primarily exist to disable one of our two buildfleets should one be sick or the finite macOS agents are congested. + Configure which operating systems are built, tested, and packaged: ```bash -SKIP_LINUX='true|false' # true skips all build/test/packaging steps on Linux distros -SKIP_MAC='true|false' # true skips all build/test/packaging steps on Mac hardware -SKIP_AMAZON_LINUX_2='true|false' # true skips all build/test/packaging steps for Amazon Linux 2 -SKIP_CENTOS_7_7='true|false' # true skips all build/test/packaging steps for Centos 7 -SKIP_CENTOS_8='true|false' # true skips all build/test/packaging steps for Centos 8 -SKIP_MACOS_10_14='true|false' # true skips all build/test/packaging steps for MacOS 10.14 -SKIP_MACOS_10_15='true|false' # true skips all build/test/packaging steps for MacOS 10.15 -SKIP_MACOS_11='true|false' # true skips all build/test/packaging steps for MacOS 11 -SKIP_UBUNTU_16_04='true|false' # true skips all build/test/packaging steps for Ubuntu 16.04 -SKIP_UBUNTU_18_04='true|false' # true skips all build/test/packaging steps for Ubuntu 18.04 -SKIP_UBUNTU_20_04='true|false' # true skips all build/test/packaging steps for Ubuntu 20.04 +RUN_ALL_TESTS='true' # run all tests in the current build (including LRTs, overridden by SKIP* variables) +SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 +SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 +SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 +SKIP_MACOS_10_14='true|false' # skip all steps for MacOS 10.14 +SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 +SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 +SKIP_UBUNTU_16_04='true|false' # skip all steps for Ubuntu 16.04 +SKIP_UBUNTU_18_04='true|false' # skip all steps for Ubuntu 18.04 +SKIP_UBUNTU_20_04='true|false' # skip all steps for Ubuntu 20.04 ``` Configure which steps are executed for each operating system: ```bash -SKIP_BUILD='true|false' # true skips all build steps for all distros -SKIP_UNIT_TESTS='true|false' # true skips all unit test executions for all distros -SKIP_WASM_SPEC_TESTS='true|false' # true skips all wasm spec test executions for all distros -SKIP_SERIAL_TESTS='true|false' # true skips all integration test executions for all distros -SKIP_LONG_RUNNING_TESTS='true|false' # true skips all long running test executions for all distros -SKIP_MULTIVERSION_TEST='true|false' # true skips all multiversion tests -SKIP_SYNC_TESTS='true|false' # true skips all sync tests -SKIP_PACKAGE_BUILDER='true|false' # true skips all package building steps for all distros +SKIP_BUILD='true|false' # skip all build steps +SKIP_UNIT_TESTS='true|false' # skip all unit tests +SKIP_WASM_SPEC_TESTS='true|false' # skip all wasm spec tests +SKIP_SERIAL_TESTS='true|false' # skip all integration tests +SKIP_LONG_RUNNING_TESTS='true|false' # skip all long running tests +SKIP_MULTIVERSION_TEST='true|false' # skip all multiversion tests +SKIP_SYNC_TESTS='true|false' # skip all sync tests +SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps ``` Configure how the steps are executed: ```bash -PINNED='true|false' # controls compiler/dependency pinning -TIMEOUT='##' # controls timeout in minutes for all steps +PINNED='true|false' # use specific versions of dependencies instead of whatever version is provided by default on a given platform +TIMEOUT='##' # set timeout in minutes for all steps ``` ### Examples @@ -71,19 +79,27 @@ SKIP_MULTIVERSION_TEST='true' SKIP_SYNC_TESTS='true' ``` -### Pipelines -There are several eosio pipelines that are exist and executed via pull requests, triggered from other builds, or scheduled to run on a regular basis: - +## Pipelines +There are several eosio pipelines that exist and are triggered by pull requests, pipelines, or schedules: Pipeline | Details ---|--- -[eosio](https://buildkite.com/EOSIO/eosio) | Primary pipeline for the EOSIO/eos Github repo. It is triggered when a pull request is created. -[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | Pipeline that performs a build without a pinned compiler. It is triggered when a pull request is created. -[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | Pipeline that only executes the long running tests. It is triggered after a pull request is merged. -[eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | Pipeline that ensures all MacOS VM and Docker container builders can be built. It is scheduled for periodic execution. -[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | Pipeline that ensure the build scripts function. It is scheduled for periodic execution. -[eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | Pipeline that performs a build only using MacOS 11 builders. It is scheduled for periodic execution. -[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | Pipeline that ensures built code can sync properly. It is triggered during pull request builds. -[eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | Pipeline that ensures that built binaries can resume from previous binary versions. It is triggered during pull request builds. +[eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly +[eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | pack EOSIO dependencies into docker and Anka base-images nightly +[eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | build and test [eos](https://github.com/EOSIO/eos) on macOS 11 "Big Sur" weekly +[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | run [eos](https://github.com/EOSIO/eos) build scripts nightly on empty operating systems +[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly +[eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit +[eosio-debug-build](https://buildkite.com/EOSIO/eosio-debug-build) | perform a debug build for [eos](https://github.com/EOSIO/eos) on every pull request and base branch commit +[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits +[eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) +[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build + +## See Also +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel
diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index aa58f3e1b38..42bbfc47569 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -1,54 +1,65 @@ #!/bin/bash - +echo '--- :evergreen_tree: Configuring Environment' set -euo pipefail - +. ./.cicd/helpers/general.sh buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . -echo ":done: download successful" - -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') -echo "$SANITIZED_BRANCH" -echo "$SANITIZED_TAG" - -# do docker build -echo ":docker::build: Building image..." -DOCKERHUB_REGISTRY="docker.io/eosio/eosio" - -BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest} -DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ." -echo "$ $DOCKER_BUILD_GEN" -eval $DOCKER_BUILD_GEN - -#tag and push on each destination AWS & DOCKERHUB - -EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") -for REG in ${EOSIO_REGS[@]}; do - DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" - DOCKER_TAG_BRANCH="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_BRANCH" - echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH" - eval $DOCKER_TAG_COMMIT +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." +# docker build +echo "+++ :docker: Build Docker Container" +DOCKERHUB_REGISTRY='docker.io/eosio/eosio' +IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" +DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ." +echo "$ $DOCKER_BUILD" +eval $DOCKER_BUILD +# docker tag +echo '--- :label: Tag Container' +REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") +for REG in ${REGISTRIES[@]}; do + DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" + echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH - DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT" - DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH" - echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH" - eval $DOCKER_PUSH_COMMIT - eval $DOCKER_PUSH_BRANCH - CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT || :" - CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH || :" - echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$" - eval $CLEAN_IMAGE_COMMIT - eval $CLEAN_IMAGE_BRANCH - if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" - DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" - DOCKER_REM="docker rmi $REG:$SANITIZED_TAG || :" - echo -e "$ \n Tagging Image: \n$DOCKER_TAG \n Cleaning Up: \n$DOCKER_REM" + DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_TAG_COMMIT" + eval $DOCKER_TAG_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'" + echo "$ $DOCKER_TAG" eval $DOCKER_TAG + fi +done +# docker push +echo '--- :arrow_up: Push Container' +for REG in ${REGISTRIES[@]}; do + DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'" + echo "$ $DOCKER_PUSH_BRANCH" + eval $DOCKER_PUSH_BRANCH + DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_PUSH_COMMIT" + eval $DOCKER_PUSH_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'" + echo "$ $DOCKER_PUSH_TAG" eval $DOCKER_PUSH_TAG - eval $DOCKER_REM fi done - -DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG || :" -echo "Clean up base image" -eval $DOCKER_GEN \ No newline at end of file +# docker rmi +echo '--- :put_litter_in_its_place: Cleanup' +for REG in ${REGISTRIES[@]}; do + CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :" + echo "$ $CLEAN_IMAGE_BRANCH" + eval $CLEAN_IMAGE_BRANCH + CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" + echo "$ $CLEAN_IMAGE_COMMIT" + eval $CLEAN_IMAGE_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_RMI="docker rmi '$REG:$SANITIZED_TAG' || :" + echo "$ $DOCKER_RMI" + eval $DOCKER_RMI + fi +done +DOCKER_RMI="docker rmi '$IMAGE' || :" +echo "$ $DOCKER_RMI" +eval $DOCKER_RMI diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 85142e60b52..33211ac88d8 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -3,8 +3,10 @@ set -eo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' @@ -23,7 +25,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND @@ -38,7 +40,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND @@ -56,7 +58,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8ae93751f5c..3de8a081e15 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -6,8 +6,13 @@ export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' -# Add build annotation -cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' +# attach pipeline documentation +export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) @@ -111,6 +116,7 @@ oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) # start with a wait step +echo 'steps:' echo ' - wait' echo '' # build steps diff --git a/.cicd/helpers/general.sh b/.cicd/helpers/general.sh index 77f836c92c1..4de9bb626b2 100644 --- a/.cicd/helpers/general.sh +++ b/.cicd/helpers/general.sh @@ -31,3 +31,15 @@ function buildkite-intrinsics() fi echo "$BK_ENV" } + + ##### sanitize branch names for use in URIs (docker containers) ##### +# tr '/' '_' # convert forward-slashes '/' to underscores '_' +# sed -E 's/[^-_.a-zA-Z0-9]+/-/g' # convert invalid docker chars to '-' +# sed -E 's/-+/-/g' # replace multiple dashes in a series "----" with a single dash '-' +# sed -E 's/-*_+-*/_/g' # replace dashes '-' and underscores '_' in-series with a single underscore '_' +# sed -E 's/_+/_/g' # replace multiple underscores in a row "___" with a single underscore '_' +# sed -E 's/(^[-_.]+|[-_.]+$)//g' # ensure tags do not begin or end with separator characters [-_.] +function sanitize() +{ + echo "$1" | tr '/' '_' | sed -E 's/[^-_.a-zA-Z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/-*_+-*/_/g' | sed -E 's/_+/_/g' | sed -E 's/(^[-_.]+|[-_.]+$)//g' +} diff --git a/.cicd/installation-build.sh b/.cicd/installation-build.sh index e35bfb350c8..e2036364903 100755 --- a/.cicd/installation-build.sh +++ b/.cicd/installation-build.sh @@ -3,7 +3,8 @@ set -eo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh export ENABLE_INSTALL='true' -export BRANCH=$(echo $BUILDKITE_BRANCH | sed 's.^/..' | sed 's/[:/]/_/g') +export SANITIZED_BRANCH=$(sanitize "$BUILDKITE_BRANCH") +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-ubuntu-18.04" export ARGS="--name ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER --init -v \"\$(pwd):$MOUNTED_DIR\"" BUILD_COMMAND="'$CICD_DIR/build.sh'" @@ -12,7 +13,7 @@ eval $BUILD_COMMAND echo '+++ :arrow_up: Pushing Container' for REGISTRY in "${CONTRACT_REGISTRIES[@]}"; do if [[ ! -z "$REGISTRY" ]]; then - COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE" "$REGISTRY:base-ubuntu-18.04-$BRANCH-$BUILDKITE_COMMIT") + COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE" "$REGISTRY:base-ubuntu-18.04-$SANITIZED_BRANCH-$BUILDKITE_COMMIT") for COMMIT in "${COMMITS[@]}"; do COMMIT_COMMAND="docker commit 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER' '$COMMIT'" echo "$ $COMMIT_COMMAND" diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index 35519a8a5c4..43e5af69803 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -19,15 +19,21 @@ while read -r a b; do done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "getting submodule info for $BASE_BRANCH" -git checkout $BASE_BRANCH 1> /dev/null -git submodule update --init 1> /dev/null +GIT_CHECKOUT="git checkout '$BASE_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT +GIT_SUBMODULE="git submodule update --init 1> /dev/null" +echo "$ $GIT_SUBMODULE" +eval $GIT_SUBMODULE + while read -r a b; do BASE_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "switching back to $CURRENT_BRANCH..." -echo "git checkout -qf $CURRENT_BRANCH" -git checkout -qf $CURRENT_BRANCH 1> /dev/null +GIT_CHECKOUT="git checkout -qf '$CURRENT_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT for k in "${!BASE_MAP[@]}"; do base_ts=${BASE_MAP[$k]} @@ -37,11 +43,12 @@ for k in "${!BASE_MAP[@]}"; do echo " timestamp on $BASE_BRANCH: $base_ts" if (( $pr_ts < $base_ts)); then echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating the difference between $CURRENT_BRANCH and $BASE_BRANCH to look for $k changing..." - if [[ ! -z $(for c in $(git --no-pager log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then + GIT_LOG="git --no-pager log '$CURRENT_BRANCH' '^$BASE_BRANCH' --pretty=format:\"%H\"" + if [[ ! -z $(for c in $(eval $GIT_LOG); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then echo "ERROR: $k has regressed" exit 1 else echo "$k was not in the diff; no regression detected" fi fi -done \ No newline at end of file +done