Travis CI Alpha Testing

.cicd/.helpers

@@ -0,0 +1,49 @@
+DRYRUN=${DRYRUN:-false}
+VERBOSE=${VERBOSE:-true}
+export PROJECT_NAME="eosio"
+
+# Set IMAGE_TAG using arg (generate-base-images.sh) or env (travis.yml)
+( [[ $(uname) != 'Darwin' ]] && [[ -z $IMAGE_TAG ]] && [[ -z $1 ]] ) && echo "You must provide the distro IMAGE_TAG name (example: ubuntu-18.04) as argument \$1 or set it within your ENV" && exit 1
+export IMAGE_TAG=${IMAGE_TAG:-$1}
+
+function execute() {
+  $VERBOSE && echo "--- Executing: $@"
+  $DRYRUN || "$@"
+}
+
+function determine-hash() {
+    # Determine the sha1 hash of all dockerfiles in the .cicd directory.
+    [[ -z $1 ]] && echo "Please provide the files to be hashed (wildcards supported)" && exit 1
+    echo "Obtaining Hash of files from $1..."
+    # Collect all files, hash them, then hash those.
+    HASHES=()
+    for FILE in $(find $1 -type f); do
+        HASH=$(sha1sum $FILE | sha1sum | awk '{ print $1 }')
+        HASHES=($HASH "${HASHES[*]}")
+        echo "$FILE - $HASH"
+    done
+    export DETERMINED_HASH=$(echo ${HASHES[*]} | sha1sum | awk '{ print $1 }')
+    export HASHED_IMAGE_TAG="${IMAGE_TAG}-${DETERMINED_HASH}"
+}
+
+function generate_docker_image() {
+    # If we cannot pull the image, we build and push it first.
+    docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
+    cd ./.cicd
+    docker build -t $FULL_TAG -f ./${IMAGE_TAG}.dockerfile .
+    docker push $FULL_TAG
+    cd -
+}
+
+function docker_tag_exists() {
+    ORG_REPO=$(echo $1 | cut -d: -f1)
+    TAG=$(echo $1 | cut -d: -f2)
+    EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
+    ( [[ $EXISTS =~ '404 page not found' ]] || [[ $EXISTS =~ 'manifest unknown' ]] ) && return 1 || return 0
+}
+
+if [[ $(uname) != 'Darwin' ]]; then # Darwin doesn't need hash (yet)
+    determine-hash ".cicd/${IMAGE_TAG}.dockerfile"
+    [[ -z $DETERMINED_HASH ]] && echo "DETERMINED_HASH empty! (check determine-hash function)" && exit 1
+    export FULL_TAG="eosio/producer:${PROJECT_NAME}-$HASHED_IMAGE_TAG"
+fi

.cicd/amazonlinux-2.dockerfile

@@ -0,0 +1,82 @@
+FROM amazonlinux:2
+
+# YUM dependencies.
+RUN yum update -y \
+  && yum install -y which git sudo procps-ng util-linux autoconf automake \
+  libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \
+  libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \
+  graphviz clang patch
+
+# Build appropriate version of CMake.
+RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \
+  && tar -xzf cmake-3.13.2.tar.gz \
+  && cd cmake-3.13.2 \
+  && ./bootstrap --prefix=/usr/local \
+  && make -j$(nproc) \
+  && make install \
+  && cd .. \
+  && rm -f cmake-3.13.2.tar.gz
+
+# Build appropriate version of LLVM.
+RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm \
+  && cd llvm \
+  && mkdir build \
+  && cd build \
+  && cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd /
+
+# Build appropriate version of Boost.
+RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \
+  && tar -xjf boost_1_70_0.tar.bz2 \
+  && cd boost_1_70_0 \
+  && ./bootstrap.sh --prefix=/usr/local \
+  && ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install \
+  && cd .. \
+  && rm -f boost_1_70_0.tar.bz2    
+
+# Build appropriate version of MongoDB.
+RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz \
+  && tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz \
+  && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz
+
+# Build appropriate version of MongoDB C Driver. 
+RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \
+  && tar -xzf mongo-c-driver-1.13.0.tar.gz \
+  && cd mongo-c-driver-1.13.0 \
+  && mkdir -p build \
+  && cd build \
+  && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd / \
+  && rm -rf mongo-c-driver-1.13.0.tar.gz
+
+# Build appropriate version of MongoDB C++ driver.
+RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \
+  && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \
+  && cd mongo-cxx-driver-r3.4.0 \
+  && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast<int64_t>(count)/' src/mongocxx/options/change_stream.cpp \
+  && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \
+  && cd build \
+  && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd / \
+  && rm -f mongo-cxx-driver-r3.4.0.tar.gz
+
+ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin
+
+# CCACHE
+RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm \
+  && yum install -y ccache-3.3.4-1.el7.x86_64.rpm
+
+# PRE_COMMANDS: Executed pre-cmake
+# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
+ENV PRE_COMMANDS="export PATH=/usr/lib64/ccache:$PATH &&"
+ENV CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'"
+
+CMD bash -c "$PRE_COMMANDS ccache -s && \
+    mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
+    ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"

.cicd/base-images.yml

@@ -0,0 +1,38 @@
+env:
+  BUILD_TIMEOUT: 120
+  TEST_TIMEOUT: 60
+  TIMEOUT: 120
+
+steps:
+
+  - label: ":aws: [Amazon] 2 Ensure Docker Image"
+    command:
+      - ".cicd/generate-base-images.sh amazonlinux-2"
+    agents:
+      queue: "automation-eos-dockerhub-image-builder-fleet"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_AMAZON_LINUX_2
+
+  - label: ":centos: [CentOS] 7 Build"
+    command:
+      - ".cicd/generate-base-images.sh centos-7"
+    agents:
+      queue: "automation-eos-dockerhub-image-builder-fleet"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_CENTOS_7
+
+  - label: ":ubuntu: [Ubuntu] 16.04 Build"
+    command:
+      - ".cicd/generate-base-images.sh ubuntu-16.04"
+    agents:
+      queue: "automation-eos-dockerhub-image-builder-fleet"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_UBUNTU_16
+
+  - label: ":ubuntu: [Ubuntu] 18.04 Build"
+    command:
+      - ".cicd/generate-base-images.sh ubuntu-18.04"
+    agents:
+      queue: "automation-eos-dockerhub-image-builder-fleet"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_UBUNTU_18

.cicd/centos-7.dockerfile

@@ -0,0 +1,97 @@
+FROM centos:7
+
+# YUM dependencies.
+RUN yum update -y \
+  && yum --enablerepo=extras install -y centos-release-scl \
+  && yum --enablerepo=extras install -y devtoolset-8 \
+  && yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \
+  graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \
+  python python-devel rh-python36 gettext-devel file libusbx-devel \
+  libcurl-devel patch 
+
+# Build appropriate version of CMake.
+RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \
+  && source /opt/rh/devtoolset-8/enable \
+  && source /opt/rh/rh-python36/enable \
+  && tar -xzf cmake-3.13.2.tar.gz \
+  && cd cmake-3.13.2 \
+  && ./bootstrap --prefix=/usr/local \
+  && make -j$(nproc) \
+  && make install \
+  && cd .. \
+  && rm -f cmake-3.13.2.tar.gz
+
+# Build appropriate version of LLVM.
+RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm \
+  && source /opt/rh/devtoolset-8/enable \
+  && source /opt/rh/rh-python36/enable \
+  && cd llvm \
+  && mkdir build \
+  && cd build \
+  && cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd /
+
+# Build appropriate version of Boost.
+RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \
+  && source /opt/rh/devtoolset-8/enable \
+  && source /opt/rh/rh-python36/enable \
+  && tar -xjf boost_1_70_0.tar.bz2 \
+  && cd boost_1_70_0 \
+  && ./bootstrap.sh --prefix=/usr/local \
+  && ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install \
+  && cd .. \
+  && rm -f boost_1_70_0.tar.bz2    
+
+# Build appropriate version of MongoDB.
+RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz \
+  && tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz \
+  && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz
+
+# Build appropriate version of MongoDB C Driver. 
+RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \
+  && source /opt/rh/devtoolset-8/enable \
+  && source /opt/rh/rh-python36/enable \
+  && tar -xzf mongo-c-driver-1.13.0.tar.gz \
+  && cd mongo-c-driver-1.13.0 \
+  && mkdir -p build \
+  && cd build \
+  && cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd / \
+  && rm -rf mongo-c-driver-1.13.0.tar.gz
+
+# Build appropriate version of MongoDB C++ driver.
+RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \
+  && source /opt/rh/devtoolset-8/enable \
+  && source /opt/rh/rh-python36/enable \
+  && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \
+  && cd mongo-cxx-driver-r3.4.0 \
+  && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast<int64_t>(count)/' src/mongocxx/options/change_stream.cpp \
+  && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \
+  && cd build \
+  && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
+  && make -j$(nproc) \
+  && make install \
+  && cd / \
+  && rm -f mongo-cxx-driver-r3.4.0.tar.gz
+
+ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin
+
+# CCACHE
+RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm \
+  && yum install -y ccache-3.3.4-1.el7.x86_64.rpm
+## Needed as devtoolset uses c++ and it's not in ccache by default
+RUN cd /usr/lib64/ccache && ln -s ../../bin/ccache c++
+## We need to tell ccache to actually use devtoolset-8 instead of the default system one (ccache resets anything set in PATH when it launches)
+ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin"
+
+# PRE_COMMANDS: Executed pre-cmake
+# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
+ENV PRE_COMMANDS="source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:$PATH &&"
+
+CMD bash -c "$PRE_COMMANDS ccache -s && \
+    mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
+    ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"

.cicd/generate-base-images.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eo pipefail
+cd $( dirname "${BASH_SOURCE[0]}" )/.. # Ensure we're in the repo root and not inside of scripts
+. ./.cicd/.helpers
+echo "Looking for $FULL_TAG"
+docker_tag_exists $FULL_TAG && echo "$FULL_TAG already exists" || generate_docker_image

.cicd/pinned_toolchain.cmake

@@ -0,0 +1,14 @@
+set(CMAKE_C_COMPILER_WORKS 1)
+set(CMAKE_CXX_COMPILER_WORKS 1)
+set(CMAKE_C_COMPILER /usr/local/bin/clang)
+set(CMAKE_CXX_COMPILER /usr/local/bin/clang++)
+
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES /usr/local/include/c++/v1 /usr/local/include /usr/include)
+
+set(CMAKE_CXX_FLAGS_INIT "-nostdinc++")
+
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+set(CMAKE_MODULE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+
+set(CMAKE_CXX_STANDARD_LIBRARIES "/usr/local/lib/libc++.a /usr/local/lib/libc++abi.a")

.cicd/pipeline.yml

@@ -0,0 +1,80 @@
+env:
+  BUILD_TIMEOUT: 120
+  TEST_TIMEOUT: 60
+  TIMEOUT: 120
+  VERBOSE: true
+
+steps:
+
+  - trigger: "eosio-base-images-beta"
+    label: ":docker: Ensure base images exist"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+
+  - wait
+
+  - label: ":aws: [Amazon] 2 Build"
+    agents:
+      queue: "automation-eos-builder-fleet"
+    plugins:
+      - docker#v3.2.0:
+          debug: $DEBUG
+          image: "eosio/producer:eosio-amazonlinux-2-1c266e25276ad1f1147a4df5c12921968079c49b"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_AMAZON_LINUX_2
+
+  - label: ":centos: [CentOS] 7 Build"
+    command:
+      - ""
+    agents:
+      queue: "automation-eos-builder-fleet"
+    plugins:
+      - docker#v3.2.0:
+          debug: $DEBUG
+          image: "eosio/producer:eosio-centos-7-d45789dfb1a5b830a62381afd205b34d3d7c2d63"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_CENTOS_7
+
+  - label: ":ubuntu: [Ubuntu] 16.04 Build"
+    agents:
+      queue: "automation-eos-builder-fleet"
+    plugins:
+      - docker#v3.2.0:
+          debug: $DEBUG
+          image: "eosio/producer:eosio-ubuntu-16.04-4f430ef21e359ef2581c25255c99e5411adfaed9"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_UBUNTU_16
+
+  - label: ":ubuntu: [Ubuntu] 18.04 Build"
+    agents:
+      queue: "automation-eos-builder-fleet"
+    plugins:
+      - docker#v3.2.0:
+          debug: $DEBUG
+          image: "eosio/producer:eosio-ubuntu-18.04-14091ac0e5618b0ea5ce026cd75b648efbde4be7"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_UBUNTU_18
+
+  - label: ":darwin: [Darwin] 10.14 Build"
+    command:
+      - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl [email protected] cmake"
+      - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "cd eos && mkdir build && cd build && cmake .. && make -j$(getconf _NPROCESSORS_ONLN)"
+      - "cd eos/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
+    plugins:
+      - chef/anka#v0.5.1:
+          no-volume: true
+          inherit-environment-vars: true
+          vm-name: 10.14.4_6C_14G_40G
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew"
+          modify-cpu: 12
+          modify-ram: 24
+          always-pull: true
+          debug: true
+          wait-network: true
+    agents:
+      - "queue=mac-anka-large-node-fleet"
+    timeout: $BUILD_TIMEOUT
+    skip: $SKIP_MOJAVE
+