diff --git a/.cicd/.helpers b/.cicd/.helpers new file mode 100644 index 00000000000..511b50d7456 --- /dev/null +++ b/.cicd/.helpers @@ -0,0 +1,49 @@ +DRYRUN=${DRYRUN:-false} +VERBOSE=${VERBOSE:-true} +export PROJECT_NAME="eosio" + +# Set IMAGE_TAG using arg (generate-base-images.sh) or env (travis.yml) +( [[ $(uname) != 'Darwin' ]] && [[ -z $IMAGE_TAG ]] && [[ -z $1 ]] ) && echo "You must provide the distro IMAGE_TAG name (example: ubuntu-18.04) as argument \$1 or set it within your ENV" && exit 1 +export IMAGE_TAG=${IMAGE_TAG:-$1} + +function execute() { + $VERBOSE && echo "--- Executing: $@" + $DRYRUN || "$@" +} + +function determine-hash() { + # Determine the sha1 hash of all dockerfiles in the .cicd directory. + [[ -z $1 ]] && echo "Please provide the files to be hashed (wildcards supported)" && exit 1 + echo "Obtaining Hash of files from $1..." + # Collect all files, hash them, then hash those. + HASHES=() + for FILE in $(find $1 -type f); do + HASH=$(sha1sum $FILE | sha1sum | awk '{ print $1 }') + HASHES=($HASH "${HASHES[*]}") + echo "$FILE - $HASH" + done + export DETERMINED_HASH=$(echo ${HASHES[*]} | sha1sum | awk '{ print $1 }') + export HASHED_IMAGE_TAG="${IMAGE_TAG}-${DETERMINED_HASH}" +} + +function generate_docker_image() { + # If we cannot pull the image, we build and push it first. + docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + cd ./.cicd + docker build -t $FULL_TAG -f ./${IMAGE_TAG}.dockerfile . + docker push $FULL_TAG + cd - +} + +function docker_tag_exists() { + ORG_REPO=$(echo $1 | cut -d: -f1) + TAG=$(echo $1 | cut -d: -f2) + EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG") + ( [[ $EXISTS =~ '404 page not found' ]] || [[ $EXISTS =~ 'manifest unknown' ]] ) && return 1 || return 0 +} + +if [[ $(uname) != 'Darwin' ]]; then # Darwin doesn't need hash (yet) + determine-hash ".cicd/${IMAGE_TAG}.dockerfile" + [[ -z $DETERMINED_HASH ]] && echo "DETERMINED_HASH empty! (check determine-hash function)" && exit 1 + export FULL_TAG="eosio/producer:${PROJECT_NAME}-$HASHED_IMAGE_TAG" +fi \ No newline at end of file diff --git a/.cicd/amazonlinux-2.dockerfile b/.cicd/amazonlinux-2.dockerfile new file mode 100644 index 00000000000..5bdc15d98c5 --- /dev/null +++ b/.cicd/amazonlinux-2.dockerfile @@ -0,0 +1,82 @@ +FROM amazonlinux:2 + +# YUM dependencies. +RUN yum update -y \ + && yum install -y which git sudo procps-ng util-linux autoconf automake \ + libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ + libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ + graphviz clang patch + +# Build appropriate version of CMake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \ + && tar -xzf cmake-3.13.2.tar.gz \ + && cd cmake-3.13.2 \ + && ./bootstrap --prefix=/usr/local \ + && make -j$(nproc) \ + && make install \ + && cd .. \ + && rm -f cmake-3.13.2.tar.gz + +# Build appropriate version of LLVM. +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm \ + && cd llvm \ + && mkdir build \ + && cd build \ + && cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ + && make -j$(nproc) \ + && make install \ + && cd / + +# Build appropriate version of Boost. +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \ + && tar -xjf boost_1_70_0.tar.bz2 \ + && cd boost_1_70_0 \ + && ./bootstrap.sh --prefix=/usr/local \ + && ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install \ + && cd .. \ + && rm -f boost_1_70_0.tar.bz2 + +# Build appropriate version of MongoDB. +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz \ + && tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz \ + && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz + +# Build appropriate version of MongoDB C Driver. +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \ + && tar -xzf mongo-c-driver-1.13.0.tar.gz \ + && cd mongo-c-driver-1.13.0 \ + && mkdir -p build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -rf mongo-c-driver-1.13.0.tar.gz + +# Build appropriate version of MongoDB C++ driver. +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \ + && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \ + && cd mongo-cxx-driver-r3.4.0 \ + && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp \ + && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \ + && cd build \ + && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -f mongo-cxx-driver-r3.4.0.tar.gz + +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin + +# CCACHE +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm \ + && yum install -y ccache-3.3.4-1.el7.x86_64.rpm + +# PRE_COMMANDS: Executed pre-cmake +# CMAKE_EXTRAS: Executed right before the cmake path (on the end) +ENV PRE_COMMANDS="export PATH=/usr/lib64/ccache:$PATH &&" +ENV CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + +CMD bash -c "$PRE_COMMANDS ccache -s && \ + mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \ + ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test" \ No newline at end of file diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml new file mode 100644 index 00000000000..6485f2319c4 --- /dev/null +++ b/.cicd/base-images.yml @@ -0,0 +1,38 @@ +env: + BUILD_TIMEOUT: 120 + TEST_TIMEOUT: 60 + TIMEOUT: 120 + +steps: + + - label: ":aws: [Amazon] 2 Ensure Docker Image" + command: + - ".cicd/generate-base-images.sh amazonlinux-2" + agents: + queue: "automation-eos-dockerhub-image-builder-fleet" + timeout: $BUILD_TIMEOUT + skip: $SKIP_AMAZON_LINUX_2 + + - label: ":centos: [CentOS] 7 Build" + command: + - ".cicd/generate-base-images.sh centos-7" + agents: + queue: "automation-eos-dockerhub-image-builder-fleet" + timeout: $BUILD_TIMEOUT + skip: $SKIP_CENTOS_7 + + - label: ":ubuntu: [Ubuntu] 16.04 Build" + command: + - ".cicd/generate-base-images.sh ubuntu-16.04" + agents: + queue: "automation-eos-dockerhub-image-builder-fleet" + timeout: $BUILD_TIMEOUT + skip: $SKIP_UBUNTU_16 + + - label: ":ubuntu: [Ubuntu] 18.04 Build" + command: + - ".cicd/generate-base-images.sh ubuntu-18.04" + agents: + queue: "automation-eos-dockerhub-image-builder-fleet" + timeout: $BUILD_TIMEOUT + skip: $SKIP_UBUNTU_18 diff --git a/.cicd/centos-7.dockerfile b/.cicd/centos-7.dockerfile new file mode 100644 index 00000000000..a0a342c8857 --- /dev/null +++ b/.cicd/centos-7.dockerfile @@ -0,0 +1,97 @@ +FROM centos:7 + +# YUM dependencies. +RUN yum update -y \ + && yum --enablerepo=extras install -y centos-release-scl \ + && yum --enablerepo=extras install -y devtoolset-8 \ + && yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ + python python-devel rh-python36 gettext-devel file libusbx-devel \ + libcurl-devel patch + +# Build appropriate version of CMake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \ + && source /opt/rh/devtoolset-8/enable \ + && source /opt/rh/rh-python36/enable \ + && tar -xzf cmake-3.13.2.tar.gz \ + && cd cmake-3.13.2 \ + && ./bootstrap --prefix=/usr/local \ + && make -j$(nproc) \ + && make install \ + && cd .. \ + && rm -f cmake-3.13.2.tar.gz + +# Build appropriate version of LLVM. +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm \ + && source /opt/rh/devtoolset-8/enable \ + && source /opt/rh/rh-python36/enable \ + && cd llvm \ + && mkdir build \ + && cd build \ + && cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ + && make -j$(nproc) \ + && make install \ + && cd / + +# Build appropriate version of Boost. +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \ + && source /opt/rh/devtoolset-8/enable \ + && source /opt/rh/rh-python36/enable \ + && tar -xjf boost_1_70_0.tar.bz2 \ + && cd boost_1_70_0 \ + && ./bootstrap.sh --prefix=/usr/local \ + && ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install \ + && cd .. \ + && rm -f boost_1_70_0.tar.bz2 + +# Build appropriate version of MongoDB. +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz \ + && tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz \ + && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz + +# Build appropriate version of MongoDB C Driver. +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \ + && source /opt/rh/devtoolset-8/enable \ + && source /opt/rh/rh-python36/enable \ + && tar -xzf mongo-c-driver-1.13.0.tar.gz \ + && cd mongo-c-driver-1.13.0 \ + && mkdir -p build \ + && cd build \ + && cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -rf mongo-c-driver-1.13.0.tar.gz + +# Build appropriate version of MongoDB C++ driver. +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \ + && source /opt/rh/devtoolset-8/enable \ + && source /opt/rh/rh-python36/enable \ + && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \ + && cd mongo-cxx-driver-r3.4.0 \ + && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp \ + && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \ + && cd build \ + && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -f mongo-cxx-driver-r3.4.0.tar.gz + +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin + +# CCACHE +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm \ + && yum install -y ccache-3.3.4-1.el7.x86_64.rpm +## Needed as devtoolset uses c++ and it's not in ccache by default +RUN cd /usr/lib64/ccache && ln -s ../../bin/ccache c++ +## We need to tell ccache to actually use devtoolset-8 instead of the default system one (ccache resets anything set in PATH when it launches) +ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin" + +# PRE_COMMANDS: Executed pre-cmake +# CMAKE_EXTRAS: Executed right before the cmake path (on the end) +ENV PRE_COMMANDS="source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:$PATH &&" + +CMD bash -c "$PRE_COMMANDS ccache -s && \ + mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \ + ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test" \ No newline at end of file diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh new file mode 100755 index 00000000000..86fff371438 --- /dev/null +++ b/.cicd/generate-base-images.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eo pipefail +cd $( dirname "${BASH_SOURCE[0]}" )/.. # Ensure we're in the repo root and not inside of scripts +. ./.cicd/.helpers +echo "Looking for $FULL_TAG" +docker_tag_exists $FULL_TAG && echo "$FULL_TAG already exists" || generate_docker_image diff --git a/.cicd/pinned_toolchain.cmake b/.cicd/pinned_toolchain.cmake new file mode 100644 index 00000000000..70d0b114ae9 --- /dev/null +++ b/.cicd/pinned_toolchain.cmake @@ -0,0 +1,14 @@ +set(CMAKE_C_COMPILER_WORKS 1) +set(CMAKE_CXX_COMPILER_WORKS 1) +set(CMAKE_C_COMPILER /usr/local/bin/clang) +set(CMAKE_CXX_COMPILER /usr/local/bin/clang++) + +set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES /usr/local/include/c++/v1 /usr/local/include /usr/include) + +set(CMAKE_CXX_FLAGS_INIT "-nostdinc++") + +set(CMAKE_EXE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++") +set(CMAKE_SHARED_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++") +set(CMAKE_MODULE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++") + +set(CMAKE_CXX_STANDARD_LIBRARIES "/usr/local/lib/libc++.a /usr/local/lib/libc++abi.a") diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml new file mode 100644 index 00000000000..3f588597c0c --- /dev/null +++ b/.cicd/pipeline.yml @@ -0,0 +1,80 @@ +env: + BUILD_TIMEOUT: 120 + TEST_TIMEOUT: 60 + TIMEOUT: 120 + VERBOSE: true + +steps: + + - trigger: "eosio-base-images-beta" + label: ":docker: Ensure base images exist" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + + - wait + + - label: ":aws: [Amazon] 2 Build" + agents: + queue: "automation-eos-builder-fleet" + plugins: + - docker#v3.2.0: + debug: $DEBUG + image: "eosio/producer:eosio-amazonlinux-2-1c266e25276ad1f1147a4df5c12921968079c49b" + timeout: $BUILD_TIMEOUT + skip: $SKIP_AMAZON_LINUX_2 + + - label: ":centos: [CentOS] 7 Build" + command: + - "" + agents: + queue: "automation-eos-builder-fleet" + plugins: + - docker#v3.2.0: + debug: $DEBUG + image: "eosio/producer:eosio-centos-7-d45789dfb1a5b830a62381afd205b34d3d7c2d63" + timeout: $BUILD_TIMEOUT + skip: $SKIP_CENTOS_7 + + - label: ":ubuntu: [Ubuntu] 16.04 Build" + agents: + queue: "automation-eos-builder-fleet" + plugins: + - docker#v3.2.0: + debug: $DEBUG + image: "eosio/producer:eosio-ubuntu-16.04-4f430ef21e359ef2581c25255c99e5411adfaed9" + timeout: $BUILD_TIMEOUT + skip: $SKIP_UBUNTU_16 + + - label: ":ubuntu: [Ubuntu] 18.04 Build" + agents: + queue: "automation-eos-builder-fleet" + plugins: + - docker#v3.2.0: + debug: $DEBUG + image: "eosio/producer:eosio-ubuntu-18.04-14091ac0e5618b0ea5ce026cd75b648efbde4be7" + timeout: $BUILD_TIMEOUT + skip: $SKIP_UBUNTU_18 + + - label: ":darwin: [Darwin] 10.14 Build" + command: + - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl boost@1.70 cmake" + - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive" + - "cd eos && mkdir build && cd build && cmake .. && make -j$(getconf _NPROCESSORS_ONLN)" + - "cd eos/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test" + plugins: + - chef/anka#v0.5.1: + no-volume: true + inherit-environment-vars: true + vm-name: 10.14.4_6C_14G_40G + vm-registry-tag: "clean::cicd::git-ssh::nas::brew" + modify-cpu: 12 + modify-ram: 24 + always-pull: true + debug: true + wait-network: true + agents: + - "queue=mac-anka-large-node-fleet" + timeout: $BUILD_TIMEOUT + skip: $SKIP_MOJAVE + \ No newline at end of file diff --git a/.cicd/travis-build.sh b/.cicd/travis-build.sh new file mode 100755 index 00000000000..d12ba8ebb05 --- /dev/null +++ b/.cicd/travis-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eo pipefail +cd $( dirname "${BASH_SOURCE[0]}" )/.. # Ensure we're in the repo root and not inside of scripts +. ./.cicd/.helpers + +CPU_CORES=$(getconf _NPROCESSORS_ONLN) +if [[ "$(uname)" == Darwin ]]; then + echo 'Detected Darwin, building natively.' + [[ -d eos ]] && cd eos + [[ ! -d build ]] && mkdir build + cd build + echo \$PATH + ccache -s + echo '$ cmake ..' + cmake .. + echo "$ make -j $CPU_CORES" + make -j $CPU_CORES + echo 'Running unit tests.' + echo "$ ctest -j $CPU_CORES -LE _tests --output-on-failure -T Test" + ctest -j $CPU_CORES -LE _tests --output-on-failure -T Test # run unit tests +else # linux + execute docker run --rm -v $(pwd):/workdir -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e CCACHE_DIR=/opt/.ccache $FULL_TAG +fi \ No newline at end of file diff --git a/.cicd/ubuntu-16.04.dockerfile b/.cicd/ubuntu-16.04.dockerfile new file mode 100644 index 00000000000..e83384cea65 --- /dev/null +++ b/.cicd/ubuntu-16.04.dockerfile @@ -0,0 +1,93 @@ +FROM ubuntu:16.04 + +# APT-GET dependencies. +RUN apt-get update && apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ + libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ + python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config + +# Build appropriate version of CMake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \ + && tar -xzf cmake-3.13.2.tar.gz \ + && cd cmake-3.13.2 \ + && ./bootstrap --prefix=/usr/local \ + && make -j$(nproc) \ + && make install \ + && cd .. \ + && rm -f cmake-3.13.2.tar.gz + +# Build appropriate version of Clang. +RUN mkdir -p /root/tmp && cd /root/tmp && git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 && cd clang8 && git checkout 18e41dc && cd tools && git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git && cd lld && git checkout d60a035 && cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git && cd polly && git checkout 1bc06e5 && cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang && cd clang && git checkout a03da8b && cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && cd .. && cd ../../../../projects && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git && cd libcxx && git checkout 1853712 && cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git && cd libunwind && git checkout 57f6739 && cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && cd ../ && cd /root/tmp/clang8 && mkdir build && cd build && cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=all -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) && make install \ + && cd / && rm -rf /root/tmp/clang8 + +COPY ./pinned_toolchain.cmake /tmp/pinned_toolchain.cmake + +# # Build appropriate version of LLVM. +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm \ + && cd llvm \ + && mkdir build \ + && cd build \ + && cmake -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' .. \ + && make -j$(nproc) \ + && make install \ + && cd / + +# Build appropriate version of Boost. +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \ + && tar -xjf boost_1_70_0.tar.bz2 \ + && cd boost_1_70_0 \ + && ./bootstrap.sh --with-toolset=clang --prefix=/usr/local \ + && ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1' linkflags='-stdlib=libc++' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install \ + && cd .. \ + && rm -f boost_1_70_0.tar.bz2 + +# Build appropriate version of MongoDB. +RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz \ + && tar -xzf mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz \ + && rm -f mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz + +# Build appropriate version of MongoDB C Driver. +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \ + && tar -xzf mongo-c-driver-1.13.0.tar.gz \ + && cd mongo-c-driver-1.13.0 \ + && mkdir -p build \ + && cd build \ + && cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -rf mongo-c-driver-1.13.0.tar.gz + +# Build appropriate version of MongoDB C++ driver. +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \ + && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \ + && cd mongo-cxx-driver-r3.4.0 \ + && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp \ + && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \ + && cd build \ + && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -f mongo-cxx-driver-r3.4.0.tar.gz + +ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1604-3.6.3/bin + +# Build appropriate version of ccache. +RUN curl -LO https://github.com/ccache/ccache/releases/download/v3.4.1/ccache-3.4.1.tar.gz \ + && tar -xzf ccache-3.4.1.tar.gz \ + && cd ccache-3.4.1 \ + && ./configure \ + && make \ + && make install \ + && cd / && rm -rf ccache-3.4.1/ + +# PRE_COMMANDS: Executed pre-cmake +# CMAKE_EXTRAS: Executed right before the cmake path (on the end) +ENV PRE_COMMANDS="export PATH=/usr/lib/ccache:$PATH &&" +ENV CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + +CMD bash -c "$PRE_COMMANDS ccache -s && \ + mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \ + ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test" \ No newline at end of file diff --git a/.cicd/ubuntu-18.04.dockerfile b/.cicd/ubuntu-18.04.dockerfile new file mode 100644 index 00000000000..1561eccfc2b --- /dev/null +++ b/.cicd/ubuntu-18.04.dockerfile @@ -0,0 +1,69 @@ +FROM ubuntu:18.04 + +# APT-GET dependencies. +RUN apt-get update && apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ + bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ + autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ + autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ + libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache + +# Build appropriate version of CMake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \ + && tar -xzf cmake-3.13.2.tar.gz \ + && cd cmake-3.13.2 \ + && ./bootstrap --prefix=/usr/local \ + && make -j$(nproc) \ + && make install \ + && cd .. \ + && rm -f cmake-3.13.2.tar.gz + +# Build appropriate version of Boost. +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 \ + && tar -xjf boost_1_70_0.tar.bz2 \ + && cd boost_1_70_0 \ + && ./bootstrap.sh --prefix=/usr/local \ + && ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j$(nproc) install \ + && cd .. \ + && rm -f boost_1_70_0.tar.bz2 + +# Build appropriate version of MongoDB. +RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz \ + && tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz \ + && rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz + +# Build appropriate version of MongoDB C Driver. +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz \ + && tar -xzf mongo-c-driver-1.13.0.tar.gz \ + && cd mongo-c-driver-1.13.0 \ + && mkdir -p build \ + && cd build \ + && cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -rf mongo-c-driver-1.13.0.tar.gz + +# Build appropriate version of MongoDB C++ driver. +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz \ + && tar -xzf mongo-cxx-driver-r3.4.0.tar.gz \ + && cd mongo-cxx-driver-r3.4.0 \ + && sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp \ + && sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt \ + && cd build \ + && cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ + && make -j$(nproc) \ + && make install \ + && cd / \ + && rm -f mongo-cxx-driver-r3.4.0.tar.gz + +ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin + +# PRE_COMMANDS: Executed pre-cmake +# CMAKE_EXTRAS: Executed right before the cmake path (on the end) +ENV PRE_COMMANDS="export PATH=/usr/lib/ccache:$PATH &&" +ENV CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + +CMD bash -c "$PRE_COMMANDS ccache -s && \ + mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \ + ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test" diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..168d4071571 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,54 @@ +language: cpp +git: + depth: false # prevent git describe failure when executing the appbase version.cmake.in +cache: + ccache: true + directories: + - $HOME/Library/Caches/Homebrew +matrix: + include: + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='ubuntu-18.04' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='ubuntu-16.04' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='amazonlinux-2' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='centos-7' + - os: osx + osx_image: xcode10.2 + addons: + homebrew: + update: true + packages: + - graphviz + - libtool + - gmp + - llvm@4 + - pkgconfig + - python + - python@2 + - doxygen + - libusb + - openssl + - boost@1.70 + - ccache + env: + - PATH="/usr/local/opt/ccache/libexec:$PATH" +script: | + ./.cicd/travis-build.sh +notifications: + webhooks: + secure: gmqODqoFAil2cR7v++ibqRNECBOSD/VJX+2qPa7XptkVWmVMzbII5CNgBQAscjFsp9arHPMXCCzkBi847PCSiHdsnYFQ4T273FLRWr3cDbLjfmR+BJ7dGKvQnlpSi2Ze2TtAPJyRl+iv+cxDj7cWE5zw2c4xbgh1a/cNO+/ayUfFkyMEIfVWRsHkdkra4gOLywou0XRLHr4CX1V60uU7uuqATnIMMi7gQYwiKKtZqjkbf8wcBvZirDhjQ6lDPN5tnZo6L4QHmqjtzNJg/UrD4h+zES53dLVI4uxlXRAwwpw+mJOFA3QE/3FT+bMQjLCffUz4gZaWcdgebPYzrwSWUbJoFdWAOwcTqivQY0FIQzcz/r6uGWcwWTavzkPEbg68BVM2BZId/0110J6feeTkpJ3MPV+UsIoGTvbg50vi/I06icftuZ/cLqDj3+Emifm7Jlr1sRTSdqtYAJj/2ImUfsb46cwgjAVhFOTvc+KuPgJQgvOXV7bZkxEr5qDWo8Al2sV8BWb83j1rMlZ4LfERokImDVqxu2kkcunchzvhtYFTesSpmwegVpwceCtOtO0rEUgATnfTEHzk2rm8nuz4UtidsQnluUKqmKD0QCqHXFfn+3ZRJsDqr+iCYdxv1BAeAVc9q1L7bgrKDMGiJgkxuhZ2v3J2SflWLvjZjFDduuc= diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 82e723e5563..31f6b801392 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -36,6 +36,40 @@ link_directories(${LLVM_LIBRARY_DIR}) add_subdirectory(contracts) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/contracts.hpp ESCAPE_QUOTES) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz1.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz1.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz2.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz2.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz3.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz3.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz4.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz4.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz5.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz5.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz6.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz6.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz7.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz7.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz8.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz8.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz9.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz9.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz10.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz10.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz11.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz11.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz12.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz12.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz13.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz13.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz14.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz14.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/fuzz15.wasm ${CMAKE_CURRENT_BINARY_DIR}/fuzz15.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/big_allocation.wasm ${CMAKE_CURRENT_BINARY_DIR}/big_allocation.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/crash_section_size_too_big.wasm ${CMAKE_CURRENT_BINARY_DIR}/crash_section_size_too_big.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_no_destructor.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_no_destructor.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readExports.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readExports.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readFunctions.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readFunctions.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readFunctions_2.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readFunctions_2.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readFunctions_3.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readFunctions_3.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readGlobals.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readGlobals.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_readImports.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_readImports.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/leak_wasm_binary_cpp_L1249.wasm ${CMAKE_CURRENT_BINARY_DIR}/leak_wasm_binary_cpp_L1249.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/readFunctions_slowness_out_of_memory.wasm ${CMAKE_CURRENT_BINARY_DIR}/readFunctions_slowness_out_of_memory.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/locals-yc.wasm ${CMAKE_CURRENT_BINARY_DIR}/locals-yc.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/locals-s.wasm ${CMAKE_CURRENT_BINARY_DIR}/locals-s.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/slowwasm_localsets.wasm ${CMAKE_CURRENT_BINARY_DIR}/slowwasm_localsets.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/getcode_deepindent.wasm ${CMAKE_CURRENT_BINARY_DIR}/getcode_deepindent.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/indent-mismatch.wasm ${CMAKE_CURRENT_BINARY_DIR}/indent-mismatch.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/deep_loops_ext_report.wasm ${CMAKE_CURRENT_BINARY_DIR}/deep_loops_ext_report.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/80k_deep_loop_with_ret.wasm ${CMAKE_CURRENT_BINARY_DIR}/80k_deep_loop_with_ret.wasm COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts/80k_deep_loop_with_void.wasm ${CMAKE_CURRENT_BINARY_DIR}/80k_deep_loop_with_void.wasm COPYONLY) ### BUILD UNIT TEST EXECUTABLE ### file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites