From 6d28e35cd2285ba85de901589acaaaf6f4e7420a Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 27 Jun 2022 11:35:31 +0800 Subject: [PATCH] build(thirdparty): Fix compile failed on some platforms (#1016) --- .github/workflows/build-push-env-docker.yml | 8 ++- .github/workflows/thirdparty-regular-push.yml | 9 ++- docker/ci-env/Dockerfile | 2 +- docker/pegasus-build-env/centos7/Dockerfile | 72 +++++++++---------- docker/thirdparties-bin/Dockerfile | 11 ++- docker/thirdparties-src/Dockerfile | 15 ++-- rdsn/thirdparty/CMakeLists.txt | 3 + 7 files changed, 66 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml index e03c111a85..19ea9571da 100644 --- a/.github/workflows/build-push-env-docker.yml +++ b/.github/workflows/build-push-env-docker.yml @@ -19,9 +19,15 @@ name: BuildCompilationEnvDocker - build and publish multi compilation OS env on: push: + branches: + - master + - 'v[0-9]+.*' # release branch + - ci-test # testing branch for github action + - '*dev' # developing branch paths: - 'docker/pegasus-build-env/**' - '.github/workflows/build-push-env-docker.yml' + - 'rdsn/thirdparty/**' # for manually triggering workflow workflow_dispatch: @@ -45,7 +51,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 8071e40e41..b2fc3e032f 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -19,10 +19,16 @@ name: BuildThirdpartyDockerRegularly - build and publish thirdparty every week on: push: + branches: + - master + - 'v[0-9]+.*' # release branch + - ci-test # testing branch for github action + - '*dev' # developing branch paths: - 'docker/thirdparties-src/**' - 'docker/thirdparties-bin/**' - '.github/workflows/thirdparty-regular-push.yml' + - 'rdsn/thirdparty/**' # for manually triggering workflow workflow_dispatch: @@ -54,7 +60,7 @@ jobs: tags: | apache/pegasus:thirdparties-src build-args: | - REGISTRY_REPO=apache/pegasus + GITHUB_BRANCH=${{ github.ref_name }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} @@ -89,6 +95,7 @@ jobs: tags: | apache/pegasus:thirdparties-bin-${{ matrix.osversion }} build-args: | + GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/docker/ci-env/Dockerfile b/docker/ci-env/Dockerfile index 89b9f65040..d49ada1f24 100644 --- a/docker/ci-env/Dockerfile +++ b/docker/ci-env/Dockerfile @@ -30,4 +30,4 @@ RUN git clone --depth=1 https://github.com/apache/incubator-pegasus.git; \ cmake --build build/ -j $(($(nproc)/2+1)); \ zip -r ~/pegasus-thirdparty-output.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper; \ cd ~; \ - rm -rf rdsn; + rm -rf incubator-pegasus; diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 6655208a3f..8497c65934 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -20,41 +20,41 @@ FROM centos:7.5.1804 LABEL maintainer=wutao RUN yum -y install centos-release-scl \ -scl-utils \ -epel-release; \ -yum -y install devtoolset-7-gcc \ -devtoolset-7-gcc-c++ \ -java-1.8.0-openjdk-devel.x86_64 \ -python3 \ -automake \ -autoconf \ -make \ -libtool \ -git \ -file \ -wget \ -ccache \ -nmap-ncat \ -zip \ -gdb \ -vim \ -unzip \ -which \ -openssl-devel \ -libaio-devel \ -snappy-devel \ -bzip2-devel \ -zlib \ -zlib-devel \ -libzstd-devel \ -lz4-devel \ -bison \ -flex \ -krb5-devel \ -cyrus-sasl-devel \ -patch; \ -yum clean all; \ -rm -rf /var/cache/yum; + scl-utils \ + epel-release; \ + yum -y install devtoolset-7-gcc \ + devtoolset-7-gcc-c++ \ + java-1.8.0-openjdk-devel.x86_64 \ + python3 \ + automake \ + autoconf \ + make \ + libtool \ + git \ + file \ + wget \ + ccache \ + nmap-ncat \ + zip \ + gdb \ + vim \ + unzip \ + which \ + openssl-devel \ + libaio-devel \ + snappy-devel \ + bzip2-devel \ + zlib \ + zlib-devel \ + libzstd-devel \ + lz4-devel \ + bison \ + flex \ + krb5-devel \ + cyrus-sasl-devel \ + patch; \ + yum clean all; \ + rm -rf /var/cache/yum; RUN pip3 install --no-cache-dir cmake @@ -63,7 +63,7 @@ RUN wget --progress=dot:giga https://archive.apache.org/dist/maven/maven-3/3.8.3 && tar -zxf apache-maven-3.8.3-bin.tar.gz \ && rm apache-maven-3.8.3-bin.tar.gz -ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_64 +ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk ENV PATH="/opt/rh/devtoolset-7/root/bin/:${PATH}" ENV PATH="/opt/maven/apache-maven-3.8.3/bin:${PATH}" diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index 12d4147e4e..23319a97c0 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -16,23 +16,22 @@ # under the License. ARG OS_VERSION=centos7 - FROM apache/pegasus:thirdparties-src as builder ARG OS_VERSION=centos7 - FROM apache/pegasus:build-env-${OS_VERSION} WORKDIR /root COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip -# NOTE: the rocksdb lib is built in portable mode. -RUN git clone --depth=1 https://github.com/apache/incubator-pegasus.git \ +ARG GITHUB_BRANCH=master +ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git +RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \ && cd incubator-pegasus/rdsn/thirdparty \ && unzip /root/thirdparties-src.zip -d . \ - && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ . \ + && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=OFF -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) \ && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper \ && cd ~ \ - && rm -rf rdsn; + && rm -rf incubator-pegasus; diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile index f01711da60..05b13a0919 100644 --- a/docker/thirdparties-src/Dockerfile +++ b/docker/thirdparties-src/Dockerfile @@ -15,26 +15,23 @@ # specific language governing permissions and limitations # under the License. -# use dockerhub by default -ARG REGISTRY_REPO=apache/pegasus - -FROM ${REGISTRY_REPO}:build-env-centos7 as builder +FROM apache/pegasus:build-env-centos7 as builder WORKDIR /root -ARG TAGNAME=master - -RUN git clone --depth=1 --branch=${TAGNAME} https://github.com/apache/incubator-pegasus.git +ARG GITHUB_BRANCH=master +ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git +RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} RUN cd incubator-pegasus/rdsn/thirdparty \ && mkdir -p build \ && cmake -DCMAKE_BUILD_TYPE=Release -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) -RUN cd rdsn/thirdparty \ +RUN cd incubator-pegasus/rdsn/thirdparty \ && zip -r ~/thirdparties-src.zip build/Download \ && cd ~ \ - && rm -rf rdsn; + && rm -rf incubator-pegasus; # Unzip thirdparties-src.zip to rdsn/thirdparty, the download will automatically be skipped. diff --git a/rdsn/thirdparty/CMakeLists.txt b/rdsn/thirdparty/CMakeLists.txt index f5068bbdbb..6731f2b878 100644 --- a/rdsn/thirdparty/CMakeLists.txt +++ b/rdsn/thirdparty/CMakeLists.txt @@ -52,6 +52,9 @@ set(OSS_URL_PREFIX "http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.co message(STATUS "Setting up third-parties...") +file(MAKE_DIRECTORY ${TP_OUTPUT}/include) +file(MAKE_DIRECTORY ${TP_OUTPUT}/lib) + ExternalProject_Add(boost URL ${OSS_URL_PREFIX}/boost_1_69_0.tar.bz2 https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.bz2