build(thirdparty): Fix compile failed on some platforms (#1016)

apache · Jun 27, 2022 · 6d28e35 · 6d28e35
1 parent f9a2e61
commit 6d28e35
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 54 deletions.
diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml
@@ -19,9 +19,15 @@ name: BuildCompilationEnvDocker - build and publish multi compilation OS env
 
 on:
   push:
+    branches:
+      - master
+      - 'v[0-9]+.*' # release branch
+      - ci-test # testing branch for github action
+      - '*dev' # developing branch
     paths:
       - 'docker/pegasus-build-env/**'
       - '.github/workflows/build-push-env-docker.yml'
+      - 'rdsn/thirdparty/**'
 
   # for manually triggering workflow
   workflow_dispatch:
@@ -45,7 +51,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml
@@ -19,10 +19,16 @@ name:  BuildThirdpartyDockerRegularly - build and publish thirdparty every week
 
 on:
   push:
+    branches:
+      - master
+      - 'v[0-9]+.*' # release branch
+      - ci-test # testing branch for github action
+      - '*dev' # developing branch
     paths:
       - 'docker/thirdparties-src/**'
       - 'docker/thirdparties-bin/**'
       - '.github/workflows/thirdparty-regular-push.yml'
+      - 'rdsn/thirdparty/**'
 
   # for manually triggering workflow
   workflow_dispatch:
@@ -54,7 +60,7 @@ jobs:
           tags: |
             apache/pegasus:thirdparties-src
           build-args: |
-            REGISTRY_REPO=apache/pegasus
+            GITHUB_BRANCH=${{ github.ref_name }}
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
@@ -89,6 +95,7 @@ jobs:
           tags: |
             apache/pegasus:thirdparties-bin-${{ matrix.osversion }}
           build-args: |
+            GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/docker/ci-env/Dockerfile b/docker/ci-env/Dockerfile
@@ -30,4 +30,4 @@ RUN git clone --depth=1 https://github.com/apache/incubator-pegasus.git; \
     cmake --build build/ -j $(($(nproc)/2+1)); \
     zip -r ~/pegasus-thirdparty-output.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper; \
     cd ~; \
-    rm -rf rdsn;
+    rm -rf incubator-pegasus;
diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile
@@ -20,41 +20,41 @@ FROM centos:7.5.1804
 LABEL maintainer=wutao
 
 RUN yum -y install centos-release-scl \
-scl-utils \
-epel-release; \
-yum -y install devtoolset-7-gcc \
-devtoolset-7-gcc-c++ \
-java-1.8.0-openjdk-devel.x86_64 \
-python3 \
-automake \
-autoconf \
-make \
-libtool \
-git \
-file \
-wget \
-ccache \
-nmap-ncat \
-zip \
-gdb \
-vim \
-unzip \
-which \
-openssl-devel \
-libaio-devel \
-snappy-devel \
-bzip2-devel \
-zlib \
-zlib-devel \
-libzstd-devel \
-lz4-devel \
-bison \
-flex \
-krb5-devel \
-cyrus-sasl-devel \
-patch; \
-yum clean all; \
-rm -rf /var/cache/yum;
+                   scl-utils \
+                   epel-release; \
+                   yum -y install devtoolset-7-gcc \
+                   devtoolset-7-gcc-c++ \
+                   java-1.8.0-openjdk-devel.x86_64 \
+                   python3 \
+                   automake \
+                   autoconf \
+                   make \
+                   libtool \
+                   git \
+                   file \
+                   wget \
+                   ccache \
+                   nmap-ncat \
+                   zip \
+                   gdb \
+                   vim \
+                   unzip \
+                   which \
+                   openssl-devel \
+                   libaio-devel \
+                   snappy-devel \
+                   bzip2-devel \
+                   zlib \
+                   zlib-devel \
+                   libzstd-devel \
+                   lz4-devel \
+                   bison \
+                   flex \
+                   krb5-devel \
+                   cyrus-sasl-devel \
+                   patch; \
+                   yum clean all; \
+                   rm -rf /var/cache/yum;
 
 RUN pip3 install --no-cache-dir cmake
 
@@ -63,7 +63,7 @@ RUN wget --progress=dot:giga https://archive.apache.org/dist/maven/maven-3/3.8.3
     && tar -zxf apache-maven-3.8.3-bin.tar.gz \
     && rm apache-maven-3.8.3-bin.tar.gz
 
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_64
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
 ENV PATH="/opt/rh/devtoolset-7/root/bin/:${PATH}"
 ENV PATH="/opt/maven/apache-maven-3.8.3/bin:${PATH}"
 

diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile
@@ -16,23 +16,22 @@
 # under the License.
 
 ARG OS_VERSION=centos7
-
 FROM apache/pegasus:thirdparties-src as builder
 
 ARG OS_VERSION=centos7
-
 FROM apache/pegasus:build-env-${OS_VERSION}
 
 WORKDIR /root
 
 COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
 
-# NOTE: the rocksdb lib is built in portable mode.
-RUN git clone --depth=1 https://github.com/apache/incubator-pegasus.git \
+ARG GITHUB_BRANCH=master
+ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
+RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
     && cd incubator-pegasus/rdsn/thirdparty \
     && unzip /root/thirdparties-src.zip -d . \
-    && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ . \
+    && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=OFF -B build/ . \
     && cmake --build build/ -j $(($(nproc)/2+1)) \
     && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper \
     && cd ~ \
-    && rm -rf rdsn;
+    && rm -rf incubator-pegasus;
diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile
@@ -15,26 +15,23 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# use dockerhub by default
-ARG REGISTRY_REPO=apache/pegasus
-
-FROM ${REGISTRY_REPO}:build-env-centos7 as builder
+FROM apache/pegasus:build-env-centos7 as builder
 
 WORKDIR /root
 
-ARG TAGNAME=master
-
-RUN git clone --depth=1 --branch=${TAGNAME} https://github.com/apache/incubator-pegasus.git
+ARG GITHUB_BRANCH=master
+ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
+RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL}
 
 RUN cd incubator-pegasus/rdsn/thirdparty \
     && mkdir -p build \
     && cmake -DCMAKE_BUILD_TYPE=Release -B build/ . \
     && cmake --build build/ -j $(($(nproc)/2+1))
 
-RUN cd rdsn/thirdparty \
+RUN cd incubator-pegasus/rdsn/thirdparty \
     && zip -r ~/thirdparties-src.zip build/Download \
     && cd ~ \
-    && rm -rf rdsn;
+    && rm -rf incubator-pegasus;
 
 # Unzip thirdparties-src.zip to rdsn/thirdparty, the download will automatically be skipped.
 

diff --git a/rdsn/thirdparty/CMakeLists.txt b/rdsn/thirdparty/CMakeLists.txt
@@ -52,6 +52,9 @@ set(OSS_URL_PREFIX "http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.co
 
 message(STATUS "Setting up third-parties...")
 
+file(MAKE_DIRECTORY ${TP_OUTPUT}/include)
+file(MAKE_DIRECTORY ${TP_OUTPUT}/lib)
+
 ExternalProject_Add(boost
         URL ${OSS_URL_PREFIX}/boost_1_69_0.tar.bz2
         https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.bz2