From 2746266addddf71d20a4fe49381497b894c4d15c Mon Sep 17 00:00:00 2001 From: Anthony Louis Date: Sat, 1 May 2021 08:02:27 +0900 Subject: [PATCH] ARROW-12591: [Java][Gandiva] Create single Gandiva jar for MacOS and Linux Today, there are two different tasks that generate Gandiva's jars for MacOS and Linux. The objective is to create a single jar with the shared lib for the two operating systems. Closes #10189 from anthonylouisbsb/feature/create-single-jar-macos-linux Authored-by: Anthony Louis Signed-off-by: Sutou Kouhei --- dev/tasks/gandiva-jars/build-java.sh | 38 +----- .../gandiva-jars/check-shared-dependencies.sh | 58 +++++++++ dev/tasks/gandiva-jars/github.linux.yml | 47 -------- dev/tasks/gandiva-jars/github.osx.yml | 46 -------- dev/tasks/gandiva-jars/github.yml | 111 ++++++++++++++++++ dev/tasks/tasks.yml | 10 +- 6 files changed, 175 insertions(+), 135 deletions(-) create mode 100755 dev/tasks/gandiva-jars/check-shared-dependencies.sh delete mode 100644 dev/tasks/gandiva-jars/github.linux.yml delete mode 100644 dev/tasks/gandiva-jars/github.osx.yml create mode 100644 dev/tasks/gandiva-jars/github.yml diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh index 7dec07115a3ef..79af606d3d08b 100755 --- a/dev/tasks/gandiva-jars/build-java.sh +++ b/dev/tasks/gandiva-jars/build-java.sh @@ -22,43 +22,13 @@ set -e CPP_BUILD_DIR=$GITHUB_WORKSPACE/arrow/dist/ pushd java - if [[ $OS_NAME == "linux" ]]; then - SO_DEP=ldd - GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.so - WHITELIST=(linux-vdso libz librt libdl libpthread libstdc++ libm libgcc_s libc ld-linux-x86-64) - else - SO_DEP="otool -L" - GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.dylib - WHITELIST=(libgandiva_jni libz libncurses libSystem libc++) - fi - - # print the shared library dependencies - eval "$SO_DEP" "$GANDIVA_LIB" - - if [[ $CHECK_SHARED_DEPENDENCIES ]] ; then - # exit if any shared library not in whitelisted set is found - echo "Checking shared dependencies" - while read -r line - do - found=false - for item in "${WHITELIST[@]}" - do - if [[ "$line" == *"$item"* ]] ; then - found=true - fi - done - if [[ "$found" == false ]] ; then - echo "Unexpected shared dependency found" - exit 1 - fi - done < <(eval "$SO_DEP" "$GANDIVA_LIB" | awk '{print $1}') - fi - # build the entire project mvn clean install -q -DskipTests -P arrow-jni -Darrow.cpp.build.dir=$CPP_BUILD_DIR # test only gandiva mvn test -q -P arrow-jni -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR - # copy the jars to distribution folder - find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} $CPP_BUILD_DIR \; + if [[ $COPY_JAR_TO_DISTRIBUTION_FOLDER ]] ; then + # copy the jars to distribution folder + find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} $CPP_BUILD_DIR \; + fi popd diff --git a/dev/tasks/gandiva-jars/check-shared-dependencies.sh b/dev/tasks/gandiva-jars/check-shared-dependencies.sh new file mode 100755 index 0000000000000..ce93ff571830d --- /dev/null +++ b/dev/tasks/gandiva-jars/check-shared-dependencies.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +CPP_BUILD_DIR=$GITHUB_WORKSPACE/arrow/dist/ + +if [[ $OS_NAME == "linux" ]]; then + SO_DEP=ldd + GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.so + WHITELIST=(linux-vdso libz librt libdl libpthread libstdc++ libm libgcc_s libc ld-linux-x86-64) +else + SO_DEP="otool -L" + GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.dylib + WHITELIST=(libgandiva_jni libz libncurses libSystem libc++) +fi + +# print the shared library dependencies +$SO_DEP "$GANDIVA_LIB" | tee dependencies_temp_file.txt + +if [[ $CHECK_SHARED_DEPENDENCIES ]] ; then + # exit if any shared library not in whitelisted set is found + echo "Checking shared dependencies" + + awk '{print $1}' dependencies_temp_file.txt | \ + while read -r line + do + found=false + + for item in "${WHITELIST[@]}" + do + if [[ "$line" == *"$item"* ]] ; then + found=true + fi + done + + if [[ "$found" == false ]] ; then + echo "Unexpected shared dependency found $line" + exit 1 + fi + done +fi \ No newline at end of file diff --git a/dev/tasks/gandiva-jars/github.linux.yml b/dev/tasks/gandiva-jars/github.linux.yml deleted file mode 100644 index aabcdbee0efc0..0000000000000 --- a/dev/tasks/gandiva-jars/github.linux.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package Gandiva - runs-on: ubuntu-18.04 - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi - git -C arrow submodule update --init --recursive - - name: Build Gandiva - run: | - python3 -VV - cd arrow - mkdir -p dist - export CC="gcc-4.9" CXX="g++-4.9" - ulimit -c unlimited -S - set -e - docker run -v $PWD:/arrow quay.io/anthonylouisbsb/arrow:gandivadocker /arrow/dev/tasks/gandiva-jars/build-cpp-linux.sh - dev/tasks/gandiva-jars/build-java.sh - env: - OS_NAME: "linux" - CHECK_SHARED_DEPENDENCIES: true - - {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/gandiva-jars/github.osx.yml b/dev/tasks/gandiva-jars/github.osx.yml deleted file mode 100644 index 3dd6fe46bb658..0000000000000 --- a/dev/tasks/gandiva-jars/github.osx.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package Gandiva - runs-on: macos-latest - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi - git -C arrow submodule update --init --recursive - - name: Build Gandiva - run: | - cd arrow - mkdir -p dist - export ARROW_TEST_DATA=$PWD/testing/data - set -e - dev/tasks/gandiva-jars/build-cpp-osx.sh - dev/tasks/gandiva-jars/build-java.sh - env: - OS_NAME: "osx" - CHECK_SHARED_DEPENDENCIES: true - MACOSX_DEPLOYMENT_TARGET: "10.11" - - {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/gandiva-jars/github.yml b/dev/tasks/gandiva-jars/github.yml new file mode 100644 index 0000000000000..a1ac093c47bf5 --- /dev/null +++ b/dev/tasks/gandiva-jars/github.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build-cpp-ubuntu: + name: Build C++ Gandiva Libs Ubuntu + runs-on: ubuntu-18.04 + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi + git -C arrow submodule update --init --recursive + - name: Build Gandiva + run: | + python3 -VV + cd arrow + mkdir -p dist + export CC="gcc-4.9" CXX="g++-4.9" + ulimit -c unlimited -S + set -e + docker run -v $PWD:/arrow quay.io/anthonylouisbsb/arrow:gandivadocker /arrow/dev/tasks/gandiva-jars/build-cpp-linux.sh + dev/tasks/gandiva-jars/check-shared-dependencies.sh + env: + OS_NAME: "linux" + CHECK_SHARED_DEPENDENCIES: true + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: ubuntu-shared-lib + path: arrow/dist/libgandiva_jni.so + build-cpp-macos: + name: Build C++ Gandiva Libs MacOS + runs-on: macos-latest + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi + git -C arrow submodule update --init --recursive + - name: Build Gandiva + run: | + cd arrow + mkdir -p dist + export ARROW_TEST_DATA=$PWD/testing/data + set -e + dev/tasks/gandiva-jars/build-cpp-osx.sh + dev/tasks/gandiva-jars/check-shared-dependencies.sh + env: + OS_NAME: "osx" + CHECK_SHARED_DEPENDENCIES: true + MACOSX_DEPLOYMENT_TARGET: "10.11" + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: macos-shared-lib + path: arrow/dist/libgandiva_jni.dylib + package-jar: + name: Build Gandiva Jar + runs-on: macos-latest + needs: [build-cpp-macos, build-cpp-ubuntu] + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi + git -C arrow submodule update --init --recursive + mkdir -p arrow/dist + - name: Download Linux Gandiva Library + uses: actions/download-artifact@v2 + with: + name: ubuntu-shared-lib + path: arrow/dist + - name: Download MacOS Gandiva Library + uses: actions/download-artifact@v2 + with: + name: macos-shared-lib + path: arrow/dist + - name: Build Gandiva Jar + run: | + cd arrow + export ARROW_TEST_DATA=$PWD/testing/data + set -e + dev/tasks/gandiva-jars/build-java.sh + env: + OS_NAME: "osx" + COPY_JAR_TO_DISTRIBUTION_FOLDER: true + MACOSX_DEPLOYMENT_TARGET: "10.11" + + {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} \ No newline at end of file diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d48da7dc114c6..c0a9fe69d26af 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -658,15 +658,9 @@ tasks: ############################## Gandiva Tasks ################################ - gandiva-jar-ubuntu: + gandiva-jar: ci: github - template: gandiva-jars/github.linux.yml - artifacts: - - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar - - gandiva-jar-osx: - ci: github - template: gandiva-jars/github.osx.yml + template: gandiva-jars/github.yml artifacts: - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar