ARROW-12591: [Java][Gandiva] Create single Gandiva jar for MacOS and …

…Linux Today, there are two different tasks that generate Gandiva's jars for MacOS and Linux. The objective is to create a single jar with the shared lib for the two operating systems. Closes #10189 from anthonylouisbsb/feature/create-single-jar-macos-linux Authored-by: Anthony Louis <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
apache · Apr 30, 2021 · 2746266 · 2746266
1 parent eec855d
commit 2746266
Show file tree

Hide file tree

Showing 6 changed files with 175 additions and 135 deletions.
diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh
@@ -22,43 +22,13 @@ set -e
 CPP_BUILD_DIR=$GITHUB_WORKSPACE/arrow/dist/
 
 pushd java
-  if [[ $OS_NAME == "linux" ]]; then
-    SO_DEP=ldd
-    GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.so
-    WHITELIST=(linux-vdso libz librt libdl libpthread libstdc++ libm libgcc_s libc ld-linux-x86-64)
-  else
-    SO_DEP="otool -L"
-    GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.dylib
-    WHITELIST=(libgandiva_jni libz libncurses libSystem libc++)
-  fi
-
-  # print the shared library dependencies
-  eval "$SO_DEP" "$GANDIVA_LIB"
-
-  if [[ $CHECK_SHARED_DEPENDENCIES ]] ; then
-    # exit if any shared library not in whitelisted set is found
-    echo "Checking shared dependencies"
-    while read -r line
-    do
-      found=false
-      for item in "${WHITELIST[@]}"
-      do
-        if [[ "$line" == *"$item"* ]] ; then
-            found=true
-        fi
-      done
-      if [[ "$found" == false ]] ; then
-        echo "Unexpected shared dependency found"
-        exit 1
-      fi
-    done < <(eval "$SO_DEP" "$GANDIVA_LIB" | awk '{print $1}')
-  fi
-
   # build the entire project
   mvn clean install -q -DskipTests -P arrow-jni -Darrow.cpp.build.dir=$CPP_BUILD_DIR
   # test only gandiva
   mvn test -q -P arrow-jni -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR
 
-  # copy the jars to distribution folder
-  find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp  {} $CPP_BUILD_DIR \;
+  if [[ $COPY_JAR_TO_DISTRIBUTION_FOLDER ]] ; then
+    # copy the jars to distribution folder
+    find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp  {} $CPP_BUILD_DIR \;
+  fi
 popd
diff --git a/dev/tasks/gandiva-jars/check-shared-dependencies.sh b/dev/tasks/gandiva-jars/check-shared-dependencies.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+CPP_BUILD_DIR=$GITHUB_WORKSPACE/arrow/dist/
+
+if [[ $OS_NAME == "linux" ]]; then
+  SO_DEP=ldd
+  GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.so
+  WHITELIST=(linux-vdso libz librt libdl libpthread libstdc++ libm libgcc_s libc ld-linux-x86-64)
+else
+  SO_DEP="otool -L"
+  GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.dylib
+  WHITELIST=(libgandiva_jni libz libncurses libSystem libc++)
+fi
+
+# print the shared library dependencies
+$SO_DEP "$GANDIVA_LIB" | tee dependencies_temp_file.txt 
+
+if [[ $CHECK_SHARED_DEPENDENCIES ]] ; then
+  # exit if any shared library not in whitelisted set is found
+  echo "Checking shared dependencies"
+
+  awk '{print $1}' dependencies_temp_file.txt | \
+  while read -r line
+  do
+    found=false
+
+    for item in "${WHITELIST[@]}"
+    do
+    if [[ "$line" == *"$item"* ]] ; then
+        found=true
+    fi
+    done
+
+    if [[ "$found" == false ]] ; then
+      echo "Unexpected shared dependency found $line"
+      exit 1
+    fi
+  done
+fi
diff --git a/dev/tasks/gandiva-jars/github.linux.yml b/dev/tasks/gandiva-jars/github.linux.yml
diff --git a/dev/tasks/gandiva-jars/github.osx.yml b/dev/tasks/gandiva-jars/github.osx.yml
diff --git a/dev/tasks/gandiva-jars/github.yml b/dev/tasks/gandiva-jars/github.yml
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  build-cpp-ubuntu:
+    name: Build C++ Gandiva Libs Ubuntu
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi
+          git -C arrow submodule update --init --recursive
+      - name: Build Gandiva
+        run: |
+          python3 -VV
+          cd arrow
+          mkdir -p dist
+          export CC="gcc-4.9" CXX="g++-4.9"
+          ulimit -c unlimited -S
+          set -e
+          docker run -v $PWD:/arrow quay.io/anthonylouisbsb/arrow:gandivadocker /arrow/dev/tasks/gandiva-jars/build-cpp-linux.sh
+          dev/tasks/gandiva-jars/check-shared-dependencies.sh
+        env:
+          OS_NAME: "linux"
+          CHECK_SHARED_DEPENDENCIES: true
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-shared-lib
+          path: arrow/dist/libgandiva_jni.so
+  build-cpp-macos:
+    name: Build C++ Gandiva Libs MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi
+          git -C arrow submodule update --init --recursive
+      - name: Build Gandiva
+        run: |
+          cd arrow
+          mkdir -p dist
+          export ARROW_TEST_DATA=$PWD/testing/data
+          set -e
+          dev/tasks/gandiva-jars/build-cpp-osx.sh
+          dev/tasks/gandiva-jars/check-shared-dependencies.sh
+        env:
+          OS_NAME: "osx"
+          CHECK_SHARED_DEPENDENCIES: true
+          MACOSX_DEPLOYMENT_TARGET: "10.11"
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: macos-shared-lib
+          path: arrow/dist/libgandiva_jni.dylib
+  package-jar:
+    name: Build Gandiva Jar
+    runs-on: macos-latest
+    needs: [build-cpp-macos, build-cpp-ubuntu]
+    steps:
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi
+          git -C arrow submodule update --init --recursive
+          mkdir -p arrow/dist
+      - name: Download Linux Gandiva Library
+        uses: actions/download-artifact@v2
+        with:
+          name: ubuntu-shared-lib
+          path: arrow/dist
+      - name: Download MacOS Gandiva Library
+        uses: actions/download-artifact@v2
+        with:
+          name: macos-shared-lib
+          path: arrow/dist
+      - name: Build Gandiva Jar
+        run: |
+          cd arrow
+          export ARROW_TEST_DATA=$PWD/testing/data
+          set -e
+          dev/tasks/gandiva-jars/build-java.sh
+        env:
+          OS_NAME: "osx"
+          COPY_JAR_TO_DISTRIBUTION_FOLDER: true
+          MACOSX_DEPLOYMENT_TARGET: "10.11"
+
+      {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -658,15 +658,9 @@ tasks:
 
   ############################## Gandiva Tasks ################################
 
-  gandiva-jar-ubuntu:
+  gandiva-jar:
     ci: github
-    template: gandiva-jars/github.linux.yml
-    artifacts:
-      - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar
-
-  gandiva-jar-osx:
-    ci: github
-    template: gandiva-jars/github.osx.yml
+    template: gandiva-jars/github.yml
     artifacts:
       - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar