ARROW-17451: [CI][Java] Use manylinux2014 image for JNI

Because our official .jar packages are built in manylinux2014 image.
apache · Aug 19, 2022 · cd20ca6 · cd20ca6
1 parent 6e8f0e4
commit cd20ca6
Show file tree

Hide file tree

Showing 8 changed files with 51 additions and 55 deletions.
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -47,7 +47,7 @@ env:
 jobs:
 
   docker:
-    name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
+    name: AMD64 manylinux2014 Java JNI
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 90
@@ -70,14 +70,14 @@ jobs:
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build
-        run: archery docker run debian-java-jni
+        run: archery docker run java-jni-manylinux-2014
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
-        run: archery docker push debian-java-jni
+        run: archery docker push java-jni-manylinux-2014
 
   docker_integration_python:
-    name: AMD64 Debian 9 Java C Data Interface Integration
+    name: AMD64 Conda Java C Data Interface Integration
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 90

diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile
@@ -33,5 +33,12 @@ RUN vcpkg install \
 
 # Install Java
 ARG java=1.8.0
-RUN yum install -y java-$java-openjdk-devel && yum clean all
+RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all
 ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/
+
+# For ci/scripts/java_*.sh
+ENV ARROW_GANDIVA_JAVA=ON \
+    ARROW_HOME=/tmp/local \
+    ARROW_JAVA_CDATA=ON \
+    ARROW_JNI=ON \
+    ARROW_PLASMA=ON
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
@@ -18,10 +18,13 @@
 
 set -ex
 
+if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
+  exit
+fi
+
 arrow_dir=${1}
 source_dir=${1}/java
 build_dir=${2}
-cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug}
 java_jni_dist_dir=${3}
 
 : ${BUILD_DOCS_JAVA:=OFF}
@@ -80,7 +83,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
 fi
 
 if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
-  ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
+  ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
 fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then

diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh
@@ -29,6 +29,7 @@ echo "=== Clear output directories and leftovers ==="
 rm -rf ${build_dir}
 
 echo "=== Building Arrow C++ libraries ==="
+install_dir=${build_dir}/cpp-install
 : ${ARROW_BUILD_TESTS:=OFF}
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FILESYSTEM:=ON}
@@ -80,7 +81,7 @@ cmake \
   -DAWSSDK_SOURCE=BUNDLED \
   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
   -DCMAKE_INSTALL_LIBDIR=lib \
-  -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
+  -DCMAKE_INSTALL_PREFIX=${install_dir} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
   -DPARQUET_BUILD_EXAMPLES=OFF \
   -DPARQUET_BUILD_EXECUTABLES=OFF \
@@ -105,9 +106,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
 
 echo "=== Copying libraries to the distribution folder ==="
 mkdir -p "${dist_dir}"
-cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libarrow_orc_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libgandiva_jni.dylib ${dist_dir}
+cp -L ${build_dir}/cpp/*/libplasma_java.dylib ${dist_dir}
 
 echo "=== Checking shared dependencies for libraries ==="
 
@@ -126,5 +128,6 @@ archery linking check-dependencies \
   libarrow_cdata_jni.dylib \
   libarrow_dataset_jni.dylib \
   libarrow_orc_jni.dylib \
-  libgandiva_jni.dylib
+  libgandiva_jni.dylib \
+  libplasma_java.dylib
 popd
diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh
@@ -91,7 +91,7 @@ cmake \
   -DARROW_ZSTD_USE_SHARED=OFF \
   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
   -DCMAKE_INSTALL_LIBDIR=lib \
-  -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
+  -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
   -DORC_SOURCE=BUNDLED \
   -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
@@ -128,9 +128,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
 
 
 echo "=== Copying libraries to the distribution folder ==="
-cp -L ${build_dir}/cpp/lib/libgandiva_jni.so ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.so ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.so ${dist_dir}
+cp -L ${ARROW_HOME}/lib/libarrow_dataset_jni.so ${dist_dir}
+cp -L ${ARROW_HOME}/lib/libarrow_orc_jni.so ${dist_dir}
+cp -L ${ARROW_HOME}/lib/libgandiva_jni.so ${dist_dir}
+cp -L ${build_dir}/cpp/*/libplasma_java.so ${dist_dir}
 
 echo "=== Checking shared dependencies for libraries ==="
 
@@ -149,5 +150,6 @@ archery linking check-dependencies \
   libarrow_cdata_jni.so \
   libarrow_dataset_jni.so \
   libarrow_orc_jni.so \
-  libgandiva_jni.so
+  libgandiva_jni.so \
+  libplasma_java.so
 popd
diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh
@@ -18,9 +18,12 @@
 
 set -ex
 
+if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
+  exit
+fi
+
 arrow_dir=${1}
 source_dir=${1}/java
-cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
 java_jni_dist_dir=${3}
 
 # For JNI and Plasma tests
@@ -36,7 +39,7 @@ pushd ${source_dir}
 ${mvn} test
 
 if [ "${ARROW_JNI}" = "ON" ]; then
-  ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
+  ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${java_jni_dist_dir}
 fi
 
 if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
@@ -46,7 +49,7 @@ fi
 if [ "${ARROW_PLASMA}" = "ON" ]; then
   pushd ${source_dir}/plasma
   java -cp target/test-classes:target/classes \
-       -Djava.library.path=${cpp_build_dir} \
+       -Djava.library.path=${java_jni_dist_dir} \
        org.apache.arrow.plasma.PlasmaClientTest
   popd
 fi

diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -28,7 +28,11 @@ jobs:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       - name: Build C++ Libs
-        run: archery docker run java-jni-manylinux-2014
+        run: |
+          archery docker run \
+            -e ARROW_JAVA_BUILD=OFF \
+            -e ARROW_JAVA_TEST=OFF \
+            java-jni-manylinux-2014
       - name: Compress into single artifact to keep directory structure
         run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
       - name: Upload Artifacts
@@ -100,11 +104,13 @@ jobs:
           test -f arrow/java-dist/libarrow_cdata_jni.dylib
           test -f arrow/java-dist/libarrow_dataset_jni.dylib
           test -f arrow/java-dist/libgandiva_jni.dylib
+          test -f arrow/java-dist/libplasma_jni.dylib
           test -f arrow/java-dist/libarrow_orc_jni.dylib
           test -f arrow/java-dist/libarrow_cdata_jni.so
           test -f arrow/java-dist/libarrow_dataset_jni.so
           test -f arrow/java-dist/libarrow_orc_jni.so
           test -f arrow/java-dist/libgandiva_jni.so
+          test -f arrow/java-dist/libplasma_jni.so
       - name: Build Bundled Jar
         run: |
           set -e

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -122,7 +122,6 @@ x-hierarchy:
     - debian-go-cgo
     - debian-go-cgo-python
   - debian-java
-  - debian-java-jni
   - debian-js
   - fedora-cpp:
     - fedora-python
@@ -999,10 +998,14 @@ services:
       <<: *ccache
     volumes:
       - .:/arrow:delegated
+      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
     command:
-      ["pip install -e /arrow/dev/archery &&
-        /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"]
+      ["pip install -e /arrow/dev/archery && \
+        /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \
+        source /opt/rh/rh-maven35/enable && \
+        /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \
+        /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]
 
   ##############################  Integration #################################
 
@@ -1541,37 +1544,6 @@ services:
         /arrow/ci/scripts/java_build.sh /arrow /build &&
         /arrow/ci/scripts/java_test.sh /arrow /build"
 
-  debian-java-jni:
-    # Includes plasma test, jni for gandiva and orc, and C data interface.
-    # Usage:
-    #   docker-compose build debian-java
-    #   docker-compose build debian-java-jni
-    #   docker-compose run debian-java-jni
-    image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
-    build:
-      context: .
-      dockerfile: ci/docker/linux-apt-jni.dockerfile
-      cache_from:
-        - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
-      args:
-        llvm: ${LLVM}
-    shm_size: *shm-size
-    environment:
-      <<: *ccache
-      ARROW_BUILD_TESTS: "OFF"
-      ARROW_S3: "OFF"
-      ARROW_SUBSTRAIT: "OFF"
-    volumes:
-      - .:/arrow:delegated
-      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
-      - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
-    command:
-      /bin/bash -c "
-        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
-        /arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/java_dist &&
-        /arrow/ci/scripts/java_build.sh /arrow /build /tmp/java_dist &&
-        /arrow/ci/scripts/java_test.sh /arrow /build /tmp/java_dist"
-
   oracle-java:
     # Usage:
     #   docker-compose build oracle-java