From b319661cb587d3ae754f778211104d4e1382f96e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 18 Aug 2022 14:45:23 +0900 Subject: [PATCH] ARROW-17451: [CI][Java] Use manylinux2014 image for JNI Because our official .jar packages are built in manylinux2014 image. --- .github/workflows/java_jni.yml | 8 ++-- ci/docker/java-jni-manylinux-201x.dockerfile | 9 ++++- ci/scripts/java_build.sh | 7 +++- ci/scripts/java_jni_macos_build.sh | 13 ++++--- ci/scripts/java_jni_manylinux_build.sh | 12 +++--- ci/scripts/java_test.sh | 9 +++-- cpp/src/plasma/CMakeLists.txt | 10 ++--- dev/tasks/java-jars/github.yml | 8 +++- docker-compose.yml | 40 +++----------------- 9 files changed, 56 insertions(+), 60 deletions(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 07cc3b1265212..6520c9eb3d0d0 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -47,7 +47,7 @@ env: jobs: docker: - name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset) + name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 90 @@ -70,14 +70,14 @@ jobs: - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build - run: archery docker run debian-java-jni + run: archery docker run java-jni-manylinux-2014 - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' continue-on-error: true - run: archery docker push debian-java-jni + run: archery docker push java-jni-manylinux-2014 docker_integration_python: - name: AMD64 Debian 9 Java C Data Interface Integration + name: AMD64 Conda Java C Data Interface Integration runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 90 diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile index 52bdb9b923dec..427974bf7dba7 100644 --- a/ci/docker/java-jni-manylinux-201x.dockerfile +++ b/ci/docker/java-jni-manylinux-201x.dockerfile @@ -33,5 +33,12 @@ RUN vcpkg install \ # Install Java ARG java=1.8.0 -RUN yum install -y java-$java-openjdk-devel && yum clean all +RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/ + +# For ci/scripts/java_*.sh +ENV ARROW_GANDIVA_JAVA=ON \ + ARROW_HOME=/tmp/local \ + ARROW_JAVA_CDATA=ON \ + ARROW_JNI=ON \ + ARROW_PLASMA=ON diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index b0362868b0a4e..6960ed0770bf6 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -18,10 +18,13 @@ set -ex +if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then + exit +fi + arrow_dir=${1} source_dir=${1}/java build_dir=${2} -cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug} java_jni_dist_dir=${3} : ${BUILD_DOCS_JAVA:=OFF} @@ -80,7 +83,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then fi if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then - ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install + ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install fi if [ "${ARROW_PLASMA}" = "ON" ]; then diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 590c469e398ff..9e1eecd9648dc 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -29,6 +29,7 @@ echo "=== Clear output directories and leftovers ===" rm -rf ${build_dir} echo "=== Building Arrow C++ libraries ===" +install_dir=${build_dir}/cpp-install : ${ARROW_BUILD_TESTS:=OFF} : ${ARROW_DATASET:=ON} : ${ARROW_FILESYSTEM:=ON} @@ -80,7 +81,7 @@ cmake \ -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \ + -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DPARQUET_BUILD_EXAMPLES=OFF \ -DPARQUET_BUILD_EXECUTABLES=OFF \ @@ -105,9 +106,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \ echo "=== Copying libraries to the distribution folder ===" mkdir -p "${dist_dir}" -cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir} -cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir} -cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir} +cp -L ${install_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir} +cp -L ${install_dir}/lib/libarrow_orc_jni.dylib ${dist_dir} +cp -L ${install_dir}/lib/libgandiva_jni.dylib ${dist_dir} +cp -L ${build_dir}/cpp/*/libplasma_java.dylib ${dist_dir} echo "=== Checking shared dependencies for libraries ===" @@ -126,5 +128,6 @@ archery linking check-dependencies \ libarrow_cdata_jni.dylib \ libarrow_dataset_jni.dylib \ libarrow_orc_jni.dylib \ - libgandiva_jni.dylib + libgandiva_jni.dylib \ + libplasma_java.dylib popd diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 008f19140ee0e..d6437cabe0a5c 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -91,7 +91,7 @@ cmake \ -DARROW_ZSTD_USE_SHARED=OFF \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \ + -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DORC_SOURCE=BUNDLED \ -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \ @@ -128,9 +128,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \ echo "=== Copying libraries to the distribution folder ===" -cp -L ${build_dir}/cpp/lib/libgandiva_jni.so ${dist_dir} -cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.so ${dist_dir} -cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.so ${dist_dir} +cp -L ${ARROW_HOME}/lib/libarrow_dataset_jni.so ${dist_dir} +cp -L ${ARROW_HOME}/lib/libarrow_orc_jni.so ${dist_dir} +cp -L ${ARROW_HOME}/lib/libgandiva_jni.so ${dist_dir} +cp -L ${build_dir}/cpp/*/libplasma_java.so ${dist_dir} echo "=== Checking shared dependencies for libraries ===" @@ -149,5 +150,6 @@ archery linking check-dependencies \ libarrow_cdata_jni.so \ libarrow_dataset_jni.so \ libarrow_orc_jni.so \ - libgandiva_jni.so + libgandiva_jni.so \ + libplasma_java.so popd diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh index 83ef26fdb1a51..bb30894d9eff0 100755 --- a/ci/scripts/java_test.sh +++ b/ci/scripts/java_test.sh @@ -18,9 +18,12 @@ set -ex +if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then + exit +fi + arrow_dir=${1} source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} java_jni_dist_dir=${3} # For JNI and Plasma tests @@ -36,7 +39,7 @@ pushd ${source_dir} ${mvn} test if [ "${ARROW_JNI}" = "ON" ]; then - ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir} + ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${java_jni_dist_dir} fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then @@ -46,7 +49,7 @@ fi if [ "${ARROW_PLASMA}" = "ON" ]; then pushd ${source_dir}/plasma java -cp target/test-classes:target/classes \ - -Djava.library.path=${cpp_build_dir} \ + -Djava.library.path=${java_jni_dist_dir} \ org.apache.arrow.plasma.PlasmaClientTest popd fi diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt index d78a5ccfdccbe..2573693738590 100644 --- a/cpp/src/plasma/CMakeLists.txt +++ b/cpp/src/plasma/CMakeLists.txt @@ -185,14 +185,14 @@ if(ARROW_PLASMA_JAVA_CLIENT) if(APPLE) target_link_libraries(plasma_java - plasma_shared - ${PLASMA_LINK_LIBS} + plasma_static + ${PLASMA_STATIC_LINK_LIBS} "-undefined dynamic_lookup" ${PTHREAD_LIBRARY}) - else(APPLE) - target_link_libraries(plasma_java plasma_shared ${PLASMA_LINK_LIBS} + else() + target_link_libraries(plasma_java plasma_static ${PLASMA_STATIC_LINK_LIBS} ${PTHREAD_LIBRARY}) - endif(APPLE) + endif() endif() # # Unit tests diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 376ba78fe14db..ec6b39b37327c 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -28,7 +28,11 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} - name: Build C++ Libs - run: archery docker run java-jni-manylinux-2014 + run: | + archery docker run \ + -e ARROW_JAVA_BUILD=OFF \ + -e ARROW_JAVA_TEST=OFF \ + java-jni-manylinux-2014 - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/ - name: Upload Artifacts @@ -100,11 +104,13 @@ jobs: test -f arrow/java-dist/libarrow_cdata_jni.dylib test -f arrow/java-dist/libarrow_dataset_jni.dylib test -f arrow/java-dist/libgandiva_jni.dylib + test -f arrow/java-dist/libplasma_jni.dylib test -f arrow/java-dist/libarrow_orc_jni.dylib test -f arrow/java-dist/libarrow_cdata_jni.so test -f arrow/java-dist/libarrow_dataset_jni.so test -f arrow/java-dist/libarrow_orc_jni.so test -f arrow/java-dist/libgandiva_jni.so + test -f arrow/java-dist/libplasma_jni.so - name: Build Bundled Jar run: | set -e diff --git a/docker-compose.yml b/docker-compose.yml index e4363f9180bf7..d584c5e5fe431 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,7 +122,6 @@ x-hierarchy: - debian-go-cgo - debian-go-cgo-python - debian-java - - debian-java-jni - debian-js - fedora-cpp: - fedora-python @@ -999,10 +998,14 @@ services: <<: *ccache volumes: - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated command: - ["pip install -e /arrow/dev/archery && - /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"] + ["pip install -e /arrow/dev/archery && \ + /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \ + source /opt/rh/rh-maven35/enable && \ + /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \ + /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"] ############################## Integration ################################# @@ -1541,37 +1544,6 @@ services: /arrow/ci/scripts/java_build.sh /arrow /build && /arrow/ci/scripts/java_test.sh /arrow /build" - debian-java-jni: - # Includes plasma test, jni for gandiva and orc, and C data interface. - # Usage: - # docker-compose build debian-java - # docker-compose build debian-java-jni - # docker-compose run debian-java-jni - image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni - build: - context: . - dockerfile: ci/docker/linux-apt-jni.dockerfile - cache_from: - - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni - args: - llvm: ${LLVM} - shm_size: *shm-size - environment: - <<: *ccache - ARROW_BUILD_TESTS: "OFF" - ARROW_S3: "OFF" - ARROW_SUBSTRAIT: "OFF" - volumes: - - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated - command: - /bin/bash -c " - /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/java_dist && - /arrow/ci/scripts/java_build.sh /arrow /build /tmp/java_dist && - /arrow/ci/scripts/java_test.sh /arrow /build /tmp/java_dist" - oracle-java: # Usage: # docker-compose build oracle-java