ARROW-16584: [Java] Java JNI with S3 support (#13157)

macOS development target is changed to 10.13 from 10.11. See also the discussion on mailing list: https://lists.apache.org/thread/pjgjrl716gvqzql586cnnoxb38nb0j5w Lead-authored-by: Artavazd Balaian <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
apache · Jul 29, 2022 · 7fe71f5 · 7fe71f5
1 parent fa7e7a3
commit 7fe71f5
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 2 deletions.
diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile
@@ -28,7 +28,8 @@ RUN vcpkg install \
         --x-feature=gcs \
         --x-feature=json \
         --x-feature=parquet \
-        --x-feature=gandiva
+        --x-feature=gandiva \
+        --x-feature=s3
 
 # Install Java
 ARG java=1.8.0

diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile
@@ -47,6 +47,7 @@ RUN apt-get update -y -q && \
         g++ \
         gcc \
         libboost-all-dev \
+        libcurl4-openssl-dev \
         libgflags-dev \
         libgoogle-glog-dev \
         libgtest-dev \
@@ -81,6 +82,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_PARQUET=ON \
     ARROW_PLASMA_JAVA_CLIENT=ON \
     ARROW_PLASMA=ON \
+    ARROW_S3=ON \
     ARROW_USE_CCACHE=ON \
     CC=gcc \
     CXX=g++ \

diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh
@@ -39,6 +39,7 @@ echo "=== Building Arrow C++ libraries ==="
 : ${ARROW_PLASMA_JAVA_CLIENT:=ON}
 : ${ARROW_PLASMA:=ON}
 : ${ARROW_PYTHON:=OFF}
+: ${ARROW_S3:=ON}
 : ${CMAKE_BUILD_TYPE:=Release}
 : ${CMAKE_UNITY_BUILD:=ON}
 
@@ -71,10 +72,12 @@ cmake \
   -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT} \
   -DARROW_PROTOBUF_USE_SHARED=OFF \
   -DARROW_PYTHON=${ARROW_PYTHON} \
+  -DARROW_S3=${ARROW_S3} \
   -DARROW_SNAPPY_USE_SHARED=OFF \
   -DARROW_THRIFT_USE_SHARED=OFF \
   -DARROW_UTF8PROC_USE_SHARED=OFF \
   -DARROW_ZSTD_USE_SHARED=OFF \
+  -DAWSSDK_SOURCE=BUNDLED \
   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
   -DCMAKE_INSTALL_LIBDIR=lib \
   -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
@@ -110,11 +113,13 @@ echo "=== Checking shared dependencies for libraries ==="
 
 pushd ${dist_dir}
 archery linking check-dependencies \
+  --allow CoreFoundation \
   --allow libSystem \
   --allow libarrow_cdata_jni \
   --allow libarrow_dataset_jni \
   --allow libarrow_orc_jni \
   --allow libc++ \
+  --allow libcurl \
   --allow libgandiva_jni \
   --allow libncurses \
   --allow libz \

diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh
@@ -43,6 +43,7 @@ devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/includ
 : ${ARROW_PLASMA:=ON}
 : ${ARROW_PLASMA_JAVA_CLIENT:=ON}
 : ${ARROW_PYTHON:=OFF}
+: ${ARROW_S3:=ON}
 : ${ARROW_BUILD_TESTS:=OFF}
 : ${CMAKE_BUILD_TYPE:=Release}
 : ${CMAKE_UNITY_BUILD:=ON}
@@ -83,6 +84,7 @@ cmake \
   -DARROW_PROTOBUF_USE_SHARED=OFF \
   -DARROW_PYTHON=${ARROW_PYTHON} \
   -DARROW_RPATH_ORIGIN=${ARROW_RPATH_ORIGIN} \
+  -DARROW_S3=${ARROW_S3} \
   -DARROW_SNAPPY_USE_SHARED=OFF \
   -DARROW_THRIFT_USE_SHARED=OFF \
   -DARROW_UTF8PROC_USE_SHARED=OFF \

diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -47,14 +47,20 @@ jobs:
     name: Build C++ Libs MacOS
     runs-on: macos-latest
     env:
-      MACOSX_DEPLOYMENT_TARGET: "10.11"
+      MACOSX_DEPLOYMENT_TARGET: "10.13"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       - name: Install Dependencies
         run: |
           brew install --overwrite git
           brew bundle --file=arrow/cpp/Brewfile
+          # We want to link aws-sdk-cpp statically but Homebrew's
+          # aws-sdk-cpp provides only shared library. If we have
+          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
+          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
+          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
+          brew uninstall aws-sdk-cpp
       - name: Build C++ Libs
         run: |
           set -e