Make Java 22 minimum version to build Arrow

Change miminum java build version to 22. This doesn't change the minimum version to use Arrow which is still Java 11 Update docker images for java jni/conda integration tests to use recent python image in order to install java 22
laurentgo · Jul 29, 2024 · 9bfdd59 · 9bfdd59
1 parent 5464525
commit 9bfdd59
Show file tree

Hide file tree

Showing 21 changed files with 202 additions and 80 deletions.
diff --git a/.env b/.env
@@ -61,7 +61,8 @@ GCC_VERSION=""
 GO=1.21.8
 STATICCHECK=v0.4.7
 HDFS=3.2.1
-JDK=11
+JDK=22
+TEST_JDK=11
 KARTOTHEK=latest
 # LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
 LLVM=14

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,12 +32,13 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Debian 12 Complete Documentation
+    name: AMD64 Ubuntu 24.04 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      JDK: 17
+      JDK: 22
+      UBUNTU: 24.04
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -50,8 +51,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: debian-docs-${{ hashFiles('cpp/**') }}
-          restore-keys: debian-docs-
+          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: ubuntu-docs-
       - name: Setup Python
         uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
@@ -62,8 +63,8 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-          JDK: 17
-        run: archery docker run debian-docs
+          JDK: 22
+        run: archery docker run ubuntu-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -74,4 +75,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push debian-docs
+        run: archery docker push ubuntu-docs
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -62,7 +62,8 @@ jobs:
         maven: [3.9.6]
         image: [java]
     env:
-      JDK: ${{ matrix.jdk }}
+      JDK: 22
+      TEST_JDK: ${{ matrix.jdk }}
       MAVEN: ${{ matrix.maven }}
     steps:
       - name: Checkout Arrow
@@ -91,6 +92,7 @@ jobs:
           archery docker run \
             -e CI=true \
             -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
+            -e "TEST_JDK=$TEST_JDK" \
             ${{ matrix.image }}
       - name: Docker Push
         if: >-
@@ -117,8 +119,10 @@ jobs:
       - name: Set up Java
         uses: actions/setup-java@v4
         with:
-          distribution: 'zulu'
-          java-version: ${{ matrix.jdk }}
+          distribution: 'temurin'
+          java-version: |
+            ${{ matrix.jdk }}
+            22
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:
@@ -128,11 +132,13 @@ jobs:
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+          TEST_JDK: ${{ matrix.jdk }}
         run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+          TEST_JDK: ${{ matrix.jdk }}
         run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
 
   windows:
@@ -148,8 +154,10 @@ jobs:
       - name: Set up Java
         uses: actions/setup-java@v4
         with:
-          java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
+          java-version: |
+            ${{ matrix.jdk }}
+            22
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:
@@ -159,9 +167,11 @@ jobs:
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+          TEST_JDK: ${{ matrix.jdk }}
         run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+          TEST_JDK: ${{ matrix.jdk }}
         run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -51,7 +51,7 @@ env:
 
 jobs:
   docker:
-    name: AMD64 manylinux2014 Java JNI
+    name: AMD64 manylinux Java JNI
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 240
@@ -68,8 +68,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }}
-          restore-keys: java-jni-manylinux-2014-
+          key: java-jni-manylinux-2-28-${{ hashFiles('cpp/**', 'java/**') }}
+          restore-keys: java-jni-manylinux-2-28-
       - name: Setup Python
         uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
@@ -80,7 +80,7 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run java-jni-manylinux-2014
+        run: archery docker run java-jni-manylinux-2-28
       - name: Docker Push
         if: >-
           success() &&
@@ -91,7 +91,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push java-jni-manylinux-2014
+        run: archery docker push java-jni-manylinux-2-28
 
   docker_integration_python:
     name: AMD64 Conda Java C Data Interface Integration
@@ -125,6 +125,7 @@ jobs:
           archery docker run \
             -e CI=true \
             -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
+            -e "TEST_JDK=$TEST_JDK" \
             conda-python-java-integration
       - name: Docker Push
         if: >-

diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile
@@ -20,7 +20,7 @@ ARG arch=amd64
 ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
-ARG jdk=11
+ARG jdk=22
 ARG maven=3.8.7
 RUN mamba install -q -y \
         maven=${maven} \

diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile → ci/docker/java-jni-manylinux.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile → ci/docker/java-jni-manylinux.dockerfile
@@ -33,7 +33,7 @@ RUN vcpkg install \
         --x-feature=s3
 
 # Install Java
-ARG java=11
+ARG java=22
 ARG maven=3.9.3
 RUN yum install -y java-$java-openjdk-devel && \
       yum clean all && \

diff --git a/ci/docker/java.dockerfile b/ci/docker/java.dockerfile
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM maven:3.9.6-eclipse-temurin-22
+
+COPY --from=maven:3.9.6-eclipse-temurin-8 /opt/java/openjdk /opt/java/openjdk8
+COPY --from=maven:3.9.6-eclipse-temurin-11 /opt/java/openjdk /opt/java/openjdk11
+COPY --from=maven:3.9.6-eclipse-temurin-17 /opt/java/openjdk /opt/java/openjdk17
+COPY --from=maven:3.9.6-eclipse-temurin-21 /opt/java/openjdk /opt/java/openjdk21
+
+
+env JAVA8_HOME /opt/java/openjdk8
+env JAVA11_HOME /opt/java/openjdk11
+env JAVA17_HOME /opt/java/openjdk17
+env JAVA21_HOME /opt/java/openjdk21
+env JAVA22_HOME /opt/java/openjdk
+
+RUN find "$JAVA8_HOME/lib" "$JAVA11_HOME/lib" "$JAVA17_HOME/lib" "$JAVA21_HOME/lib" "$JAVA22_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \
+    ldconfig;
+
+COPY ci/maven-toolchains.xml /usr/share/maven/conf/toolchains.xml
+
+CMD ["mvn"]
diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
@@ -23,21 +23,18 @@ ARG jdk=11
 
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-# See R install instructions at https://cloud.r-project.org/bin/linux/
+# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu
 RUN apt-get update -y && \
     apt-get install -y \
         apt-transport-https \
+        software-properties-common \
         dirmngr \
         gpg \
         lsb-release && \
-    gpg --keyserver keyserver.ubuntu.com \
-        --recv-key 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 && \
-    gpg --export 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 | \
-        gpg --no-default-keyring \
-            --keyring /usr/share/keyrings/cran.gpg \
-            --import - && \
-    echo "deb [signed-by=/usr/share/keyrings/cran.gpg] https://cloud.r-project.org/bin/linux/$(lsb_release -is | tr 'A-Z' 'a-z') $(lsb_release -cs)-cran40/" | \
-        tee /etc/apt/sources.list.d/cran.list && \
+    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \
+        tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \
+    # NOTE: Only R >= 4.0 is available in this repo
+    add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \
     if [ -f /etc/apt/sources.list.d/debian.sources ]; then \
         sed -i \
             -e 's/main$/main contrib non-free non-free-firmware/g' \
@@ -47,8 +44,7 @@ RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
         autoconf-archive \
         automake \
-        chromium \
-        chromium-sandbox \
+        chromium-browser \
         curl \
         doxygen \
         gi-docgen \

diff --git a/ci/maven-toolchains.xml b/ci/maven-toolchains.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<toolchains xmlns="http://maven.apache.org/TOOLCHAINS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/TOOLCHAINS/1.1.0 http://maven.apache.org/xsd/toolchains-1.1.0.xsd">
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>11</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA11_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>17</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA17_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>21</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA21_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>22</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA22_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+</toolchains>
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
@@ -98,6 +98,10 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
   mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni"
 fi
 
+if [ "${TEST_JDK}" != "" ]; then
+  mvn="${mvn} -Darrow.test.jdk-version=${TEST_JDK}"
+fi
+
 ${mvn} clean install
 
 if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then

diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh
@@ -35,9 +35,14 @@ echo "=== Clear output directories and leftovers ==="
 rm -rf ${build_dir}
 
 echo "=== Building Arrow C++ libraries ==="
-devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \
+devtoolset_version=$(rpm -qa "devtoolset-*-gcc" "gcc-toolset-*-gcc" --queryformat %{VERSION} | \
                        grep -o "^[0-9]*")
-devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+if [ -d "/opt/rh/devtoolset-${devtoolset_version}" ]; then
+  devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+else
+  devtoolset_include_cpp="/opt/rh/gcc-toolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+fi
+
 : ${ARROW_ACERO:=ON}
 export ARROW_ACERO
 : ${ARROW_BUILD_TESTS:=ON}

diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh
@@ -35,6 +35,10 @@ mvn="${mvn} -T 2C"
 
 pushd ${source_dir}
 
+if [ "${TEST_JDK}" != "" ]; then
+  mvn="${mvn} -Darrow.test.jdk-version=${TEST_JDK}"
+fi
+
 ${mvn} clean test
 
 projects=()

diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -57,7 +57,7 @@ jobs:
           archery docker run \
             -e ARROW_JAVA_BUILD=OFF \
             -e ARROW_JAVA_TEST=OFF \
-            java-jni-manylinux-2014
+            java-jni-manylinux-2-28
       - name: Compress into single artifact to keep directory structure
         run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
@@ -69,7 +69,7 @@ jobs:
       {{ macros.github_login_dockerhub()|indent }}
       - name: Push Docker image
         shell: bash
-        run: archery docker push java-jni-manylinux-2014
+        run: archery docker push java-jni-manylinux-2-28
     {% endif %}
 
   build-cpp-macos:
@@ -151,7 +151,7 @@ jobs:
         run: |
           set -e
           # make brew Java available to CMake
-          export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
+          export JAVA_HOME=$(brew --prefix openjdk@22)/libexec/openjdk.jdk/Contents/Home
           arrow/ci/scripts/java_jni_macos_build.sh \
             $GITHUB_WORKSPACE/arrow \
             $GITHUB_WORKSPACE/arrow/cpp-build \
@@ -172,7 +172,7 @@ jobs:
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
-          java-version: '11'
+          java-version: '22'
           distribution: 'temurin'
       - name: Download Timezone Database
         shell: bash