diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile
index d9b43afdaec9e..3603ba2b7dd97 100644
--- a/ci/docker/conda-python-jpype.dockerfile
+++ b/ci/docker/conda-python-jpype.dockerfile
@@ -20,7 +20,7 @@ ARG arch=amd64
 ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
-ARG jdk=11
+ARG jdk=21
 ARG maven=3.8.7
 RUN mamba install -q -y \
         maven=${maven} \
diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile
deleted file mode 100644
index 8b73c73c1d240..0000000000000
--- a/ci/docker/java-jni-manylinux-201x.dockerfile
+++ /dev/null
@@ -1,55 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ARG base
-FROM ${base}
-
-# Install the libraries required by the Gandiva to run
-# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
-RUN vcpkg install \
-        --clean-after-build \
-        --x-install-root=${VCPKG_ROOT}/installed \
-        --x-manifest-root=/arrow/ci/vcpkg \
-        --x-feature=dev \
-        --x-feature=flight \
-        --x-feature=gcs \
-        --x-feature=json \
-        --x-feature=parquet \
-        --x-feature=gandiva \
-        --x-feature=s3
-
-# Install Java
-ARG java=1.8.0
-ARG maven=3.9.3
-RUN yum install -y java-$java-openjdk-devel && \
-      yum clean all && \
-      curl \
-        --fail \
-        --location \
-        "https://www.apache.org/dyn/closer.lua?action=download&filename=maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" | \
-        tar xfz - -C /usr/local && \
-      ln -s /usr/local/apache-maven-${maven}/bin/mvn /usr/local/bin
-
-# Install the gcs testbench
-COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
-RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default
-
-# For ci/scripts/{cpp,java}_*.sh
-ENV ARROW_HOME=/tmp/local \
-    ARROW_JAVA_CDATA=ON \
-    ARROW_JAVA_JNI=ON \
-    ARROW_USE_CCACHE=ON
diff --git a/docker-compose.yml b/docker-compose.yml
index 317a42977e534..940a96adc2002 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -169,8 +169,8 @@ x-hierarchy:
   - impala
   - postgres
   - python-wheel-manylinux-2014:
-    - java-jni-manylinux-2014
   - python-wheel-manylinux-2-28
+    - java-jni-manylinux-2-28
   - python-wheel-manylinux-test-imports
   - python-wheel-manylinux-test-unittests
   - python-wheel-windows-vs2019
@@ -1163,22 +1163,22 @@ services:
         target: "C:/arrow"
     command: arrow\\ci\\scripts\\python_wheel_windows_test.bat
 
-  java-jni-manylinux-2014:
-    image: ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
+  java-jni-manylinux-2-28:
+    image: ${REPO}:${ARCH}-java-jni-manylinux-2_28-vcpkg-${VCPKG}
     build:
       args:
         base: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
-        java: 1.8.0
+        java: 21
       context: .
-      dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile
+      dockerfile: ci/docker/java-jni-manylinux-2_28.dockerfile
       cache_from:
-        - ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
+        - ${REPO}:${ARCH}-java-jni-manylinux-2_28-vcpkg-${VCPKG}
     environment:
       <<: [*common, *ccache]
     volumes:
       - .:/arrow:delegated
       - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
-      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
+      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2_28-ccache:/ccache:delegated
     command:
       ["pip install -e /arrow/dev/archery && \
         /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \
diff --git a/java/bom/pom.xml b/java/bom/pom.xml
index 44127b8539ed1..0fac6bd4e7f4b 100644
--- a/java/bom/pom.xml
+++ b/java/bom/pom.xml
@@ -27,6 +27,16 @@
 
   <properties>
     <arrow.vector.classifier></arrow.vector.classifier>
+    <!-- org.apache:apache overrides -->
+    <minimalJavaBuildVersion>21</minimalJavaBuildVersion>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <maven.compiler.release>8</maven.compiler.release>
+    <!--
+      Downgrade maven-jar-plugin until https://github.com/codehaus-plexus/plexus-archiver/issues/332
+      is addressed
+    -->
+    <version.maven-jar-plugin>3.2.2</version.maven-jar-plugin>
   </properties>
 
   <dependencyManagement>
diff --git a/java/maven/pom.xml b/java/maven/pom.xml
index 43f352615b668..cdc4b514cb1e1 100644
--- a/java/maven/pom.xml
+++ b/java/maven/pom.xml
@@ -35,9 +35,15 @@
   <properties>
     <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
     <!-- org.apache:apache overrides -->
+    <minimalJavaBuildVersion>21</minimalJavaBuildVersion>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.release>8</maven.compiler.release>
+    <!--
+      Downgrade maven-jar-plugin until https://github.com/codehaus-plexus/plexus-archiver/issues/332
+      is addressed
+    -->
+    <version.maven-jar-plugin>3.2.2</version.maven-jar-plugin>
   </properties>
 
   <build>
diff --git a/java/pom.xml b/java/pom.xml
index 327c5a20a01de..980e0e79460db 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -103,6 +103,7 @@
     <!-- List of add-opens arg line arguments if tests are running on JDK 11 or higher -->
     <surefire.add-opens.argLine>--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</surefire.add-opens.argLine>
     <!-- org.apache:apache overrides -->
+    <minimalJavaBuildVersion>21</minimalJavaBuildVersion>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.release>8</maven.compiler.release>