diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 76bc57a6c712c..56f4bc71ec403 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -49,8 +49,8 @@ env:
 
 jobs:
 
-  debian:
-    name: ${{ matrix.title }}
+  ubuntu:
+    name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
@@ -58,23 +58,8 @@ jobs:
       fail-fast: false
       matrix:
         jdk: [8, 11, 17, 21]
-        include:
-        - jdk: 8
-          title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4
-          maven: 3.5.4
-          image: debian-java
-        - jdk: 11
-          title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2
-          maven: 3.6.2
-          image: debian-java
-        - jdk: 17
-          title: AMD64 Ubuntu 22.04 Java JDK 17 Maven 3.9.4
-          maven: 3.9.4
-          image: eclipse-java
-        - jdk: 21
-          title: AMD64 Ubuntu 22.04 Java JDK 21 Maven 3.9.4
-          maven: 3.9.4
-          image: eclipse-java
+        maven: [3.9.5]
+        image: [java]
     env:
       JDK: ${{ matrix.jdk }}
       MAVEN: ${{ matrix.maven }}
diff --git a/docker-compose.yml b/docker-compose.yml
index e54c609e54138..c8160a80dd033 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -135,11 +135,10 @@ x-hierarchy:
   - debian-go:
     - debian-go-cgo
     - debian-go-cgo-python
-  - debian-java
   - debian-js
-  - eclipse-java
   - fedora-cpp:
     - fedora-python
+  - java
   - python-sdist
   - ubuntu-cpp:
     - ubuntu-cpp-static
@@ -1676,14 +1675,14 @@ services:
 
   ################################ Java #######################################
 
-  debian-java:
+  java:
     # Usage:
-    #   docker-compose build debian-java
-    #   docker-compose run debian-java
+    #   docker-compose build java
+    #   docker-compose run java
     # Parameters:
-    #   MAVEN: 3.5.4, 3.6.2
-    #   JDK: 8, 11
-    image: ${ARCH}/maven:${MAVEN}-jdk-${JDK}
+    #   MAVEN: 3.9.5
+    #   JDK: 8, 11, 17, 21
+    image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
     shm_size: *shm-size
     volumes: &java-volumes
       - .:/arrow:delegated
@@ -1693,18 +1692,6 @@ services:
         /arrow/ci/scripts/java_build.sh /arrow /build &&
         /arrow/ci/scripts/java_test.sh /arrow /build"
 
-  eclipse-java:
-    # Usage:
-    #   docker-compose build eclipse-java
-    #   docker-compose run eclipse-java
-    # Parameters:
-    #   MAVEN: 3.9.4
-    #   JDK: 17, 21
-    image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
-    shm_size: *shm-size
-    volumes: *java-volumes
-    command: *java-command
-
   ############################## Integration ##################################
 
   conda-integration:
diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst
index 061c616d4b971..8b2a504631fdb 100644
--- a/docs/source/developers/java/building.rst
+++ b/docs/source/developers/java/building.rst
@@ -76,7 +76,7 @@ Docker compose
     $ cd arrow/java
     $ export JAVA_HOME=<absolute path to your java home>
     $ java --version
-    $ docker-compose run debian-java
+    $ docker-compose run java
 
 Archery
 ~~~~~~~
@@ -86,7 +86,7 @@ Archery
     $ cd arrow/java
     $ export JAVA_HOME=<absolute path to your java home>
     $ java --version
-    $ archery docker run debian-java
+    $ archery docker run java
 
 Building JNI Libraries (\*.dylib / \*.so / \*.dll)
 --------------------------------------------------