From 078c5a5f01d0241960e1688e29b46f7312b83978 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Wed, 13 Mar 2024 16:12:52 -0400 Subject: [PATCH 01/23] Bump Maven to 3.8.8 --- dev/release/verify-release-candidate.sh | 2 +- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 08b3feac13e2f..e609efc6edf6b 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -20,7 +20,7 @@ # Requirements # - Ruby >= 2.3 -# - Maven >= 3.3.9 +# - Maven >= 3.8.8 # - JDK >=7 # - gcc >= 4.8 # - Node.js >= 18 diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 37d14ad412d88..b50305295e988 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -30,7 +30,7 @@ - 3.3.9 + 3.8.8 From cb7ec69302c647ee4d3e2df508135b1f6b2f4dd3 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Wed, 13 Mar 2024 16:59:25 -0400 Subject: [PATCH 02/23] Bump Maven to 3.8.8 in .env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 877bd50864d38..5f9317ab1cc1c 100644 --- a/.env +++ b/.env @@ -65,7 +65,7 @@ JDK=8 KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=14 -MAVEN=3.6.3 +MAVEN=3.8.8 NODE=18 NUMBA=latest NUMPY=latest From 04a7f0bce03fb49b6d3e96134777578aed0d0626 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Wed, 13 Mar 2024 17:12:06 -0400 Subject: [PATCH 03/23] Bump older versions of Maven to 3.8.8 in ci/docker/*.dockerfile --- ci/docker/conda-integration.dockerfile | 2 +- ci/docker/conda-python-hdfs.dockerfile | 2 +- ci/docker/conda-python-jpype.dockerfile | 2 +- ci/docker/conda-python-spark.dockerfile | 2 +- ci/docker/linux-apt-docs.dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index 074021677d6fd..70b9ea8384f43 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -20,7 +20,7 @@ ARG arch=amd64 FROM ${repo}:${arch}-conda-cpp ARG arch=amd64 -ARG maven=3.5 +ARG maven=3.8.8 ARG node=16 ARG yarn=1.22 ARG jdk=8 diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile index c7b01edf8c29c..018a5b51b86b2 100644 --- a/ci/docker/conda-python-hdfs.dockerfile +++ b/ci/docker/conda-python-hdfs.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.5 +ARG maven=3.8.8 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile index 0e7fecf03625b..baa384d7bbf38 100644 --- a/ci/docker/conda-python-jpype.dockerfile +++ b/ci/docker/conda-python-jpype.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=11 -ARG maven=3.6 +ARG maven=3.8.8 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index 221c840992440..24474f3c20167 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.5 +ARG maven=3.8.8 ARG numpy=latest COPY ci/scripts/install_numpy.sh /arrow/ci/scripts/ diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 5ac39424ed560..4d15fae29c12d 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -61,7 +61,7 @@ RUN apt-get update -y && \ ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 -ARG maven=3.6.3 +ARG maven=3.8.8 COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/util_download_apache.sh \ "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt From f57d0590e97d16568d2c86104c7366d21bf1d272 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Wed, 13 Mar 2024 17:28:04 -0400 Subject: [PATCH 04/23] Change version to 3.9.6 --- .env | 2 +- ci/docker/conda-integration.dockerfile | 2 +- ci/docker/conda-python-hdfs.dockerfile | 2 +- ci/docker/conda-python-jpype.dockerfile | 2 +- ci/docker/conda-python-spark.dockerfile | 2 +- ci/docker/linux-apt-docs.dockerfile | 2 +- dev/release/verify-release-candidate.sh | 2 +- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 5f9317ab1cc1c..66afeb0e3d525 100644 --- a/.env +++ b/.env @@ -65,7 +65,7 @@ JDK=8 KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=14 -MAVEN=3.8.8 +MAVEN=3.9.6 NODE=18 NUMBA=latest NUMPY=latest diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index 70b9ea8384f43..c856000b3671e 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -20,7 +20,7 @@ ARG arch=amd64 FROM ${repo}:${arch}-conda-cpp ARG arch=amd64 -ARG maven=3.8.8 +ARG maven=3.9.6 ARG node=16 ARG yarn=1.22 ARG jdk=8 diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile index 018a5b51b86b2..81aa567c302c7 100644 --- a/ci/docker/conda-python-hdfs.dockerfile +++ b/ci/docker/conda-python-hdfs.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.8.8 +ARG maven=3.9.6 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile index baa384d7bbf38..0fe5dd84b4b14 100644 --- a/ci/docker/conda-python-jpype.dockerfile +++ b/ci/docker/conda-python-jpype.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=11 -ARG maven=3.8.8 +ARG maven=3.9.6 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index 24474f3c20167..8972e389868dd 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.8.8 +ARG maven=3.9.6 ARG numpy=latest COPY ci/scripts/install_numpy.sh /arrow/ci/scripts/ diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 4d15fae29c12d..7c6302002f0b3 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -61,7 +61,7 @@ RUN apt-get update -y && \ ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 -ARG maven=3.8.8 +ARG maven=3.9.6 COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/util_download_apache.sh \ "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index e609efc6edf6b..c1804d5f9789b 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -20,7 +20,7 @@ # Requirements # - Ruby >= 2.3 -# - Maven >= 3.8.8 +# - Maven >= 3.9.6 # - JDK >=7 # - gcc >= 4.8 # - Node.js >= 18 diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index b50305295e988..2dacc97342cfa 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -30,7 +30,7 @@ - 3.8.8 + 3.9.6 From 4eb604c3ff6581efa24bfaca24b9d2aecd2e096a Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Wed, 13 Mar 2024 17:36:22 -0400 Subject: [PATCH 05/23] Change version to 3.8.7 --- .env | 2 +- ci/docker/conda-integration.dockerfile | 2 +- ci/docker/conda-python-hdfs.dockerfile | 2 +- ci/docker/conda-python-jpype.dockerfile | 2 +- ci/docker/conda-python-spark.dockerfile | 2 +- ci/docker/linux-apt-docs.dockerfile | 2 +- dev/release/verify-release-candidate.sh | 2 +- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 66afeb0e3d525..aba11f972f5d3 100644 --- a/.env +++ b/.env @@ -65,7 +65,7 @@ JDK=8 KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=14 -MAVEN=3.9.6 +MAVEN=3.8.7 NODE=18 NUMBA=latest NUMPY=latest diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index c856000b3671e..8406a419c06ab 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -20,7 +20,7 @@ ARG arch=amd64 FROM ${repo}:${arch}-conda-cpp ARG arch=amd64 -ARG maven=3.9.6 +ARG maven=3.8.7 ARG node=16 ARG yarn=1.22 ARG jdk=8 diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile index 81aa567c302c7..fa4fa0d1fb772 100644 --- a/ci/docker/conda-python-hdfs.dockerfile +++ b/ci/docker/conda-python-hdfs.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.9.6 +ARG maven=3.8.7 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile index 0fe5dd84b4b14..d9b43afdaec9e 100644 --- a/ci/docker/conda-python-jpype.dockerfile +++ b/ci/docker/conda-python-jpype.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=11 -ARG maven=3.9.6 +ARG maven=3.8.7 RUN mamba install -q -y \ maven=${maven} \ openjdk=${jdk} \ diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index 8972e389868dd..866f6f37f8bd9 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -21,7 +21,7 @@ ARG python=3.8 FROM ${repo}:${arch}-conda-python-${python} ARG jdk=8 -ARG maven=3.9.6 +ARG maven=3.8.7 ARG numpy=latest COPY ci/scripts/install_numpy.sh /arrow/ci/scripts/ diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 7c6302002f0b3..76b5ae6f14363 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -61,7 +61,7 @@ RUN apt-get update -y && \ ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 -ARG maven=3.9.6 +ARG maven=3.8.7 COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/util_download_apache.sh \ "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c1804d5f9789b..c66ade6ebb697 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -20,7 +20,7 @@ # Requirements # - Ruby >= 2.3 -# - Maven >= 3.9.6 +# - Maven >= 3.8.7 # - JDK >=7 # - gcc >= 4.8 # - Node.js >= 18 diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 2dacc97342cfa..d4e68c07c2b52 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -30,7 +30,7 @@ - 3.9.6 + 3.8.7 From 1a0b94e13925b738eed9cfc51d073000ed76f3da Mon Sep 17 00:00:00 2001 From: vibhatha Date: Thu, 14 Mar 2024 08:13:37 +0530 Subject: [PATCH 06/23] fix: testing conda usage for maven dep --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9b0610fe553b5..c3824e13297de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1979,4 +1979,4 @@ services: command: > /bin/bash -c " git config --global --add safe.directory /arrow && - /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" + USE_CONDA=1 /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" From b7ca8e0dafa710eadd5e706554ef1a8e1b8336e4 Mon Sep 17 00:00:00 2001 From: vibhatha Date: Thu, 14 Mar 2024 08:34:35 +0530 Subject: [PATCH 07/23] fix: testing conda usage --- dev/tasks/verify-rc/github.linux.amd64.docker.yml | 1 + docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/verify-rc/github.linux.amd64.docker.yml b/dev/tasks/verify-rc/github.linux.amd64.docker.yml index 65b30b5c8d4df..080d16311876b 100644 --- a/dev/tasks/verify-rc/github.linux.amd64.docker.yml +++ b/dev/tasks/verify-rc/github.linux.amd64.docker.yml @@ -41,6 +41,7 @@ jobs: -e VERIFY_RC="{{ rc|default("") }}" \ -e TEST_DEFAULT=0 \ -e TEST_{{ target|upper }}=1 \ + -e USE_CONDA=1 \ {{ distro }}-verify-rc {% if arrow.is_default_branch() %} diff --git a/docker-compose.yml b/docker-compose.yml index c3824e13297de..9b0610fe553b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1979,4 +1979,4 @@ services: command: > /bin/bash -c " git config --global --add safe.directory /arrow && - USE_CONDA=1 /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" + /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" From 7bac746559708cc5c1a707ea63cea02f78d6c325 Mon Sep 17 00:00:00 2001 From: vibhatha Date: Fri, 15 Mar 2024 08:19:06 +0530 Subject: [PATCH 08/23] fix: testing ci failure --- java/dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index a003fd18068ec..4c302ea59dbe3 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -150,7 +150,7 @@ commons-io commons-io - 2.15.1 + 2.7 test From fb1ecfe568ffd2fb92868bf69374f8ca4092d23d Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Wed, 20 Mar 2024 07:33:21 +0530 Subject: [PATCH 09/23] fix: reverse commons io downgrade --- java/dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 4c302ea59dbe3..a003fd18068ec 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -150,7 +150,7 @@ commons-io commons-io - 2.7 + 2.15.1 test From f5d023a4278b5efec44a88f4d6cea4fcaa724e8f Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Wed, 20 Mar 2024 09:06:15 +0530 Subject: [PATCH 10/23] fix: testing maven build functionality with upgrade on non-conda --- dev/tasks/verify-rc/github.linux.amd64.docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/verify-rc/github.linux.amd64.docker.yml b/dev/tasks/verify-rc/github.linux.amd64.docker.yml index 080d16311876b..65b30b5c8d4df 100644 --- a/dev/tasks/verify-rc/github.linux.amd64.docker.yml +++ b/dev/tasks/verify-rc/github.linux.amd64.docker.yml @@ -41,7 +41,6 @@ jobs: -e VERIFY_RC="{{ rc|default("") }}" \ -e TEST_DEFAULT=0 \ -e TEST_{{ target|upper }}=1 \ - -e USE_CONDA=1 \ {{ distro }}-verify-rc {% if arrow.is_default_branch() %} From ce1e39644e6ab970c3b321bf85eefa073f7e3482 Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Wed, 20 Mar 2024 11:11:37 +0530 Subject: [PATCH 11/23] fix: adding custom maven installation --- dev/release/verify-release-candidate.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c66ade6ebb697..c3c06e8d2eb4f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -76,6 +76,7 @@ esac # which is different from the ARROW_SOURCE_DIR set in ensure_source_directory() SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" ARROW_DIR="$(cd "${SOURCE_DIR}/../.." && pwd)" +MAVEN_VERSION=3.8.7 show_header() { echo "" @@ -468,6 +469,16 @@ install_conda() { conda activate base } +install_maven() { + show_info "Installing Maven version ${MAVEN_VERSION}..." + APACHE_MIRROR="http://www.apache.org/dyn/closer.cgi?action=download&filename=" + curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz + tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz + export PATH=`pwd`/apache-maven-${MAVEN_VERSION}/bin:$PATH + show_info "Installed Maven version $(mvn -v)" +} + maybe_setup_conda() { # Optionally setup conda environment with the passed dependencies local env="conda-${CONDA_ENV:-source}" @@ -566,6 +577,7 @@ test_package_java() { show_header "Build and test Java libraries" maybe_setup_conda maven openjdk + install_maven pushd java @@ -1203,6 +1215,7 @@ test_wheels() { test_jars() { show_header "Testing Java JNI jars" maybe_setup_conda maven python + install_maven local download_dir=${ARROW_TMPDIR}/jars mkdir -p ${download_dir} From 2a07df744fba3fdeba85159155061f756992a605 Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Wed, 20 Mar 2024 20:38:53 +0530 Subject: [PATCH 12/23] fix: addressing pr reviews --- dev/release/verify-release-candidate.sh | 27 +++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c3c06e8d2eb4f..b50c8a29d1f5b 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -76,7 +76,6 @@ esac # which is different from the ARROW_SOURCE_DIR set in ensure_source_directory() SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" ARROW_DIR="$(cd "${SOURCE_DIR}/../.." && pwd)" -MAVEN_VERSION=3.8.7 show_header() { echo "" @@ -469,14 +468,20 @@ install_conda() { conda activate base } -install_maven() { - show_info "Installing Maven version ${MAVEN_VERSION}..." - APACHE_MIRROR="http://www.apache.org/dyn/closer.cgi?action=download&filename=" - curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz - tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz - export PATH=`pwd`/apache-maven-${MAVEN_VERSION}/bin:$PATH - show_info "Installed Maven version $(mvn -v)" +maybe_setup_maven() { + MAVEN_VERSION=3.8.7 + SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') + if [[ $(echo "$SYSTEM_MAVEN_VERSION $MAVEN_VERSION" | awk '{print ($1 < $2)}') -eq 1 ]]; then + show_info "Installing Maven version ${MAVEN_VERSION}..." + APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" + curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz + tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz + export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH + show_info "Installed Maven version $(mvn -v)" + else + show_info "System Maven version is equal or newer than ${MAVEN_VERSION}. Skipping installation." + fi } maybe_setup_conda() { @@ -577,7 +582,7 @@ test_package_java() { show_header "Build and test Java libraries" maybe_setup_conda maven openjdk - install_maven + maybe_setup_maven pushd java @@ -1215,7 +1220,7 @@ test_wheels() { test_jars() { show_header "Testing Java JNI jars" maybe_setup_conda maven python - install_maven + maybe_setup_maven local download_dir=${ARROW_TMPDIR}/jars mkdir -p ${download_dir} From 686cc779bc316bd31ee7180c829dee667b65a419 Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Thu, 21 Mar 2024 08:01:44 +0530 Subject: [PATCH 13/23] fix: updating maven version verification logic --- dev/release/verify-release-candidate.sh | 50 ++++++++++++++++--------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index b50c8a29d1f5b..b9d56d47402cc 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -468,22 +468,6 @@ install_conda() { conda activate base } -maybe_setup_maven() { - MAVEN_VERSION=3.8.7 - SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') - if [[ $(echo "$SYSTEM_MAVEN_VERSION $MAVEN_VERSION" | awk '{print ($1 < $2)}') -eq 1 ]]; then - show_info "Installing Maven version ${MAVEN_VERSION}..." - APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" - curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz - tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz - export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH - show_info "Installed Maven version $(mvn -v)" - else - show_info "System Maven version is equal or newer than ${MAVEN_VERSION}. Skipping installation." - fi -} - maybe_setup_conda() { # Optionally setup conda environment with the passed dependencies local env="conda-${CONDA_ENV:-source}" @@ -515,6 +499,36 @@ maybe_setup_conda() { fi } +install_maven() { + MAVEN_VERSION=3.8.7 + SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') + + if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then + echo "Skip Install - versions are the same" + else + # the following logic only verifies released versions + older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sort --version-sort | head -n1) + if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" && "$SYSTEM_MAVEN_VERSION" != "$MAVEN_VERSION" ]]; then + show_info "Installing Maven version ${MAVEN_VERSION}..." + APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" + curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz + tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz + export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH + show_info "Installed Maven version $(mvn -v)" + else + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is equal or newer than asked Maven version ${MAVEN_VERSION}. Skipping installation." + fi + fi +} + +maybe_setup_maven() { + show_info "Ensuring that Maven is installed..." + if [ "${USE_CONDA}" -eq 0 ]; then + install_maven + fi +} + maybe_setup_virtualenv() { # Optionally setup pip virtualenv with the passed dependencies local env="venv-${VENV_ENV:-source}" @@ -581,8 +595,8 @@ maybe_setup_nodejs() { test_package_java() { show_header "Build and test Java libraries" - maybe_setup_conda maven openjdk maybe_setup_maven + maybe_setup_conda maven openjdk pushd java @@ -1219,8 +1233,8 @@ test_wheels() { test_jars() { show_header "Testing Java JNI jars" - maybe_setup_conda maven python maybe_setup_maven + maybe_setup_conda maven python local download_dir=${ARROW_TMPDIR}/jars mkdir -p ${download_dir} From ea290c5a928df289a98ac8261a7c53d679820475 Mon Sep 17 00:00:00 2001 From: Vibhatha Abeykoon Date: Thu, 21 Mar 2024 17:45:05 +0530 Subject: [PATCH 14/23] fix: address reviews --- dev/release/verify-release-candidate.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index b9d56d47402cc..c03d2c1ab5569 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -508,16 +508,16 @@ install_maven() { else # the following logic only verifies released versions older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sort --version-sort | head -n1) - if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" && "$SYSTEM_MAVEN_VERSION" != "$MAVEN_VERSION" ]]; then - show_info "Installing Maven version ${MAVEN_VERSION}..." - APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" - curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz - tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz - export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH - show_info "Installed Maven version $(mvn -v)" + if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then + show_info "Installing Maven version ${MAVEN_VERSION}..." + APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" + curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz + tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz + export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH + show_info "Installed Maven version $(mvn -v | head -n 1 | awk '{print $3}')" else - show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is equal or newer than asked Maven version ${MAVEN_VERSION}. Skipping installation." + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is equal or newer than asked Maven version ${MAVEN_VERSION}. Skipping installation." fi fi } @@ -1233,6 +1233,7 @@ test_wheels() { test_jars() { show_header "Testing Java JNI jars" + maybe_setup_maven maybe_setup_conda maven python From f233d5fd2cd3664f758bc761248f5260222d73b0 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 11:19:49 -0400 Subject: [PATCH 15/23] Require JDK >= 8 for RC verification Co-authored-by: Dane Pitkin <48041712+danepitkin@users.noreply.github.com> --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c03d2c1ab5569..166745c7ddcf7 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -21,7 +21,7 @@ # Requirements # - Ruby >= 2.3 # - Maven >= 3.8.7 -# - JDK >=7 +# - JDK >=8 # - gcc >= 4.8 # - Node.js >= 18 # - Go >= 1.19 From d7cae29b7e50404f550e3ac48aa2640e2177776a Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 11:29:34 -0400 Subject: [PATCH 16/23] Handle unreleased Maven versions in version check --- dev/release/verify-release-candidate.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 166745c7ddcf7..7e278529af8cc 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -506,8 +506,7 @@ install_maven() { if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then echo "Skip Install - versions are the same" else - # the following logic only verifies released versions - older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sort --version-sort | head -n1) + older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then show_info "Installing Maven version ${MAVEN_VERSION}..." APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" From c53f3993801bfb614480d9c098cadc4c806e600e Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 13:58:05 -0400 Subject: [PATCH 17/23] Improve Maven version check --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 7e278529af8cc..320e15aabe933 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -504,7 +504,7 @@ install_maven() { SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then - echo "Skip Install - versions are the same" + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." else older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then From 1c76b93d2e041934cd0eb4faf77834468cd43a19 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 13:58:17 -0400 Subject: [PATCH 18/23] Improve Maven version check --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 320e15aabe933..97036cbb6b5e8 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -516,7 +516,7 @@ install_maven() { export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH show_info "Installed Maven version $(mvn -v | head -n 1 | awk '{print $3}')" else - show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is equal or newer than asked Maven version ${MAVEN_VERSION}. Skipping installation." + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is newer than minimum required Maven version ${MAVEN_VERSION}. Skipping installation." fi fi } From 107527b99eef51b945ebfb4da7e60ecf2bfaa759 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 14:07:36 -0400 Subject: [PATCH 19/23] Shorten show_info --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 97036cbb6b5e8..daf44a635a09c 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -516,7 +516,7 @@ install_maven() { export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH show_info "Installed Maven version $(mvn -v | head -n 1 | awk '{print $3}')" else - show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is newer than minimum required Maven version ${MAVEN_VERSION}. Skipping installation." + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is newer than minimum version ${MAVEN_VERSION}. Skipping installation." fi fi } From 5ef6101989c555d7ceb13d21420fd28bb46a19b4 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 16:27:09 -0400 Subject: [PATCH 20/23] Linting Co-authored-by: Sutou Kouhei --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index daf44a635a09c..9396e5f2dce5f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -504,7 +504,7 @@ install_maven() { SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then - show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." else older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then From 61b5efbce1b34fb853ff318bff5075b9da09adf6 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 16:27:27 -0400 Subject: [PATCH 21/23] Linting Co-authored-by: Sutou Kouhei --- dev/release/verify-release-candidate.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 9396e5f2dce5f..75ea7d5b7a5de 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -506,18 +506,18 @@ install_maven() { if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." else - older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) - if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then - show_info "Installing Maven version ${MAVEN_VERSION}..." - APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" - curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz - tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz - export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH - show_info "Installed Maven version $(mvn -v | head -n 1 | awk '{print $3}')" - else - show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is newer than minimum version ${MAVEN_VERSION}. Skipping installation." - fi + older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) + if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then + show_info "Installing Maven version ${MAVEN_VERSION}..." + APACHE_MIRROR="https://www.apache.org/dyn/closer.lua?action=download&filename=" + curl -sL -o apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + ${APACHE_MIRROR}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz + tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz + export PATH=$(pwd)/apache-maven-${MAVEN_VERSION}/bin:$PATH + show_info "Installed Maven version $(mvn -v | head -n 1 | awk '{print $3}')" + else + show_info "System Maven version ${SYSTEM_MAVEN_VERSION} is newer than minimum version ${MAVEN_VERSION}. Skipping installation." + fi fi } From 9eea1601f66ff11649f60a0ae64127f31c3918d6 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 16:28:51 -0400 Subject: [PATCH 22/23] Handle case when Maven is not installed --- dev/release/verify-release-candidate.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 75ea7d5b7a5de..fe94d6e5db2ba 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -501,7 +501,13 @@ maybe_setup_conda() { install_maven() { MAVEN_VERSION=3.8.7 - SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') + if command -v mvn > /dev/null; then + SYSTEM_MAVEN_VERSION=$(mvn -v | head -n 1 | awk '{print $3}') + show_info "Found Maven version ${SYSTEM_MAVEN_VERSION} at $(command -v mvn)." + else + SYSTEM_MAVEN_VERSION=0.0.0 + show_info "Maven installation not found." + fi if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." From 9547977e5e4234de4d5a5d643d4aa495d00f8594 Mon Sep 17 00:00:00 2001 From: Ian Cook Date: Thu, 21 Mar 2024 17:55:59 -0400 Subject: [PATCH 23/23] Add comment explaining Maven version sort --- dev/release/verify-release-candidate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index fe94d6e5db2ba..d74ce1f67066d 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -512,6 +512,9 @@ install_maven() { if [[ "$MAVEN_VERSION" == "$SYSTEM_MAVEN_VERSION" ]]; then show_info "System Maven version ${SYSTEM_MAVEN_VERSION} matches required Maven version ${MAVEN_VERSION}. Skipping installation." else + # Append pipe character to make preview release versions like "X.Y.Z-beta-1" sort + # as older than their corresponding release version "X.Y.Z". This works because + # `sort -V` orders the pipe character lower than any version number character. older_version=$(printf '%s\n%s\n' "$SYSTEM_MAVEN_VERSION" "$MAVEN_VERSION" | sed 's/$/|/' | sort -V | sed 's/|$//' | head -n1) if [[ "$older_version" == "$SYSTEM_MAVEN_VERSION" ]]; then show_info "Installing Maven version ${MAVEN_VERSION}..."