DX-86156: upgrade netty to 4.1.104.Final (#63)

* DX-86156: Cherry pick a4a3d3f for apacheGH-39265 * Update README_DREMIO.md * ARROW-16996: [Java] Configure Netty/GRPC/Protobuf base on BOM configuration + upgrade of dependencies by CVE (apache#13544) - Configure Netty/GRPC/Protobuf base on BOM Bill Of Material configuration to dependencies versions be added by configuration (netty/netty#5994). - Upgrade Netty/GRPC/Protobuf dependencies. Netty [CVE](GHSA-269q-hmxg-m83q) Authored-by: david dali susanibar arce <[email protected]> Signed-off-by: David Li <[email protected]> * Revert "ARROW-16996: [Java] Configure Netty/GRPC/Protobuf base on BOM configuration + upgrade of dependencies by CVE (apache#13544)" This reverts commit 9b18459. * Update pom.xml Remove unused dependency. * apacheGH-38996: [Java] Update dependencies and plugins for JPMS modules (apache#38994) Update dependencies and plugins to versions that work correctly with JPMS modules. * Update several plugins to use module-enabled versions: * maven-compiler-plugin is updated to 3.11.0 * maven-shade-plugin is updated to 3.2.4 * maven-dependency-plugin is updated to 3.1.2 * CycloneDX is updated to 2.7.10 * Update grpc-java to 1.59 for module support * Update jackson to 2.16.0 as 2.15.1 had corrupt module-info.class files that broke module support. Existing tests only Users may need to update their own dependencies if they have the same ones. * Closes: apache#38996 Authored-by: James Duong <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]> * Update README_DREMIO.md --------- Signed-off-by: David Li <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]> Co-authored-by: panbingkun <[email protected]> Co-authored-by: david dali susanibar arce <[email protected]> Co-authored-by: James Duong <[email protected]>
dremio · Jan 12, 2024 · eecb9a6 · eecb9a6
1 parent 7e5b03e
commit eecb9a6
Show file tree

Hide file tree

Showing 11 changed files with 58 additions and 29 deletions.
diff --git a/README_DREMIO.md b/README_DREMIO.md
@@ -2,6 +2,8 @@ git checkout -b dremio_24.3_12.0 apache-arrow-12.0.1
 
 Apply starting from the bottom and working upwards.
 
+git cherry-pick ccae6e95d09ce62a5134e4d3a868c8d30c609338
+git cherry-pick a4a3d3f4825eb025657121e70c9d86e8d6ecff35
 git cherry-pick a225426e1400a583bcf4dbd90dfd45563ff78a46
 git cherry-pick edab273d91adbcbeab812f15eb2344940d2151f4
 git cherry-pick f2dbe77d1b320a54afc3c618ec3cb1879bd0d517

diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml
@@ -32,6 +32,7 @@
       <artifactId>arrow-vector</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml
@@ -58,10 +58,6 @@
       <groupId>io.grpc</groupId>
       <artifactId>grpc-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-context</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-protobuf</artifactId>
@@ -153,7 +149,13 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.1.1</version>
+        <!--
+          Downgrade maven-shade-plugin specifically for this module.
+          Using a newer version up to at least 3.5.1 will cause
+          issues in the arrow-tools tests looking up FlatBuffer
+          dependencies.
+         -->
+        <version>3.2.4</version>
         <executions>
           <execution>
             <id>shade-main</id>

diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml
@@ -49,12 +49,13 @@
     </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
-      <artifactId>grpc-core</artifactId>
+      <artifactId>grpc-stub</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-inprocess</artifactId>
+      <scope>test</scope>
     </dependency>
-     <dependency>
-       <groupId>io.grpc</groupId>
-       <artifactId>grpc-stub</artifactId>
-     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>

diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml
@@ -41,7 +41,6 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.4.1</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml
@@ -56,6 +56,7 @@
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-jdbc</artifactId>
       <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
@@ -74,6 +75,7 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.derby</groupId>

diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -71,7 +71,7 @@ public UnsafeDirectLittleEndian allocate(long size) {
   }
 
   public int getChunkSize() {
-    return allocator.chunkSize;
+    return allocator.chunkSize();
   }
 
   public long getHugeBufferSize() {
@@ -137,7 +137,6 @@ private class InnerAllocator extends PooledByteBufAllocator {
 
     private final PoolArena<ByteBuffer>[] directArenas;
     private final MemoryStatusThread statusThread;
-    private final int chunkSize;
 
     public InnerAllocator() {
       super(true);
@@ -150,8 +149,6 @@ public InnerAllocator() {
         throw new RuntimeException("Failure while initializing allocator.  Unable to retrieve direct arenas field.", e);
       }
 
-      this.chunkSize = directArenas[0].chunkSize;
-
       if (memoryLogger.isTraceEnabled()) {
         statusThread = new MemoryStatusThread();
         statusThread.start();
@@ -166,7 +163,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
 
       if (directArena != null) {
 
-        if (initialCapacity > directArena.chunkSize) {
+        if (initialCapacity > chunkSize()) {
           // This is beyond chunk size so we'll allocate separately.
           ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
 

diff --git a/java/performance/pom.xml b/java/performance/pom.xml
@@ -26,6 +26,7 @@
             <groupId>org.openjdk.jmh</groupId>
             <artifactId>jmh-core</artifactId>
             <version>${jmh.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.openjdk.jmh</groupId>
@@ -38,11 +39,13 @@
             <artifactId>arrow-vector</artifactId>
             <version>${project.version}</version>
             <classifier>${arrow.vector.classifier}</classifier>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory-core</artifactId>
             <version>${project.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
@@ -54,11 +57,13 @@
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
             <version>${dep.avro.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-avro</artifactId>
             <version>${project.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.h2database</groupId>
@@ -70,6 +75,7 @@
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-jdbc</artifactId>
             <version>${project.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
@@ -109,7 +115,6 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>2.2</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/java/pom.xml b/java/pom.xml
@@ -30,22 +30,24 @@
   <properties>
     <target.gen.source.path>${project.build.directory}/generated-sources</target.gen.source.path>
     <dep.junit.platform.version>1.9.0</dep.junit.platform.version>
-    <dep.junit.jupiter.version>5.9.0</dep.junit.jupiter.version>
-    <dep.slf4j.version>1.7.25</dep.slf4j.version>
-    <dep.guava-bom.version>31.1-jre</dep.guava-bom.version>
-    <dep.netty-bom.version>4.1.96.Final</dep.netty-bom.version>
-    <dep.grpc-bom.version>1.56.0</dep.grpc-bom.version>
-    <dep.protobuf-bom.version>3.21.9</dep.protobuf-bom.version>
-    <dep.jackson-bom.version>2.13.4</dep.jackson-bom.version>
+    <dep.junit.jupiter.version>5.10.1</dep.junit.jupiter.version>
+    <dep.slf4j.version>2.0.9</dep.slf4j.version>
+    <dep.guava-bom.version>32.1.3-jre</dep.guava-bom.version>
+    <dep.netty-bom.version>4.1.104.Final</dep.netty-bom.version>
+    <dep.grpc-bom.version>1.60.0</dep.grpc-bom.version>
+    <dep.protobuf-bom.version>3.23.1</dep.protobuf-bom.version>
+    <dep.jackson-bom.version>2.16.0</dep.jackson-bom.version>
     <dep.hadoop.version>2.7.1</dep.hadoop.version>
     <dep.fbs.version>1.12.0</dep.fbs.version>
     <dep.avro.version>1.10.0</dep.avro.version>
     <arrow.vector.classifier />
     <forkCount>2</forkCount>
     <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
     <errorprone.javac.version>9+181-r4173-1</errorprone.javac.version>
-    <error_prone_core.version>2.16</error_prone_core.version>
-    <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
+    <error_prone_core.version>2.22.0</error_prone_core.version>
+    <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
+    <mockito.core.version>5.5.0</mockito.core.version>
+    <mockito.inline.version>5.2.0</mockito.inline.version>
   </properties>
 
   <scm>
@@ -358,7 +360,7 @@
       <plugin>
         <groupId>org.cyclonedx</groupId>
         <artifactId>cyclonedx-maven-plugin</artifactId>
-        <version>2.7.6</version>
+        <version>2.7.10</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -375,7 +377,12 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-dependency-plugin</artifactId>
-          <version>3.0.1</version>
+          <!--
+            This appears to report a false positive with versions
+            greater than 3.1.2 (tested up to 3.6.0) when compiling
+            arrow-tools about Jackson being only used for tests.
+          -->
+          <version>3.1.2</version>
         </plugin>
         <plugin>
           <groupId>org.apache.rat</groupId>
@@ -392,6 +399,7 @@
           <artifactId>maven-compiler-plugin</artifactId>
           <version>${maven-compiler-plugin.version}</version>
           <configuration>
+            <useModulePath>false</useModulePath>
             <annotationProcessorPaths>
               <path>
                 <groupId>org.immutables</groupId>
@@ -405,6 +413,11 @@
           <artifactId>maven-enforcer-plugin</artifactId>
           <version>3.0.0-M2</version>
         </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>3.5.1</version>
+        </plugin>
         <plugin>
           <artifactId>maven-surefire-plugin</artifactId>
           <version>3.0.0-M7</version>

diff --git a/java/tools/pom.xml b/java/tools/pom.xml
@@ -40,6 +40,7 @@
         <dependency>
           <groupId>com.google.guava</groupId>
           <artifactId>guava</artifactId>
+          <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>

diff --git a/java/vector/pom.xml b/java/vector/pom.xml
@@ -177,7 +177,13 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.1.1</version>
+        <!--
+          Downgrade maven-shade-plugin specifically for this module.
+          Using a newer version up to at least 3.5.1 will cause
+          issues in the arrow-tools tests looking up FlatBuffer
+          dependencies.
+         -->
+        <version>3.2.4</version>
         <executions>
           <execution>
             <phase>package</phase>