Zero-copy fixes

Lagrang · Aug 26, 2022 · a0cba8d · a0cba8d
1 parent 6d0a2d0
commit a0cba8d
Show file tree

Hide file tree

Showing 23 changed files with 41 additions and 32 deletions.
diff --git a/cpp/submodules/parquet-testing b/cpp/submodules/parquet-testing
diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml
@@ -16,7 +16,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml
@@ -16,7 +16,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml
@@ -87,7 +87,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <artifactId>arrow-algorithm</artifactId>
   <name>Arrow Algorithms</name>

diff --git a/java/c/pom.xml b/java/c/pom.xml
@@ -13,7 +13,7 @@
     <parent>
         <artifactId>arrow-java-root</artifactId>
         <groupId>org.apache.arrow</groupId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/compression/pom.xml b/java/compression/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <artifactId>arrow-compression</artifactId>
   <name>Arrow Compression</name>

diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml
@@ -15,7 +15,7 @@
     <parent>
         <artifactId>arrow-java-root</artifactId>
         <groupId>org.apache.arrow</groupId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java
@@ -17,6 +17,7 @@
 
 package org.apache.arrow.flight;
 
+import io.netty.buffer.PooledByteBufAllocator;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -130,15 +131,23 @@ public static HeaderType getHeader(byte b) {
   // Pre-allocated buffers for padding serialized ArrowMessages.
   private static final List<ByteBuf> PADDING_BUFFERS = Arrays.asList(
       null,
-      Unpooled.copiedBuffer(new byte[] { 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0, 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0, 0, 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0, 0, 0, 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0, 0, 0, 0, 0 }),
-      Unpooled.copiedBuffer(new byte[] { 0, 0, 0, 0, 0, 0, 0 })
+      paddingBuf(1),
+      paddingBuf(2),
+      paddingBuf(3),
+      paddingBuf(4),
+      paddingBuf(5),
+      paddingBuf(6),
+      paddingBuf(7)
   );
 
+  private static ByteBuf paddingBuf(int size) {
+    ByteBuf byteBuf = Unpooled.directBuffer(size, size);
+    byte[] padding = new byte[size];
+    Arrays.fill(padding, (byte) 0);
+    byteBuf.writeBytes(padding);
+    return byteBuf;
+  }
+
   private final IpcOption writeOption;
   private final FlightDescriptor descriptor;
   private final MessageMetadataResult message;
@@ -435,10 +444,10 @@ private InputStream asInputStream(BufferAllocator allocator) {
       cos.writeUInt32NoTag(size);
       cos.flush();
 
-      ByteBuf initialBuf = Unpooled.buffer(baos.size());
+      ByteBuf initialBuf = Unpooled.directBuffer(baos.size());
       initialBuf.writeBytes(baos.toByteArray());
       final CompositeByteBuf bb;
-      final int maxNumComponents = Math.max(2, bufs.size() + 1);
+      final int maxNumComponents = Math.max(2, allBufs.size() * 3);
       final ImmutableList<ByteBuf> byteBufs = ImmutableList.<ByteBuf>builder()
           .add(initialBuf)
           .addAll(allBufs)

diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-java-root</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>

diff --git a/java/format/pom.xml b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>6.0.0</version>
+  <version>6.0.100</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>

diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml
@@ -14,7 +14,7 @@
     <parent>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-java-root</artifactId>
-      <version>6.0.0</version>
+      <version>6.0.100</version>
     </parent>
 
     <groupId>org.apache.arrow.gandiva</groupId>

diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>arrow-memory</artifactId>
     <groupId>org.apache.arrow</groupId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/memory/pom.xml b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>

diff --git a/java/performance/pom.xml b/java/performance/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <artifactId>arrow-java-root</artifactId>
         <groupId>org.apache.arrow</groupId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
     </parent>
     <artifactId>arrow-performance</artifactId>
     <packaging>jar</packaging>
@@ -86,7 +86,7 @@
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-algorithm</artifactId>
-            <version>6.0.0</version>
+            <version>6.0.100</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

diff --git a/java/plasma/pom.xml b/java/plasma/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
     </parent>
     <artifactId>arrow-plasma</artifactId>
     <name>Arrow Plasma Client</name>

diff --git a/java/pom.xml b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>6.0.0</version>
+  <version>6.0.100</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>

diff --git a/java/tools/pom.xml b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>6.0.0</version>
+        <version>6.0.100</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>

diff --git a/java/vector/pom.xml b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>6.0.0</version>
+    <version>6.0.100</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

diff --git a/testing b/testing
+9 −4		data/README.md
+ −		data/data_index_bloom_encoding_stats.parquet
+52 −0		data/delta_byte_array.md
+ −		data/delta_byte_array.parquet
+1,001 −0		data/delta_byte_array_expect.csv
+69 −0		data/delta_encoding_optional_column.md
+ −		data/delta_encoding_optional_column.parquet
+101 −0		data/delta_encoding_optional_column_expect.csv
+71 −0		data/delta_encoding_required_column.md
+ −		data/delta_encoding_required_column.parquet
+101 −0		data/delta_encoding_required_column_expect.csv
+ −		data/null_list.parquet
+ −		data/arrow-ipc-file/clusterfuzz-testcase-minimized-arrow-ipc-file-fuzz-6177196536889344
+ −		data/arrow-ipc-file/clusterfuzz-testcase-minimized-arrow-ipc-file-fuzz-6318558565498880.fuzz
+ −		data/arrow-ipc-stream/clusterfuzz-testcase-minimized-arrow-ipc-stream-fuzz-4961281405222912
+ −		data/parquet/fuzzing/clusterfuzz-testcase-5913005913407488
+ −		data/parquet/fuzzing/clusterfuzz-testcase-6606237035003904
+ −		data/parquet/fuzzing/clusterfuzz-testcase-dictbitwidth-4680774947569664
+ −		data/parquet/fuzzing/clusterfuzz-testcase-dictbitwidth-5882232959270912
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-4738122420715520
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-4866999088447488
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-4938338763669504
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5004902418481152
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5103039558582272
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5106889906585600
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5152654819459072.fuzz
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5251250357141504
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5385788188131328
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5798108001337344
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-5915095763386368
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-6122962147737600.fuzz
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-6289584196026368
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-6539993600884736
+ −		data/parquet/fuzzing/clusterfuzz-testcase-minimized-parquet-arrow-fuzz-6696667471020032
+ −		data/parquet/fuzzing/clusterfuzz-testcase-parquet-arrow-fuzz-5004902418481152
+ −		data/parquet/fuzzing/clusterfuzz-testcase-parquet-arrow-fuzz-5973249794637824
+ −		data/parquet/fuzzing/clusterfuzz-testcase-parquet-arrow-fuzz-6196357887557632.fuzz
+ −		data/parquet/fuzzing/crash-61d6204d481340860da54e30f1937b67234ad0f7
+ −		data/parquet/fuzzing/crash-649c71a618ae2fd80cec177a9676eb3e280fc1fa
+ −		data/parquet/fuzzing/crash-9840a7b1a0d24996069f6ee0779bbe9875e8aca3