minor cleaning up, test IPC reader with more files

apache · Nov 19, 2019 · 84e31b7 · 84e31b7
1 parent c710076
commit 84e31b7
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 11 deletions.
diff --git a/rust/arrow/src/datatypes.rs b/rust/arrow/src/datatypes.rs
@@ -184,7 +184,8 @@ impl ArrowNativeType for u64 {
 
 impl ArrowNativeType for f32 {
     fn into_json_value(self) -> Option<Value> {
-        Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0).map(|num| VNumber(num))
+        Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0)
+            .map(|num| VNumber(num))
     }
 }
 

diff --git a/rust/arrow/src/ipc/file/reader.rs b/rust/arrow/src/ipc/file/reader.rs
@@ -34,7 +34,6 @@ static ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1'];
 /// Read a buffer based on offset and length
 fn read_buffer(buf: &ipc::Buffer, a_data: &Vec<u8>) -> Buffer {
     let start_offset = buf.offset() as usize;
-    dbg!(&buf);
     let end_offset = start_offset + buf.length() as usize;
     let buf_data = &a_data[start_offset..end_offset];
     Buffer::from(&buf_data)
@@ -174,7 +173,6 @@ fn create_array(
             );
             node_index = node_index + 1;
             buffer_index = buffer_index + 2;
-            dbg!((array.len(), &array));
             array
         }
     };
@@ -291,6 +289,8 @@ fn create_primitive_array(
     make_array(array_data)
 }
 
+/// Reads the correct number of buffers based on list type an null_count, and creates a
+/// list array ref
 fn create_list_array(
     field_node: &ipc::FieldNode,
     data_type: &DataType,
@@ -328,6 +328,7 @@ fn create_list_array(
     }
 }
 
+/// Creates a record batch from binary data using the `ipc::RecordBatch` indexes and the `Schema`
 fn read_record_batch(
     buf: &Vec<u8>,
     batch: ipc::RecordBatch,
@@ -530,9 +531,9 @@ mod tests {
         // the test is repetitive, thus we can read all supported files at once
         let paths = vec![
             // "generated_datetime",
-            // "generated_nested",
-            // "generated_primitive_no_batches",
-            // "generated_primitive_zerolength",
+            "generated_nested",
+            "generated_primitive_no_batches",
+            "generated_primitive_zerolength",
             "generated_primitive",
         ];
         paths.iter().for_each(|path| {

diff --git a/rust/arrow/src/util/integration_util.rs b/rust/arrow/src/util/integration_util.rs
@@ -63,7 +63,7 @@ struct ArrowJsonColumn {
 }
 
 impl ArrowJson {
-    // Compare the Arrow JSON with a record batch reader
+    /// Compare the Arrow JSON with a record batch reader
     pub fn equals_reader(&self, reader: &mut RecordBatchReader) -> bool {
         if !self.schema.equals_schema(&reader.schema()) {
             return false;
@@ -115,7 +115,6 @@ impl ArrowJsonBatch {
                     return false;
                 }
                 let json_array: Vec<Value> = json_from_col(&col, field.data_type());
-                println!("Data type: {:?}", field.data_type());
                 match field.data_type() {
                     DataType::Boolean => {
                         let arr = arr.as_any().downcast_ref::<BooleanArray>().unwrap();
@@ -158,9 +157,6 @@ impl ArrowJsonBatch {
                     }
                     DataType::Float32 => {
                         let arr = arr.as_any().downcast_ref::<Float32Array>().unwrap();
-                        dbg!(&arr);
-                        dbg!(&arr.len());
-                        dbg!(&json_array);
                         arr.equals_json(&json_array.iter().collect::<Vec<&Value>>()[..])
                     }
                     DataType::Float64 => {