Skip to content

Commit

Permalink
Review feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
tustvold committed May 24, 2022
1 parent e39fa42 commit 2716ede
Show file tree
Hide file tree
Showing 13 changed files with 268 additions and 225 deletions.
2 changes: 1 addition & 1 deletion parquet/src/arrow/array_reader/list_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ mod tests {
.set_max_row_group_size(200)
.build();

let mut writer = ArrowWriter::try_new(
let writer = ArrowWriter::try_new(
file.try_clone().unwrap(),
Arc::new(arrow_schema),
Some(props),
Expand Down
4 changes: 2 additions & 2 deletions parquet/src/arrow/arrow_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ mod tests {
use crate::errors::Result;
use crate::file::properties::{WriterProperties, WriterVersion};
use crate::file::reader::{FileReader, SerializedFileReader};
use crate::file::writer::{FileWriter, SerializedFileWriter};
use crate::file::writer::SerializedFileWriter;
use crate::schema::parser::parse_message_type;
use crate::schema::types::{Type, TypePtr};
use crate::util::cursor::SliceableCursor;
Expand Down Expand Up @@ -952,7 +952,7 @@ mod tests {
writer.close()
}

fn get_test_reader(file_name: &str) -> Arc<dyn FileReader> {
fn get_test_reader(file_name: &str) -> Arc<SerializedFileReader<File>> {
let file = get_test_file(file_name);

let reader =
Expand Down
21 changes: 9 additions & 12 deletions parquet/src/arrow/arrow_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,8 @@ use super::schema::{
use crate::column::writer::ColumnWriter;
use crate::errors::{ParquetError, Result};
use crate::file::properties::WriterProperties;
use crate::file::writer::SerializedColumnWriter;
use crate::{
data_type::*,
file::writer::{FileWriter, RowGroupWriter, SerializedFileWriter},
};
use crate::file::writer::{SerializedColumnWriter, SerializedRowGroupWriter};
use crate::{data_type::*, file::writer::SerializedFileWriter};

/// Arrow writer
///
Expand Down Expand Up @@ -187,7 +184,7 @@ impl<W: Write> ArrowWriter<W> {
})
.collect();

write_leaves(row_group_writer.as_mut(), &arrays, &mut levels)?;
write_leaves(&mut row_group_writer, &arrays, &mut levels)?;
}

row_group_writer.close().unwrap();
Expand All @@ -197,25 +194,25 @@ impl<W: Write> ArrowWriter<W> {
}

/// Close and finalize the underlying Parquet writer
pub fn close(&mut self) -> Result<parquet_format::FileMetaData> {
pub fn close(mut self) -> Result<parquet_format::FileMetaData> {
self.flush()?;
self.writer.close()
}
}

/// Convenience method to get the next ColumnWriter from the RowGroupWriter
#[inline]
fn get_col_writer(
row_group_writer: &mut dyn RowGroupWriter,
) -> Result<SerializedColumnWriter<'_>> {
fn get_col_writer<'a, W: Write>(
row_group_writer: &'a mut SerializedRowGroupWriter<'_, W>,
) -> Result<SerializedColumnWriter<'a>> {
let col_writer = row_group_writer
.next_column()?
.expect("Unable to get column writer");
Ok(col_writer)
}

fn write_leaves(
row_group_writer: &mut dyn RowGroupWriter,
fn write_leaves<W: Write>(
row_group_writer: &mut SerializedRowGroupWriter<'_, W>,
arrays: &[ArrayRef],
levels: &mut [Vec<LevelInfo>],
) -> Result<()> {
Expand Down
4 changes: 2 additions & 2 deletions parquet/src/arrow/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1591,7 +1591,7 @@ mod tests {

// write to an empty parquet file so that schema is serialized
let file = tempfile::tempfile().unwrap();
let mut writer = ArrowWriter::try_new(
let writer = ArrowWriter::try_new(
file.try_clone().unwrap(),
Arc::new(schema.clone()),
None,
Expand Down Expand Up @@ -1660,7 +1660,7 @@ mod tests {

// write to an empty parquet file so that schema is serialized
let file = tempfile::tempfile().unwrap();
let mut writer = ArrowWriter::try_new(
let writer = ArrowWriter::try_new(
file.try_clone().unwrap(),
Arc::new(schema.clone()),
None,
Expand Down
22 changes: 11 additions & 11 deletions parquet/src/column/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
//! file::{
//! properties::WriterProperties,
//! reader::{FileReader, SerializedFileReader},
//! writer::{FileWriter, SerializedFileWriter},
//! writer::SerializedFileWriter,
//! },
//! schema::parser::parse_message_type,
//! };
Expand All @@ -66,17 +66,17 @@
//! let props = Arc::new(WriterProperties::builder().build());
//! let file = fs::File::create(path).unwrap();
//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
//! {
//! let mut row_group_writer = writer.next_row_group().unwrap();
//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
//! col_writer
//! .typed::<Int32Type>()
//! .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
//! .unwrap();
//! col_writer.close().unwrap();
//! }
//! row_group_writer.close().unwrap();
//!
//! let mut row_group_writer = writer.next_row_group().unwrap();
//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
//! col_writer
//! .typed::<Int32Type>()
//! .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
//! .unwrap();
//! col_writer.close().unwrap();
//! }
//! row_group_writer.close().unwrap();
//!
//! writer.close().unwrap();
//!
//! // Reading data using column reader API.
Expand Down
14 changes: 6 additions & 8 deletions parquet/src/file/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
//! use parquet::{
//! file::{
//! properties::WriterProperties,
//! writer::{FileWriter, SerializedFileWriter},
//! writer::SerializedFileWriter,
//! },
//! schema::parser::parse_message_type,
//! };
Expand All @@ -48,14 +48,12 @@
//! let props = Arc::new(WriterProperties::builder().build());
//! let file = fs::File::create(&path).unwrap();
//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
//! {
//! let mut row_group_writer = writer.next_row_group().unwrap();
//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
//! // ... write values to a column writer
//! col_writer.close().unwrap()
//! }
//! row_group_writer.close().unwrap();
//! let mut row_group_writer = writer.next_row_group().unwrap();
//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
//! // ... write values to a column writer
//! col_writer.close().unwrap()
//! }
//! row_group_writer.close().unwrap();
//! writer.close().unwrap();
//!
//! let bytes = fs::read(&path).unwrap();
Expand Down
Loading

0 comments on commit 2716ede

Please sign in to comment.