Skip to content

Commit

Permalink
Arrow2 test fixes (apache#18)
Browse files Browse the repository at this point in the history
* initialize the vector with zero strings when reading from arrow2->json and truncate empty records

* fixphysical_plan::planner::tests::bad_extension_planner
  • Loading branch information
Igosuki authored Jan 14, 2022
1 parent b5cb938 commit e53d165
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 7 deletions.
3 changes: 2 additions & 1 deletion datafusion/src/physical_plan/file_format/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ impl<R: BufRead> Iterator for JsonBatchReader<R> {

fn next(&mut self) -> Option<Self::Item> {
// json::read::read_rows iterates on the empty vec and reads at most n rows
let mut rows: Vec<String> = Vec::with_capacity(self.batch_size);
let mut rows = vec![String::default(); self.batch_size];
let read = json::read::read_rows(&mut self.reader, rows.as_mut_slice());
read.and_then(|records_read| {
if records_read > 0 {
Expand All @@ -81,6 +81,7 @@ impl<R: BufRead> Iterator for JsonBatchReader<R> {
} else {
self.schema.fields.clone()
};
rows.truncate(records_read);
json::read::deserialize(&rows, fields).map(Some)
} else {
Ok(None)
Expand Down
8 changes: 2 additions & 6 deletions datafusion/src/physical_plan/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1654,18 +1654,14 @@ mod tests {
name: \"a\", \
data_type: Int32, \
nullable: false, \
dict_id: 0, \
dict_is_ordered: false, \
metadata: None } }\
metadata: {} } }\
] }, \
ExecutionPlan schema: Schema { fields: [\
Field { \
name: \"b\", \
data_type: Int32, \
nullable: false, \
dict_id: 0, \
dict_is_ordered: false, \
metadata: None }\
metadata: {} }\
], metadata: {} }";
match plan {
Ok(_) => panic!("Expected planning failure"),
Expand Down

0 comments on commit e53d165

Please sign in to comment.