Split out arrow-ipc #3022
Split out arrow-ipc #3022
Conversation
| @@ -22,6 +22,24 @@ use crate::{new_empty_array, Array, ArrayRef, StructArray}; | |||
| use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef}; | |||
| use std::sync::Arc; | |||
|
|
|||
| /// Trait for types that can read `RecordBatch`'s. | |||
| pub trait RecordBatchReader: Iterator<Item = Result<RecordBatch, ArrowError>> { | |||
There was a problem hiding this comment.
This is moved from the arrow crate to allow using it in arrow-ipc and a future arrow-csv
| } | ||
| } | ||
|
|
||
| #[cfg(feature = "lz4")] |
There was a problem hiding this comment.
We now have separate features for the different compression codecs
| @@ -1503,18 +1501,18 @@ mod tests { | |||
| ], | |||
| ) | |||
| .unwrap(); | |||
| let file_name = format!("target/debug/testdata/nulls_{}.arrow_file", suffix); | |||
| let mut file = tempfile::tempfile().unwrap(); | |||
There was a problem hiding this comment.
This was failing about files not existing, I suspect some issue with moving to a different crate. Fortunately switching to use tempfile is cleaner and avoids this issue
| use std::io::Seek; | ||
|
|
||
| #[test] | ||
| fn read_union_017() { |
There was a problem hiding this comment.
This test needed to be moved as test_util is still part of the top-level crate, this makes more sense as an integration test anyway so I think this is fine
| @@ -1626,45 +1610,6 @@ mod tests { | |||
| assert!(dict_tracker.written.contains_key(&2)); | |||
| } | |||
|
|
|||
| #[test] | |||
There was a problem hiding this comment.
Moved to an integration test
| use crate::ipc::CompressionType; | ||
|
|
||
| #[derive(Debug, Clone, Copy, PartialEq, Eq)] | ||
| pub enum CompressionCodec {} |
There was a problem hiding this comment.
We no longer use a stub CompressionCodec, and instead generate runtime errors if the necessary features aren't enabled. This is consistent with how we handle this in parquet and how we handle other optional features such as timezone support
|
Benchmark runs are scheduled for baseline = 108e7d2 and contender = deb6455. deb6455 is a master commit associated with this PR. Results will be available as each benchmark for each run completes. |
Which issue does this PR close?
Part of #2594
Rationale for this change
This will allow arrow-flight, parquet and potentially more to no longer need to depend on the full arrow crate
What changes are included in this PR?
Splits out the arrow-ipc crate
Are there any user-facing changes?
No breaking changes, there is a slight change in that CompressionCodec now contains all variants even when no features are enabled.