Skip to content

Commit

Permalink
OrcExec should read the specified file range data (#605)
Browse files Browse the repository at this point in the history
  • Loading branch information
harveyyue authored Oct 6, 2024
1 parent e536ce0 commit 948f59b
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 3 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ datafusion-expr = { git = "https://github.com/harveyyue/datafusion.git", rev = "
datafusion-execution = { git = "https://github.com/harveyyue/datafusion.git", rev = "d33877f8fbc7c57de946dc6081b2b357eedd0df9"}
datafusion-optimizer = { git = "https://github.com/harveyyue/datafusion.git", rev = "d33877f8fbc7c57de946dc6081b2b357eedd0df9"}
datafusion-physical-expr = { git = "https://github.com/harveyyue/datafusion.git", rev = "d33877f8fbc7c57de946dc6081b2b357eedd0df9"}
orc-rust = { git = "https://github.com/harveyyue/datafusion-orc.git", rev = "507600707a08c4bf7e2605a7ea40a5941456779d"}
orc-rust = { git = "https://github.com/harveyyue/datafusion-orc.git", rev = "f0ff4bcffa762b62e8c57ed4c2f6e1a9547b4abb"}

# arrow: branch=v50-blaze
arrow = { git = "https://github.com/blaze-init/arrow-rs.git", rev = "7471d70f7ae6edd5d4da82b7d966a8ede720e499"}
Expand Down
6 changes: 5 additions & 1 deletion native-engine/datafusion-ext-plans/src/orc_exec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,9 +221,13 @@ impl FileOpener for OrcOpener {
let schema_adapter = SchemaAdapter::new(projected_schema);

Ok(Box::pin(async move {
let builder = ArrowReaderBuilder::try_new_async(reader)
let mut builder = ArrowReaderBuilder::try_new_async(reader)
.await
.map_err(ArrowError::from)?;
if let Some(range) = file_meta.range.clone() {
let range = range.start as usize..range.end as usize;
builder = builder.with_file_byte_range(range);
}
let file_schema = builder.schema();
let (schema_mapping, adapted_projections) = schema_adapter.map_schema(&file_schema)?;
// Offset by 1 since index 0 is the root
Expand Down

0 comments on commit 948f59b

Please sign in to comment.