Skip to content

Commit

Permalink
proper input schema for stats evaluator
Browse files Browse the repository at this point in the history
  • Loading branch information
nicklan committed Jun 26, 2024
1 parent 48f834b commit 18a1578
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 3 deletions.
3 changes: 1 addition & 2 deletions kernel/src/engine/arrow_expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,7 @@ fn extract_column<'array, 'path>(
let (_, field) = input_schema
.column_with_name(path_step)
.ok_or(ArrowError::SchemaError(format!(
"No such field in schema {:#?}: {}",
input_schema, path_step,
"Field '{path_step}' is not in schema: {input_schema:?}"
)))?;
let child_opt = array.column_by_name(path_step);
if child_opt.is_none() && field.is_nullable() {
Expand Down
4 changes: 3 additions & 1 deletion kernel/src/scan/data_skipping.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use std::sync::Arc;

use tracing::debug;

use crate::actions::{get_log_schema, ADD_NAME};
use crate::actions::visitors::SelectionVectorVisitor;
use crate::error::DeltaResult;
use crate::expressions::{BinaryOperator, Expression as Expr, UnaryOperator, VariadicOperator};
Expand Down Expand Up @@ -242,7 +243,8 @@ impl DataSkippingFilter {
// 3. The selection evaluator does DISTINCT(col(predicate), 'false') to produce true (= keep) when
// the predicate is true/null and false (= skip) when the predicate is false.
let select_stats_evaluator = engine.get_expression_handler().get_evaluator(
stats_schema.clone(),
// safety: kernel is very broken if we don't have the schema for Add actions
get_log_schema().project(&[ADD_NAME]).unwrap(),
STATS_EXPR.clone(),
DataType::STRING,
);
Expand Down

0 comments on commit 18a1578

Please sign in to comment.