Skip to content

Commit

Permalink
[fix](nereids)Solve the problem of pruning wrong partitions in multi-…
Browse files Browse the repository at this point in the history 
…column partition pruning (apache#43332)

For example, with a partition defined as PARTITION BY RANGE (a, dt)
[(0, '2024-01-01 00:00:00'), (10, '2024-01-10 00:00:00')).
With the predicate：
WHERE a = 0 AND date_trunc(dt, 'day') <= '2024-01-10 00:00:00',

partition pruning will expand the partition ranges to:

a = 0, dt in ['2024-01-01 00:00:00', +∞)
a = 1, dt in (-∞, +∞)
a = 2, dt in (-∞, +∞)
...
a = 10, dt in (-∞, '2024-01-10 00:00:00')

Each of these eleven ranges will be evaluated against the predicate. If
all evaluations return False, the partition can be pruned.
During the evaluation of the first range
(a = 0, dt in ['2024-01-01 00:00:00', +∞)),
the range of date_trunc(dt, 'day') is calculated as
['2024-01-01', +∞) and stored in rangeMap.

However, subsequent evaluations (e.g., for a = 2, dt in (-∞, +∞)
 reuse this range ['2024-01-01', +∞),
which is incorrect. For a = 2, the correct range should be
(-∞, +∞) for date_trunc(dt, 'day').

Due to this incorrect reuse, the range for a = 2, dt in (-∞, +∞) will
incorrectly evaluate to False, causing improper pruning of the
partition.
The correct approach is to place rangeMap within the context, so that a
new rangeMap is constructed for each evaluation.
  • Loading branch information
@feiniaofeiafei
feiniaofeiafei committed Nov 12, 2024
1 parent fe7a88f commit 684b261
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 11 deletions.
27 changes: 16 additions & 11 deletions ...main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator
private final List<List<Expression>> inputs;
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
private final Map<Expression, ColumnRange> rangeMap = new HashMap<>();

/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots,
Expand Down Expand Up @@ -175,8 +174,8 @@ public List<Map<Slot, PartitionSlotInput>> getOnePartitionInputs() {
@Override
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
Map<Expression, ColumnRange> defaultColumnRanges = currentInputs.values().iterator().next().columnRanges;
rangeMap.putAll(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs));
Map<Expression, ColumnRange> rangeMap = new HashMap<>(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs, rangeMap));
return result.result;
}

Expand Down Expand Up @@ -397,6 +396,7 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.intersect(rightRange));

result = returnFalseIfExistEmptyRange(result);
Expand Down Expand Up @@ -425,6 +425,7 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) {
result.childrenResult);
}
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.union(rightRange));
return returnFalseIfExistEmptyRange(result);
}
Expand All @@ -437,8 +438,8 @@ public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
for (Map.Entry<Expression, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
Expression expr = entry.getKey();
ColumnRange childRange = entry.getValue();
ColumnRange partitionRange = rangeMap.containsKey(expr)
? rangeMap.get(expr) : ColumnRange.all();
ColumnRange partitionRange = context.rangeMap.containsKey(expr)
? context.rangeMap.get(expr) : ColumnRange.all();
newRanges.put(expr, partitionRange.intersect(childRange.complete()));
}
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
Expand Down Expand Up @@ -562,6 +563,7 @@ private Map<Expression, ColumnRange> replaceExprRange(Map<Expression, ColumnRang

private EvaluateRangeResult mergeRanges(
Expression originResult, EvaluateRangeResult left, EvaluateRangeResult right,
Map<Expression, ColumnRange> rangeMap,
BiFunction<ColumnRange, ColumnRange, ColumnRange> mergeFunction) {

Map<Expression, ColumnRange> leftRanges = left.columnRanges;
Expand Down Expand Up @@ -623,7 +625,7 @@ public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInpu
if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
partitionSlotContainsNull.put(dateTrunc, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

@Override
Expand All @@ -636,7 +638,7 @@ public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) {
if (partitionSlotContainsNull.containsKey(dateChild)) {
partitionSlotContainsNull.put(date, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

@Override
Expand All @@ -649,7 +651,7 @@ public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInpu
if (partitionSlotContainsNull.containsKey(converTzChild)) {
partitionSlotContainsNull.put(convertTz, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

private boolean isPartitionSlot(Slot slot) {
Expand Down Expand Up @@ -681,10 +683,12 @@ private Map<Slot, PartitionSlotInput> fillSlotRangesToInputs(

/** EvaluateRangeInput */
public static class EvaluateRangeInput {
private Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Expression, ColumnRange> rangeMap;

public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput) {
public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput, Map<Expression, ColumnRange> rangeMap) {
this.slotToInput = slotToInput;
this.rangeMap = rangeMap;
}
}

Expand Down Expand Up @@ -816,7 +820,8 @@ private List<Map<Slot, PartitionSlotInput>> commonComputeOnePartitionInputs() {
return onePartitionInputs;
}

private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) {
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result,
Map<Expression, ColumnRange> rangeMap) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
Expand Down
25 changes: 25 additions & 0 deletions regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,4 +334,29 @@ suite("test_date_trunc_prune") {
( NOT ( ( table1 . `col_int_undef_signed` != table1 . `col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' ) AND
table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1 ORDER BY field1 LIMIT 1000;
"""

// test multi column partition table and predicate with date_trunc
sql "drop table if exists t_multi_column_partition"
sql """
create table t_multi_column_partition(a int, dt datetime, v int) partition by range(a, dt)
(
partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')),
partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')),
partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')),
partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')),
partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00'))
)
distributed by hash(a) properties("replication_num"="1");
"""
sql """
insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),
(10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2),
(25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2),
(40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000),
(2,'2023-01-10 01:00:00',1000)
"""
explain {
sql """select * from t_multi_column_partition where a=2 and date_trunc(dt,'day') <'2024-01-1 00:00:00';"""
contains ("partitions=1/5 (p0)")
}
}

0 comments on commit 684b261

Please sign in to comment.