From 684b261b8fac579afe58ace927f8637fb43a5723 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 12 Nov 2024 11:24:49 +0800 Subject: [PATCH] [fix](nereids)Solve the problem of pruning wrong partitions in multi-column partition pruning (#43332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For example, with a partition defined as PARTITION BY RANGE (a, dt) [(0, '2024-01-01 00:00:00'), (10, '2024-01-10 00:00:00')). With the predicate： WHERE a = 0 AND date_trunc(dt, 'day') <= '2024-01-10 00:00:00', partition pruning will expand the partition ranges to: a = 0, dt in ['2024-01-01 00:00:00', +∞) a = 1, dt in (-∞, +∞) a = 2, dt in (-∞, +∞) ... a = 10, dt in (-∞, '2024-01-10 00:00:00') Each of these eleven ranges will be evaluated against the predicate. If all evaluations return False, the partition can be pruned. During the evaluation of the first range (a = 0, dt in ['2024-01-01 00:00:00', +∞)), the range of date_trunc(dt, 'day') is calculated as ['2024-01-01', +∞) and stored in rangeMap. However, subsequent evaluations (e.g., for a = 2, dt in (-∞, +∞) reuse this range ['2024-01-01', +∞), which is incorrect. For a = 2, the correct range should be (-∞, +∞) for date_trunc(dt, 'day'). Due to this incorrect reuse, the range for a = 2, dt in (-∞, +∞) will incorrectly evaluate to False, causing improper pruning of the partition. The correct approach is to place rangeMap within the context, so that a new rangeMap is constructed for each evaluation. --- .../rules/OneRangePartitionEvaluator.java | 27 +++++++++++-------- .../test_date_trunc_prune.groovy | 25 +++++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index c3345c346932a1..84a037171f32c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator private final List> inputs; private final Map partitionSlotContainsNull; private final Map slotToType; - private final Map rangeMap = new HashMap<>(); /** OneRangePartitionEvaluator */ public OneRangePartitionEvaluator(long partitionId, List partitionSlots, @@ -175,8 +174,8 @@ public List> getOnePartitionInputs() { @Override public Expression evaluate(Expression expression, Map currentInputs) { Map defaultColumnRanges = currentInputs.values().iterator().next().columnRanges; - rangeMap.putAll(defaultColumnRanges); - EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs)); + Map rangeMap = new HashMap<>(defaultColumnRanges); + EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs, rangeMap)); return result.result; } @@ -397,6 +396,7 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) { EvaluateRangeResult result = evaluateChildrenThenThis(and, context); result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1), + context.rangeMap, (leftRange, rightRange) -> leftRange.intersect(rightRange)); result = returnFalseIfExistEmptyRange(result); @@ -425,6 +425,7 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) { result.childrenResult); } result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1), + context.rangeMap, (leftRange, rightRange) -> leftRange.union(rightRange)); return returnFalseIfExistEmptyRange(result); } @@ -437,8 +438,8 @@ public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) { for (Map.Entry entry : result.childrenResult.get(0).columnRanges.entrySet()) { Expression expr = entry.getKey(); ColumnRange childRange = entry.getValue(); - ColumnRange partitionRange = rangeMap.containsKey(expr) - ? rangeMap.get(expr) : ColumnRange.all(); + ColumnRange partitionRange = context.rangeMap.containsKey(expr) + ? context.rangeMap.get(expr) : ColumnRange.all(); newRanges.put(expr, partitionRange.intersect(childRange.complete())); } result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult); @@ -562,6 +563,7 @@ private Map replaceExprRange(Map rangeMap, BiFunction mergeFunction) { Map leftRanges = left.columnRanges; @@ -623,7 +625,7 @@ public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInpu if (partitionSlotContainsNull.containsKey(dateTruncChild)) { partitionSlotContainsNull.put(dateTrunc, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } @Override @@ -636,7 +638,7 @@ public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) { if (partitionSlotContainsNull.containsKey(dateChild)) { partitionSlotContainsNull.put(date, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } @Override @@ -649,7 +651,7 @@ public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInpu if (partitionSlotContainsNull.containsKey(converTzChild)) { partitionSlotContainsNull.put(convertTz, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } private boolean isPartitionSlot(Slot slot) { @@ -681,10 +683,12 @@ private Map fillSlotRangesToInputs( /** EvaluateRangeInput */ public static class EvaluateRangeInput { - private Map slotToInput; + private final Map slotToInput; + private final Map rangeMap; - public EvaluateRangeInput(Map slotToInput) { + public EvaluateRangeInput(Map slotToInput, Map rangeMap) { this.slotToInput = slotToInput; + this.rangeMap = rangeMap; } } @@ -816,7 +820,8 @@ private List> commonComputeOnePartitionInputs() { return onePartitionInputs; } - private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) { + private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result, + Map rangeMap) { Monotonic func = (Monotonic) result.result; if (rangeMap.containsKey(func)) { return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func, diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy index 4ad3afbb9db040..c5eb3855247ad0 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy @@ -334,4 +334,29 @@ suite("test_date_trunc_prune") { ( NOT ( ( table1 . `col_int_undef_signed` != table1 . `col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' ) AND table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1 ORDER BY field1 LIMIT 1000; """ + + // test multi column partition table and predicate with date_trunc + sql "drop table if exists t_multi_column_partition" + sql """ + create table t_multi_column_partition(a int, dt datetime, v int) partition by range(a, dt) + ( + partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')), + partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')), + partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')), + partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')), + partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00')) + ) + distributed by hash(a) properties("replication_num"="1"); + """ + sql """ + insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2), + (10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2), + (25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2), + (40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000), + (2,'2023-01-10 01:00:00',1000) + """ + explain { + sql """select * from t_multi_column_partition where a=2 and date_trunc(dt,'day') <'2024-01-1 00:00:00';""" + contains ("partitions=1/5 (p0)") + } } \ No newline at end of file