Skip to content

Commit

Permalink
[enhance](nerieds) data_format, to_monday, last_day, to_date, year im…
Browse files Browse the repository at this point in the history
…plement Monotonic

[enhance](nerieds) data_format, to_monday, last_day, to_date, year implement Monotonic
  • Loading branch information
feiniaofeiafei committed Dec 17, 2024
1 parent 5fcf945 commit 34693be
Show file tree
Hide file tree
Showing 9 changed files with 262 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,9 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
context.rangeMap.get(func)), result.childrenResult);
}
if (!func.isMonotonic()) {
return result;
}
int childIndex = func.getMonotonicFunctionChildIndex();
Expression funcChild = func.child(childIndex);
boolean isNullable = partitionSlotContainsNull.getOrDefault(funcChild, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@

/** monotonicity of expressions */
public interface Monotonic extends ExpressionTrait {
default boolean isMonotonic() {
return true;
}

// true means that the function is an increasing function
boolean isPositive();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitConvertTz(this, context);
}

@Override
public boolean isMonotonic() {
return child(1).isConstant() && child(2).isConstant();
}

@Override
public boolean isPositive() {
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateTimeType;
Expand All @@ -32,14 +34,16 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import java.util.List;
import java.util.Set;

/**
* ScalarFunction 'date_format'. This class is generated by GenerateFunction.
*/
public class DateFormat extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral {
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic {

public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
Expand Down Expand Up @@ -74,4 +78,33 @@ public List<FunctionSignature> getSignatures() {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitDateFormat(this, context);
}

@Override
public boolean isMonotonic() {
Expression format = child(1);
if (!(format instanceof VarcharLiteral)) {
return false;
}
VarcharLiteral varcharLiteral = (VarcharLiteral) format;
String str = varcharLiteral.getValue();
Set<String> monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "%Y", "%Y-%m",
"%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", "%Y-%m-%d %T",
"%Y%m%d", "%Y%m");
return monoFormat.contains(str);
}

@Override
public boolean isPositive() {
return true;
}

@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}

@Override
public Expression withConstantArgs(Expression literal) {
return new DateFormat(literal, child(1));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -37,7 +38,7 @@
* ScalarFunction 'last_day'. This class is generated by GenerateFunction.
*/
public class LastDay extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args {
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic {

public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateV2Type.INSTANCE).args(DateV2Type.INSTANCE),
Expand Down Expand Up @@ -71,4 +72,19 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public boolean isPositive() {
return true;
}

@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}

@Override
public Expression withConstantArgs(Expression literal) {
return new LastDay(literal);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -38,7 +39,7 @@
* ScalarFunction 'to_date'. This class is generated by GenerateFunction.
*/
public class ToDate extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral {
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic {

private static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT),
Expand Down Expand Up @@ -70,4 +71,19 @@ public List<FunctionSignature> getSignatures() {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitToDate(this, context);
}

@Override
public boolean isPositive() {
return true;
}

@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}

@Override
public Expression withConstantArgs(Expression literal) {
return new ToDate(literal);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -37,7 +38,7 @@
* ScalarFunction 'to_monday'. This class is generated by GenerateFunction.
*/
public class ToMonday extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args {
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic {

private static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateV2Type.INSTANCE).args(DateV2Type.INSTANCE),
Expand Down Expand Up @@ -71,4 +72,19 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public boolean isPositive() {
return true;
}

@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}

@Override
public Expression withConstantArgs(Expression literal) {
return new ToMonday(literal);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -37,7 +38,7 @@
* ScalarFunction 'year'. This class is generated by GenerateFunction.
*/
public class Year extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args {
implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic {

private static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(SmallIntType.INSTANCE).args(DateV2Type.INSTANCE),
Expand Down Expand Up @@ -70,4 +71,19 @@ public List<FunctionSignature> getSignatures() {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitYear(this, context);
}

@Override
public boolean isPositive() {
return true;
}

@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}

@Override
public Expression withConstantArgs(Expression literal) {
return new Year(literal);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("always_mono_func") {
sql "drop table if exists always_mono_func"
sql """create table always_mono_func (a int, dt datetime, d date, c varchar(100)) duplicate key(a)
partition by range(dt) (
partition p1 values less than ("2017-01-01"),
partition p2 values less than ("2018-01-01"),
partition p3 values less than ("2019-01-01"),
partition p4 values less than ("2020-01-01"),
partition p5 values less than ("2021-01-01")
) distributed by hash(a) properties("replication_num"="1");"""
sql """INSERT INTO always_mono_func SELECT number,
date_add('2016-01-01 00:00:00', interval number month),
cast(date_add('2022-01-01 00:00:00', interval number month) as date), cast(number as varchar(65533)) FROM numbers('number'='55');"""
sql "INSERT INTO always_mono_func values(3,null,null,null);"

// format 测试满足格式的裁剪,不满足不裁剪
explain {
sql """select * from always_mono_func where date_format(dt, "yyyyMMdd") <"20190101" """
contains("partitions=3/5 (p1,p2,p3)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "yyyyMMdd") <"2019-01-01" """
contains("partitions=3/5 (p1,p2,p3)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") < "2019-01-01" """
contains("partitions=3/5 (p1,p2,p3)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") > "2019-01-01" """
contains("partitions=2/5 (p4,p5)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd HH:mm:ss") < "2019-01-01" """
contains("partitions=3/5 (p1,p2,p3)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") = "2019-01-01" """
contains("partitions=2/5 (p3,p4)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd HH:mm:ss") = "2019-01-01 00:00:00" """
contains("partitions=2/5 (p3,p4)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y") <= "2018" and date_format(dt, "%Y") > "2017" """
contains("partitions=2/5 (p2,p3)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y") <= "2019-01-01" """
contains("partitions=4/5 (p1,p2,p3,p4)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m")>="2019-01-01" """
contains("partitions=2/5 (p4,p5)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d")<>"2019-01-01" """
contains("partitions=5/5 (p1,p2,p3,p4,p5)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H")<="2019-01-01" """
contains("partitions=3/5 (p1,p2,p3)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i")> "2019-01-01" """
contains("partitions=3/5 (p3,p4,p5)")
}

explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i:%s")> "2019-01-01 00:00:00" """
contains("partitions=2/5 (p4,p5)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i:%S")> "2019-01-01 00:00:00" """
contains("partitions=2/5 (p4,p5)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %T") > "2019-01-01" """
contains("partitions=3/5 (p3,p4,p5)")
}
explain {
sql """select * from always_mono_func where date_format(dt, "%Y%m") > "20190101" """
contains("partitions=2/5 (p4,p5)")
}


explain {
sql """ select * from always_mono_func where last_day(dt) > "2019-01-01" """
contains("partitions=3/5 (p3,p4,p5)")
}

explain {
sql """ select * from always_mono_func where to_date(dt) > "2019-01-01" """
contains("partitions=2/5 (p4,p5)")
}

explain {
sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """
contains("partitions=2/5 (p4,p5)")
}

explain {
sql """ select * from always_mono_func where year(dt) >= 2019 """
contains("partitions=3/5 (p3,p4,p5)")
}

explain {
sql """select * from always_mono_func where date_format(to_monday(dt), 'yyyyMMdd') >= "20190101" """
contains("partitions=2/5 (p4,p5)")
}
explain {
sql """select * from always_mono_func where date_format(last_day(to_monday(dt)), 'yyyyMMdd') < "20190101" """
contains("partitions=4/5 (p1,p2,p3,p4)")
}
explain {
sql """select * from always_mono_func where date_format(last_day(to_monday(to_date(dt))), 'yyyyMMdd') <= "20190101" """
contains("partitions=4/5 (p1,p2,p3,p4)")
}

explain {
sql """select * from always_mono_func where date_format(date_trunc(last_day(to_monday(dt)),'month'), 'yyyyMMdd') > "20190101" """
contains("partitions=2/5 (p4,p5)")
}
}

0 comments on commit 34693be

Please sign in to comment.