From b1139731961f19fc45ad48bb97a29e1bf0c58c5d Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Thu, 14 Mar 2024 21:10:45 +0800 Subject: [PATCH 01/15] [bug](fold) fix fold constant core dump with variant type --- be/src/runtime/fold_constant_executor.cpp | 10 +++++++++- .../main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 8a19ca1b35c81a..657d0b3c0119d0 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -63,6 +63,12 @@ using std::map; namespace doris { +static std::unordered_set PRIMITIVE_TYPE_SET { + TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, + TYPE_LARGEINT, TYPE_FLOAT, TYPE_TIME, TYPE_DOUBLE, TYPE_TIMEV2, + TYPE_CHAR, TYPE_VARCHAR, TYPE_STRING, TYPE_HLL, TYPE_OBJECT, + TYPE_DATE, TYPE_DATETIME, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_DECIMALV2}; + Status FoldConstantExecutor::fold_constant_vexpr(const TFoldConstantParams& params, PConstantExprResult* response) { const auto& expr_map = params.expr_map; @@ -106,7 +112,9 @@ Status FoldConstantExecutor::fold_constant_vexpr(const TFoldConstantParams& para } else { expr_result.set_success(true); StringRef string_ref; - if (!ctx->root()->type().is_complex_type()) { + auto type = ctx->root()->type().type; + //eg: strcut, array, map VARIANT... will not impl get_data_at, so could use column->to_string() + if (PRIMITIVE_TYPE_SET.contains(type)) { string_ref = column_ptr->get_data_at(0); } RETURN_IF_ERROR(_get_result((void*)string_ref.data, string_ref.size, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7a03865d1dc413..df0c867c4d0d6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -936,7 +936,7 @@ public class SessionVariable implements Serializable, Writable { private boolean enableJoinReorderBasedCost = false; @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) - private boolean enableFoldConstantByBe = false; + private boolean enableFoldConstantByBe = true; @VariableMgr.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; From 89d58a5d5045ccc9f0ab4328e4b5f8c34d9e1332 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Fri, 15 Mar 2024 11:15:57 +0800 Subject: [PATCH 02/15] update case --- .../src/test/java/org/apache/doris/analysis/SqlModeTest.java | 1 + .../test/java/org/apache/doris/catalog/CreateFunctionTest.java | 3 ++- .../java/org/apache/doris/planner/ConstantExpressTest.java | 1 + .../src/test/java/org/apache/doris/planner/QueryPlanTest.java | 1 + .../doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java | 2 +- .../java/org/apache/doris/rewrite/InferFiltersRuleTest.java | 2 +- .../org/apache/doris/rewrite/RewriteInPredicateRuleTest.java | 2 +- .../suites/correctness/test_date_function_const.groovy | 2 ++ regression-test/suites/correctness_p0/test_cast_null.groovy | 2 +- .../agg_state/nereids/test_agg_state_nereids.groovy | 3 ++- .../suites/nereids_tpcds_shape_sf1000_p0/shape/query21.groovy | 1 + .../nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query21.groovy | 1 + .../nereids_tpcds_shape_sf100_p0/no_stats_shape/query21.groovy | 1 + .../nereids_tpcds_shape_sf100_p0/rf_prune/query21.groovy | 1 + .../suites/nereids_tpcds_shape_sf100_p0/shape/query21.groovy | 1 + 15 files changed, 18 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SqlModeTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SqlModeTest.java index d70dfc826f0346..c27743a951ac06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SqlModeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SqlModeTest.java @@ -100,6 +100,7 @@ public void testPipesAsConcatModeNull() { } analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + analyzer.getContext().getSessionVariable().setEnableFoldConstantByBe(false); try { parsedStmt.analyze(analyzer); ExprRewriter rewriter = analyzer.getExprRewriter(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index c38438cfd0adb1..0f464ba2946b7d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -74,7 +74,7 @@ public static void teardown() { public void test() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); ctx.getSessionVariable().setEnableNereidsPlanner(false); - + ctx.getSessionVariable().setEnableFoldConstantByBe(false); // create database db1 createDatabase(ctx, "create database db1;"); @@ -206,6 +206,7 @@ public void test() throws Exception { public void testCreateGlobalFunction() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); ctx.getSessionVariable().setEnableNereidsPlanner(false); + ctx.getSessionVariable().setEnableFoldConstantByBe(false); // 1. create database db2 createDatabase(ctx, "create database db2;"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java index dd16f69c60c675..3809f56b272ec0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java @@ -37,6 +37,7 @@ public class ConstantExpressTest { public static void beforeClass() throws Exception { UtFrameUtils.startFEServer(runningDir); connectContext = UtFrameUtils.createDefaultCtx(); + connectContext.getSessionVariable().setEnableFoldConstantByBe(false); } private static void testConstantExpressResult(String sql, String result) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index c39724efb4bdf6..76fe82e6599a87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -67,6 +67,7 @@ protected void runBeforeAll() throws Exception { // create database createDatabase("test"); connectContext.getSessionVariable().setEnableNereidsPlanner(false); + connectContext.getSessionVariable().setEnableFoldConstantByBe(false); Config.enable_odbc_mysql_broker_table = true; createTable("create table test.test1\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java index c2f31ab90a3fca..8165c97b8dc6f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java @@ -237,7 +237,7 @@ public void testComplexQuery() throws Exception { + "\"storage_format\" = \"V2\"\n" + ");"; dorisAssert.withTable(createTableSQL); - String query = "select /*+ SET_VAR(enable_nereids_planner=false) */ sum(l_extendedprice* (1 - l_discount)) as revenue " + String query = "select /*+ SET_VAR(enable_nereids_planner=false,enable_fold_constant_by_be=false) */ sum(l_extendedprice* (1 - l_discount)) as revenue " + "from lineitem, part " + "where ( p_partkey = l_partkey and p_brand = 'Brand#11' " + "and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') " diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/InferFiltersRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/InferFiltersRuleTest.java index 0710871d817efe..d5cb69272f3803 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/InferFiltersRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/InferFiltersRuleTest.java @@ -301,7 +301,7 @@ public void testWhereIsNotNullPredicate() throws Exception { sessionVariable.setEnableInferPredicate(true); sessionVariable.setEnableRewriteElementAtToSlot(false); Assert.assertTrue(sessionVariable.isEnableInferPredicate()); - String query = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb1 inner join tb2 inner join tb3" + String query = "select /*+ SET_VAR(enable_nereids_planner=false,enable_fold_constant_by_be=false) */ * from tb1 inner join tb2 inner join tb3" + " where tb1.k1 = tb3.k1 and tb2.k1 = tb3.k1 and tb1.k1 is not null"; String planString = dorisAssert.query(query).explainQuery(); Assert.assertTrue(planString.contains("`tb3`.`k1` IS NOT NULL")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteInPredicateRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteInPredicateRuleTest.java index 6d84a967e86810..ba13e6602a4c07 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteInPredicateRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteInPredicateRuleTest.java @@ -131,7 +131,7 @@ private void testBase(int childrenNum, PrimitiveType type, long expectedOfChild1 List list = Lists.newArrayList(); Lists.newArrayList(literals).forEach(e -> list.add("%s")); list.remove(list.size() - 1); - String queryFormat = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from %s where id in (" + Joiner.on(", ").join(list) + ");"; + String queryFormat = "select /*+ SET_VAR(enable_nereids_planner=false,enable_fold_constant_by_be=false) */ * from %s where id in (" + Joiner.on(", ").join(list) + ");"; String query = String.format(queryFormat, literals); StmtExecutor executor1 = getSqlStmtExecutor(query); Expr expr1 = ((SelectStmt) executor1.getParsedStmt()).getWhereClause(); diff --git a/regression-test/suites/correctness/test_date_function_const.groovy b/regression-test/suites/correctness/test_date_function_const.groovy index fd5607d519ece7..85d8e5ba711370 100644 --- a/regression-test/suites/correctness/test_date_function_const.groovy +++ b/regression-test/suites/correctness/test_date_function_const.groovy @@ -17,6 +17,7 @@ suite("test_date_function_const") { sql 'set enable_nereids_planner=false' + sql 'set enable_fold_constant_by_be = false;' qt_select1 """ select hours_add('2023-03-30 22:23:45.23452',8) @@ -36,6 +37,7 @@ suite("test_date_function_const") { """ sql 'set enable_nereids_planner=true' + sql 'set enable_fold_constant_by_be = true;' sql 'set enable_fallback_to_original_planner=false' diff --git a/regression-test/suites/correctness_p0/test_cast_null.groovy b/regression-test/suites/correctness_p0/test_cast_null.groovy index 368a0a1ae793a2..9ea3b1259030a8 100644 --- a/regression-test/suites/correctness_p0/test_cast_null.groovy +++ b/regression-test/suites/correctness_p0/test_cast_null.groovy @@ -61,7 +61,7 @@ suite("test_cast_null") { """ explain { - sql """SELECT * FROM test_table_t53 LEFT JOIN test_table_t0 ON (('I4') LIKE (CAST(CAST(DATE '1970-05-06' AS FLOAT) AS VARCHAR) ));""" + sql """SELECT /*+SET_VAR(enable_fold_constant_by_be=false)*/ * FROM test_table_t53 LEFT JOIN test_table_t0 ON (('I4') LIKE (CAST(CAST(DATE '1970-05-06' AS FLOAT) AS VARCHAR) ));""" contains "19700506" } diff --git a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy index c7a0a6d748dab5..7a1b9f8c3194cd 100644 --- a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy +++ b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy @@ -16,9 +16,10 @@ // under the License. suite("test_agg_state_nereids") { - sql "set global enable_agg_state=true" + sql "set enable_agg_state=true" sql "set enable_nereids_planner=true;" sql "set enable_fallback_to_original_planner=false;" + sql "set enable_fold_constant_by_be=false;" sql """ DROP TABLE IF EXISTS d_table; """ sql """ diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/shape/query21.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/shape/query21.groovy index 4fd5428ec77a64..08785b431e4f35 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/shape/query21.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/shape/query21.groovy @@ -22,6 +22,7 @@ suite("query21") { sql "use ${db}" sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query21.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query21.groovy index a65a00bd684e20..0ef9181f791a49 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query21.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query21.groovy @@ -22,6 +22,7 @@ suite("query21") { sql "use ${db}" sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set enable_runtime_filter_prune=true' diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/no_stats_shape/query21.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/no_stats_shape/query21.groovy index f970316dc616d1..7636c71c8d831d 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/no_stats_shape/query21.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/no_stats_shape/query21.groovy @@ -22,6 +22,7 @@ suite("query21") { sql "use ${db}" sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf_prune/query21.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf_prune/query21.groovy index fa7b40fa909a74..0814922087779e 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf_prune/query21.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf_prune/query21.groovy @@ -22,6 +22,7 @@ suite("query21") { sql "use ${db}" sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/shape/query21.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/shape/query21.groovy index f4ce2bc117e586..a5986974daba6b 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/shape/query21.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/shape/query21.groovy @@ -22,6 +22,7 @@ suite("query21") { sql "use ${db}" sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' From 3d0ab498338027909fe3ad706f506e1bce524559 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Fri, 15 Mar 2024 13:15:15 +0800 Subject: [PATCH 03/15] update 2 --- .../suites/correctness/test_time_diff_microseconds.groovy | 2 ++ regression-test/suites/job_p0/test_base_insert_job.groovy | 1 + 2 files changed, 3 insertions(+) diff --git a/regression-test/suites/correctness/test_time_diff_microseconds.groovy b/regression-test/suites/correctness/test_time_diff_microseconds.groovy index e754515250bc0a..335a237ddd5284 100644 --- a/regression-test/suites/correctness/test_time_diff_microseconds.groovy +++ b/regression-test/suites/correctness/test_time_diff_microseconds.groovy @@ -43,6 +43,7 @@ suite("test_time_diff_microseconds") { """ sql """set enable_nereids_planner=false""" + sql """set enable_fold_constant_by_be=false""" qt_select1 """ select timediff(t1,t2) from tbl_time order by id @@ -68,6 +69,7 @@ suite("test_time_diff_microseconds") { """ sql """set enable_nereids_planner=true """ + sql """set enable_fold_constant_by_be=true""" sql """set enable_fallback_to_original_planner=false""" qt_select5 """ diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy index b4400e89f2dff8..760520ae5f627c 100644 --- a/regression-test/suites/job_p0/test_base_insert_job.groovy +++ b/regression-test/suites/job_p0/test_base_insert_job.groovy @@ -25,6 +25,7 @@ suite("test_base_insert_job") { def jobName = "insert_recovery_test_base_insert_job" def jobMixedName = "Insert_recovery_Test_base_insert_job" sql """drop table if exists `${tableName}` force""" + sql """set enable_fold_constant_by_be = false;""" sql """ DROP JOB IF EXISTS where jobname = '${jobName}' """ From c830f6d02a21fde0292bd2257125a8573090a78d Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Fri, 15 Mar 2024 16:53:49 +0800 Subject: [PATCH 04/15] disable fold constant --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- regression-test/suites/job_p0/test_base_insert_job.groovy | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index df0c867c4d0d6d..7a03865d1dc413 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -936,7 +936,7 @@ public class SessionVariable implements Serializable, Writable { private boolean enableJoinReorderBasedCost = false; @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) - private boolean enableFoldConstantByBe = true; + private boolean enableFoldConstantByBe = false; @VariableMgr.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy index 760520ae5f627c..b4400e89f2dff8 100644 --- a/regression-test/suites/job_p0/test_base_insert_job.groovy +++ b/regression-test/suites/job_p0/test_base_insert_job.groovy @@ -25,7 +25,6 @@ suite("test_base_insert_job") { def jobName = "insert_recovery_test_base_insert_job" def jobMixedName = "Insert_recovery_Test_base_insert_job" sql """drop table if exists `${tableName}` force""" - sql """set enable_fold_constant_by_be = false;""" sql """ DROP JOB IF EXISTS where jobname = '${jobName}' """ From 4cb4b62ae5318e92fe875c7335d9092367625c0c Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Fri, 15 Mar 2024 17:00:21 +0800 Subject: [PATCH 05/15] update --- be/src/runtime/fold_constant_executor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 657d0b3c0119d0..9120fa266d7b9c 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -272,6 +272,7 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const TypeDescr result = column_type->to_string(*column_ptr, 0); break; } + case TYPE_VARIANT: case TYPE_QUANTILE_STATE: { result = column_type->to_string(*column_ptr, 0); break; From b20c64152f4c37e1f476f8a472681f3b188569eb Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Mon, 18 Mar 2024 13:01:00 +0800 Subject: [PATCH 06/15] update insert task --- .../java/org/apache/doris/job/extensions/insert/InsertTask.java | 1 + .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java index 8fe786555ce04a..7bf79521d23a51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -153,6 +153,7 @@ public void before() throws JobException { TUniqueId queryId = generateQueryId(UUID.randomUUID().toString()); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().enableNereidsDML = true; + ctx.getSessionVariable().enableFoldConstantByBe = false; stmtExecutor = new StmtExecutor(ctx, (String) null); ctx.setQueryId(queryId); if (StringUtils.isNotEmpty(sql)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7a03865d1dc413..0b3e25cae4b0b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -936,7 +936,7 @@ public class SessionVariable implements Serializable, Writable { private boolean enableJoinReorderBasedCost = false; @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) - private boolean enableFoldConstantByBe = false; + public boolean enableFoldConstantByBe = true; @VariableMgr.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; From 60339194bd043b7d44fb031d54bee5fc22db617f Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Mon, 18 Mar 2024 15:10:03 +0800 Subject: [PATCH 07/15] update --- .../apache/doris/job/extensions/insert/InsertTask.java | 9 ++++++++- .../suites/job_p0/test_base_insert_job.groovy | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java index 7bf79521d23a51..6bf9add05c30ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Column; +import org.apache.doris.common.Config; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.util.TimeUtils; @@ -153,7 +154,13 @@ public void before() throws JobException { TUniqueId queryId = generateQueryId(UUID.randomUUID().toString()); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().enableNereidsDML = true; - ctx.getSessionVariable().enableFoldConstantByBe = false; + // in test_base_insert_job.groovy, have a case of insert values('2023-07-19', + // sleep(10000), 1001); + // when enableFoldConstantByBe=true, the execute of sleep function will cause + // timeout, so set it to false when it's test case of insert task. + if (Config.enable_job_schedule_second_for_test) { + ctx.getSessionVariable().enableFoldConstantByBe = false; + } stmtExecutor = new StmtExecutor(ctx, (String) null); ctx.setQueryId(queryId); if (StringUtils.isNotEmpty(sql)) { diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy index b4400e89f2dff8..760520ae5f627c 100644 --- a/regression-test/suites/job_p0/test_base_insert_job.groovy +++ b/regression-test/suites/job_p0/test_base_insert_job.groovy @@ -25,6 +25,7 @@ suite("test_base_insert_job") { def jobName = "insert_recovery_test_base_insert_job" def jobMixedName = "Insert_recovery_Test_base_insert_job" sql """drop table if exists `${tableName}` force""" + sql """set enable_fold_constant_by_be = false;""" sql """ DROP JOB IF EXISTS where jobname = '${jobName}' """ From 4921ad6a4b00ba112b300cbbaf5bbce22b5b33ef Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Mon, 18 Mar 2024 15:17:21 +0800 Subject: [PATCH 08/15] formmat --- .../java/org/apache/doris/job/extensions/insert/InsertTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java index 6bf9add05c30ab..18ae4e7f02b47d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -19,9 +19,9 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Column; -import org.apache.doris.common.Config; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.Config; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.job.exception.JobException; import org.apache.doris.job.task.AbstractTask; From 6c10193ecc6b6fa999f02df1ff69079a6b0d768f Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Mon, 18 Mar 2024 18:11:23 +0800 Subject: [PATCH 09/15] fuzzy fold --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 0b3e25cae4b0b1..d6403be839d196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -936,7 +936,7 @@ public class SessionVariable implements Serializable, Writable { private boolean enableJoinReorderBasedCost = false; @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) - public boolean enableFoldConstantByBe = true; + public boolean enableFoldConstantByBe = false; @VariableMgr.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; @@ -1815,8 +1815,10 @@ public void initFuzzyModeVariables() { if (Config.pull_request_id > 0) { if (Config.pull_request_id % 2 == 1) { this.batchSize = 4064; + this.enableFoldConstantByBe = true; } else { this.batchSize = 50; + this.enableFoldConstantByBe = false; } } } From bbc9a67f6a3b7f0ef18b684c1af87004f9c21b2d Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Tue, 19 Mar 2024 10:47:36 +0800 Subject: [PATCH 10/15] open fold constant --- .../apache/doris/job/extensions/insert/InsertTask.java | 8 -------- .../rules/expression/rules/FoldConstantRuleOnBE.java | 8 ++++++++ .../main/java/org/apache/doris/qe/SessionVariable.java | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java index 18ae4e7f02b47d..8fe786555ce04a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.common.Config; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.job.exception.JobException; import org.apache.doris.job.task.AbstractTask; @@ -154,13 +153,6 @@ public void before() throws JobException { TUniqueId queryId = generateQueryId(UUID.randomUUID().toString()); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().enableNereidsDML = true; - // in test_base_insert_job.groovy, have a case of insert values('2023-07-19', - // sleep(10000), 1001); - // when enableFoldConstantByBe=true, the execute of sleep function will cause - // timeout, so set it to false when it's test case of insert task. - if (Config.enable_job_schedule_second_for_test) { - ctx.getSessionVariable().enableFoldConstantByBe = false; - } stmtExecutor = new StmtExecutor(ctx, (String) null); ctx.setQueryId(queryId); if (StringUtils.isNotEmpty(sql)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 27e1b34f9d0284..dc308d615eedae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -32,6 +32,8 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; @@ -132,6 +134,12 @@ private void collectConst(Expression expr, Map constMap, Map if (expr.isLiteral()) { return; } + // if sleep(5) will cause rpc timeout + if (expr instanceof Sleep) { + if (((IntegerLiteral) expr.child(0)).getValue() >= 5) { + return; + } + } String id = idGenerator.getNextId().toString(); constMap.put(id, expr); Expr staleExpr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index d6403be839d196..2e4b5051e5c3a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -936,7 +936,7 @@ public class SessionVariable implements Serializable, Writable { private boolean enableJoinReorderBasedCost = false; @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) - public boolean enableFoldConstantByBe = false; + public boolean enableFoldConstantByBe = true; @VariableMgr.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; @@ -1740,10 +1740,12 @@ public void initFuzzyModeVariables() { this.rewriteOrToInPredicateThreshold = 100000; this.enableFunctionPushdown = false; this.enableDeleteSubPredicateV2 = false; + this.enableFoldConstantByBe = false; } else { this.rewriteOrToInPredicateThreshold = 2; this.enableFunctionPushdown = true; this.enableDeleteSubPredicateV2 = true; + this.enableFoldConstantByBe = true; } /* switch (randomInt) { @@ -1815,10 +1817,8 @@ public void initFuzzyModeVariables() { if (Config.pull_request_id > 0) { if (Config.pull_request_id % 2 == 1) { this.batchSize = 4064; - this.enableFoldConstantByBe = true; } else { this.batchSize = 50; - this.enableFoldConstantByBe = false; } } } From f7860b8177d55146f6281e7515b699cae8cc3bae Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Tue, 19 Mar 2024 14:53:32 +0800 Subject: [PATCH 11/15] update open fold --- .../rules/FoldConstantRuleOnBE.java | 24 +++++++++++++------ .../org/apache/doris/qe/SessionVariable.java | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index dc308d615eedae..0f1fa23d1c11a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -129,16 +129,16 @@ private void collectConst(Expression expr, Map constMap, Map if (((Cast) expr).child().isNullLiteral()) { return; } + if (skipSleepFunction(((Cast) expr).child())) { + return; + } } // skip literal expr if (expr.isLiteral()) { return; } - // if sleep(5) will cause rpc timeout - if (expr instanceof Sleep) { - if (((IntegerLiteral) expr.child(0)).getValue() >= 5) { - return; - } + if (skipSleepFunction(expr)) { + return; } String id = idGenerator.getNextId().toString(); constMap.put(id, expr); @@ -158,6 +158,16 @@ private void collectConst(Expression expr, Map constMap, Map } } + // if sleep(5) will cause rpc timeout + private boolean skipSleepFunction(Expression expr) { + if (expr instanceof Sleep) { + if (((IntegerLiteral) expr.child(0)).getValue() >= 5) { + return true; + } + } + return false; + } + private Map evalOnBE(Map> paramMap, Map constMap, ConnectContext context) { @@ -191,8 +201,8 @@ private Map evalOnBE(Map> paramMa // TODO: will be delete the debug log after find problem of timeout. LOG.info("fold query {} ", DebugUtil.printId(context.queryId())); - Future future = - BackendServiceProxy.getInstance().foldConstantExpr(brpcAddress, tParams); + Future future = BackendServiceProxy.getInstance().foldConstantExpr(brpcAddress, + tParams); PConstantExprResult result = future.get(5, TimeUnit.SECONDS); if (result.getStatus().getStatusCode() == 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2e4b5051e5c3a0..24df5c0e864008 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1740,12 +1740,12 @@ public void initFuzzyModeVariables() { this.rewriteOrToInPredicateThreshold = 100000; this.enableFunctionPushdown = false; this.enableDeleteSubPredicateV2 = false; - this.enableFoldConstantByBe = false; + // this.enableFoldConstantByBe = false; } else { this.rewriteOrToInPredicateThreshold = 2; this.enableFunctionPushdown = true; this.enableDeleteSubPredicateV2 = true; - this.enableFoldConstantByBe = true; + // this.enableFoldConstantByBe = true; } /* switch (randomInt) { From d13e832506186a4cb045665bbe7cb17f2b03725c Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Tue, 19 Mar 2024 20:57:00 +0800 Subject: [PATCH 12/15] update sleep cast --- .../rules/expression/rules/FoldConstantRuleOnBE.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0f1fa23d1c11a4..43b9df881eb595 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -33,8 +33,8 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -161,8 +161,12 @@ private void collectConst(Expression expr, Map constMap, Map // if sleep(5) will cause rpc timeout private boolean skipSleepFunction(Expression expr) { if (expr instanceof Sleep) { - if (((IntegerLiteral) expr.child(0)).getValue() >= 5) { - return true; + Expression param = expr.child(0); + if (param instanceof Cast) { + param = param.child(0); + } + if (param instanceof NumericLiteral) { + return ((NumericLiteral) param).getDouble() >= 5.0; } } return false; From f537f58ac57d6a7b21929e1e6865236338f5a3d8 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Thu, 21 Mar 2024 21:13:17 +0800 Subject: [PATCH 13/15] update --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 -- .../datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy | 1 - regression-test/suites/job_p0/test_base_insert_job.groovy | 1 - 3 files changed, 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 24df5c0e864008..0b3e25cae4b0b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1740,12 +1740,10 @@ public void initFuzzyModeVariables() { this.rewriteOrToInPredicateThreshold = 100000; this.enableFunctionPushdown = false; this.enableDeleteSubPredicateV2 = false; - // this.enableFoldConstantByBe = false; } else { this.rewriteOrToInPredicateThreshold = 2; this.enableFunctionPushdown = true; this.enableDeleteSubPredicateV2 = true; - // this.enableFoldConstantByBe = true; } /* switch (randomInt) { diff --git a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy index 7a1b9f8c3194cd..29b3f483148981 100644 --- a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy +++ b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy @@ -19,7 +19,6 @@ suite("test_agg_state_nereids") { sql "set enable_agg_state=true" sql "set enable_nereids_planner=true;" sql "set enable_fallback_to_original_planner=false;" - sql "set enable_fold_constant_by_be=false;" sql """ DROP TABLE IF EXISTS d_table; """ sql """ diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy index 760520ae5f627c..b4400e89f2dff8 100644 --- a/regression-test/suites/job_p0/test_base_insert_job.groovy +++ b/regression-test/suites/job_p0/test_base_insert_job.groovy @@ -25,7 +25,6 @@ suite("test_base_insert_job") { def jobName = "insert_recovery_test_base_insert_job" def jobMixedName = "Insert_recovery_Test_base_insert_job" sql """drop table if exists `${tableName}` force""" - sql """set enable_fold_constant_by_be = false;""" sql """ DROP JOB IF EXISTS where jobname = '${jobName}' """ From f7f34eed10928e45d67a9fc9a451517f387ff5cd Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Thu, 21 Mar 2024 23:55:43 +0800 Subject: [PATCH 14/15] update agg_state --- .../nereids/rules/expression/rules/FoldConstantRuleOnBE.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 43b9df881eb595..38c6a483c9f777 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -137,6 +137,10 @@ private void collectConst(Expression expr, Map constMap, Map if (expr.isLiteral()) { return; } + // eg: avg_state(1) return is agg function serialize data + if (expr.getDataType().isAggStateType()) { + return; + } if (skipSleepFunction(expr)) { return; } From a2e719c1ba2acb921c315de186735b967375a660 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Fri, 22 Mar 2024 10:29:41 +0800 Subject: [PATCH 15/15] update fuzzy --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 0b3e25cae4b0b1..3749e2e25c68b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1789,7 +1789,12 @@ public void initFuzzyModeVariables() { default: break; } - + randomInt = random.nextInt(2); + if (randomInt % 2 == 0) { + this.enableFoldConstantByBe = false; + } else { + this.enableFoldConstantByBe = true; + } this.runtimeFilterType = 1 << randomInt; this.enableParallelScan = Config.pull_request_id % 2 == 0 ? randomInt % 2 == 0 : randomInt % 1 == 0; switch (randomInt) {