[Fix](Planner) fix varchar does not show real length

LiBinfeng-01 · Nov 6, 2023 · 78558f9 · 78558f9
1 parent 7a0ee04
commit 78558f9
Show file tree

Hide file tree

Showing 18 changed files with 34 additions and 57 deletions.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -598,7 +598,7 @@ public String toString() {
             return "TIMEV2(" + scale + ")";
         } else if (type == PrimitiveType.VARCHAR) {
             if (isWildcardVarchar()) {
-                return "VARCHAR(*)";
+                return "VARCHAR(" + MAX_VARCHAR_LENGTH + ")";
             }
             return "VARCHAR(" + len + ")";
         } else if (type == PrimitiveType.STRING) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -367,7 +367,6 @@
 import org.apache.doris.nereids.types.MapType;
 import org.apache.doris.nereids.types.StructField;
 import org.apache.doris.nereids.types.StructType;
-import org.apache.doris.nereids.types.coercion.CharacterType;
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.RelationUtil;
 import org.apache.doris.policy.FilterType;
@@ -1603,8 +1602,7 @@ public Expression visitSearchedCase(DorisParser.SearchedCaseContext context) {
     public Expression visitCast(DorisParser.CastContext ctx) {
         return ParserUtils.withOrigin(ctx, () -> {
             DataType dataType = ((DataType) typedVisit(ctx.dataType())).conversion();
-            Expression cast = new Cast(getExpression(ctx.expression()), dataType, true);
-            return processCast(cast, dataType);
+            return new Cast(getExpression(ctx.expression()), dataType, true);
         });
     }
 
@@ -1648,24 +1646,10 @@ public Expression visitConvertCharSet(DorisParser.ConvertCharSetContext ctx) {
     public Expression visitConvertType(DorisParser.ConvertTypeContext ctx) {
         return ParserUtils.withOrigin(ctx, () -> {
             DataType dataType = ((DataType) typedVisit(ctx.type)).conversion();
-            Expression cast = new Cast(getExpression(ctx.argument), dataType, true);
-            return processCast(cast, dataType);
+            return new Cast(getExpression(ctx.argument), dataType, true);
         });
     }
 
-    private Expression processCast(Expression cast, DataType dataType) {
-        if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
-            List<Expression> args = ImmutableList.of(
-                    cast,
-                    new TinyIntLiteral((byte) 1),
-                    Literal.of(((CharacterType) dataType).getLen())
-            );
-            return new UnboundFunction("substr", args);
-        } else {
-            return cast;
-        }
-    }
-
     @Override
     public Expression visitFunctionCall(DorisParser.FunctionCallContext ctx) {
         return ParserUtils.withOrigin(ctx, () -> {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java
@@ -36,12 +36,10 @@
 import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.coercion.CharacterType;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -174,17 +172,7 @@ protected LogicalPlan visitTable(io.trino.sql.tree.Table node, ParserContext con
     protected Expression visitCast(io.trino.sql.tree.Cast node, ParserContext context) {
         Expression expr = visit(node.getExpression(), context, Expression.class);
         DataType dataType = mappingType(node.getType());
-        Expression cast = new Cast(expr, dataType);
-        if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
-            List<Expression> args = ImmutableList.of(
-                    cast,
-                    new TinyIntLiteral((byte) 1),
-                    Literal.of(((CharacterType) dataType).getLen())
-            );
-            return new UnboundFunction("substr", args);
-        } else {
-            return cast;
-        }
+        return new Cast(expr, dataType);
     }
 
     /* ********************************************************************************************

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java
@@ -67,7 +67,7 @@ public DataType defaultConcreteType() {
     @Override
     public String toSql() {
         if (len == -1) {
-            return "VARCHAR(*)";
+            return "VARCHAR(" + MAX_VARCHAR_LENGTH + ")";
         }
         return "VARCHAR(" + len + ")";
     }
@@ -85,4 +85,8 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), len);
     }
+
+    public boolean isWildcardVarchar() {
+        return len == -1 || len == MAX_VARCHAR_LENGTH;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
@@ -114,7 +114,7 @@ public void test() throws Exception {
 
         queryStr = "select db1.id_masking(k1) from db1.tbl1";
         Assert.assertTrue(
-                dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 4))"));
+                dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
 
         // create alias function with cast
         // cast any type to decimal with specific precision and scale
@@ -149,7 +149,7 @@ public void test() throws Exception {
 
         // cast any type to varchar with fixed length
         createFuncStr = "create alias function db1.varchar(all) with parameter(text) as "
-                + "cast(text as varchar(*));";
+                + "cast(text as varchar(65533));";
         createFunctionStmt = (CreateFunctionStmt) UtFrameUtils.parseAndAnalyzeStmt(createFuncStr, ctx);
         Env.getCurrentEnv().createFunction(createFunctionStmt);
 
@@ -172,7 +172,7 @@ public void test() throws Exception {
         Assert.assertTrue(constExprLists.get(0).get(0) instanceof StringLiteral);
 
         queryStr = "select db1.varchar(k1, 4) from db1.tbl1;";
-        Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(*))"));
+        Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(65533))"));
 
         // cast any type to char with fixed length
         createFuncStr = "create alias function db1.to_char(all, int) with parameter(text, length) as "
@@ -235,7 +235,7 @@ public void testCreateGlobalFunction() throws Exception {
 
         queryStr = "select id_masking(k1) from db2.tbl1";
         Assert.assertTrue(
-                dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 4))"));
+                dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
 
         // 4. create alias function with cast
         // cast any type to decimal with specific precision and scale
@@ -270,7 +270,7 @@ public void testCreateGlobalFunction() throws Exception {
         testFunctionQuery(ctx, queryStr, true);
 
         queryStr = "select varchar(k1, 4) from db2.tbl1;";
-        Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(*))"));
+        Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(65533))"));
 
         // 6. cast any type to char with fixed length
         createFuncStr = "create global alias function db2.to_char(all, int) with parameter(text, length) as "

diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java
@@ -773,7 +773,7 @@ public void testNDVToHll() throws Exception {
         String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
                 + "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
         dorisAssert.withMaterializedView(createUserTagMVSql);
-        String query = "select ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";";
+        String query = "select /*+ SET_VAR(enable_fallback_to_original_planner=false) */ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";";
         dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg");
     }
 
@@ -782,7 +782,7 @@ public void testApproxCountDistinctToHll() throws Exception {
         String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
                 + "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
         dorisAssert.withMaterializedView(createUserTagMVSql);
-        String query = "select approx_count_distinct(tag_id) from " + USER_TAG_TABLE_NAME + ";";
+        String query = "select /*+ SET_VAR(enable_fallback_to_original_planner=false) */ approx_count_distinct(tag_id) from " + USER_TAG_TABLE_NAME + ";";
         dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg");
     }
 

diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -564,11 +564,11 @@ public void testTypeCast() throws Exception {
         // disable cast hll/bitmap to string
         assertSQLPlanOrErrorMsgContains(
                 "select cast(id2 as varchar) from test.hll_table;",
-                "Invalid type cast of `id2` from HLL to VARCHAR(*)"
+                "Invalid type cast of `id2` from HLL to VARCHAR(65533)"
         );
         assertSQLPlanOrErrorMsgContains(
                 "select cast(id2 as varchar) from test.bitmap_table;",
-                "Invalid type cast of `id2` from BITMAP to VARCHAR(*)"
+                "Invalid type cast of `id2` from BITMAP to VARCHAR(65533)"
         );
         // disable implicit cast hll/bitmap to string
         assertSQLPlanOrErrorMsgContains(

diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java
@@ -273,27 +273,27 @@ public void testRewriteLikePredicate() throws Exception {
         // tinyint
         String sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k1 like '%4%';";
         LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
-        dorisAssert.query(sql).explainContains("CAST(`k1` AS VARCHAR(*)) LIKE '%4%'");
+        dorisAssert.query(sql).explainContains("CAST(`k1` AS VARCHAR(65533)) LIKE '%4%'");
 
         // smallint
         sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k2 like '%4%';";
         LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
-        dorisAssert.query(sql).explainContains("CAST(`k2` AS VARCHAR(*)) LIKE '%4%'");
+        dorisAssert.query(sql).explainContains("CAST(`k2` AS VARCHAR(65533)) LIKE '%4%'");
 
         // int
         sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k3 like '%4%';";
         LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
-        dorisAssert.query(sql).explainContains("CAST(`k3` AS VARCHAR(*)) LIKE '%4%'");
+        dorisAssert.query(sql).explainContains("CAST(`k3` AS VARCHAR(65533)) LIKE '%4%'");
 
         // bigint
         sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k4 like '%4%';";
         LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
-        dorisAssert.query(sql).explainContains("CAST(`k4` AS VARCHAR(*)) LIKE '%4%'");
+        dorisAssert.query(sql).explainContains("CAST(`k4` AS VARCHAR(65533)) LIKE '%4%'");
 
         // largeint
         sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k5 like '%4%';";
         LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
-        dorisAssert.query(sql).explainContains("CAST(`k5` AS VARCHAR(*)) LIKE '%4%'");
+        dorisAssert.query(sql).explainContains("CAST(`k5` AS VARCHAR(65533)) LIKE '%4%'");
     }
 
     @Test

diff --git a/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out b/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out
@@ -10,7 +10,7 @@ bigint_key	BIGINT	No	true	\N	BLOOM_FILTER
 char_50_key	CHAR(50)	No	true	\N	BLOOM_FILTER
 character_key	VARCHAR(500)	No	true	\N	BLOOM_FILTER
 char_key	CHAR(1)	No	true	\N	BLOOM_FILTER
-character_most_key	VARCHAR(*)	No	true	\N	BLOOM_FILTER
+character_most_key	VARCHAR(65533)	No	true	\N	BLOOM_FILTER
 decimal_key	DECIMAL(20, 6)	No	true	\N	BLOOM_FILTER
 decimal_most_key	DECIMAL(27, 9)	No	true	\N	BLOOM_FILTER
 decimal32_key	DECIMAL(5, 1)	No	true	\N	BLOOM_FILTER
@@ -29,7 +29,7 @@ bigint_value	BIGINT	No	false	\N	SUM
 char_50_value	CHAR(50)	No	false	\N	REPLACE
 character_value	VARCHAR(500)	No	false	\N	REPLACE
 char_value	CHAR(1)	No	false	\N	REPLACE
-character_most_value	VARCHAR(*)	No	false	\N	REPLACE
+character_most_value	VARCHAR(65533)	No	false	\N	REPLACE
 decimal_value	DECIMAL(20, 6)	No	false	\N	SUM
 decimal_most_value	DECIMAL(27, 9)	No	false	\N	SUM
 decimal32_value	DECIMAL(5, 1)	No	false	\N	SUM

diff --git a/regression-test/data/ddl_p0/test_createtable_strlen.out b/regression-test/data/ddl_p0/test_createtable_strlen.out
@@ -2,6 +2,6 @@
 -- !create --
 k1	CHAR(1)	Yes	true	\N	
 K2	CHAR(10)	Yes	false	\N	NONE
-K3	VARCHAR(*)	Yes	false	\N	NONE
+K3	VARCHAR(65533)	Yes	false	\N	NONE
 K4	VARCHAR(10)	Yes	false	\N	NONE
 
diff --git a/regression-test/data/mv_p0/varchar_length/varchar_length.out b/regression-test/data/mv_p0/varchar_length/varchar_length.out
@@ -4,5 +4,5 @@ test1	UNIQUE_KEYS	vid	VARCHAR(1)	VARCHAR(1)	No	true	\N		true
 		report_time	INT	INT	No	true	\N		true		
 
 mv_test	UNIQUE_KEYS	mv_report_time	INT	INT	No	true	\N		true	`report_time`	
-		mv_vid	VARCHAR(*)	VARCHAR(*)	No	true	\N	REPLACE	true	`vid`	
+		mv_vid	VARCHAR(65533)	VARCHAR(65533)	No	true	\N	REPLACE	true	`vid`	
 
diff --git a/regression-test/data/nereids_p0/create_table/test_create_blocked.out b/regression-test/data/nereids_p0/create_table/test_create_blocked.out
@@ -2,6 +2,6 @@
 -- !create --
 k1	CHAR(1)	Yes	true	\N	
 K2	CHAR(10)	Yes	false	\N	NONE
-K3	VARCHAR(*)	Yes	false	\N	NONE
+K3	VARCHAR(65533)	Yes	false	\N	NONE
 K4	VARCHAR(10)	Yes	false	\N	NONE
 
diff --git a/regression-test/data/nereids_syntax_p0/rollup/hll/hll.out b/regression-test/data/nereids_syntax_p0/rollup/hll/hll.out
@@ -10,5 +10,5 @@ test_materialized_view_hll1	DUP_KEYS	record_id	INT	INT	Yes	true	\N		true
 		sale_amt	BIGINT	BIGINT	Yes	false	\N	NONE	true		
 
 amt_count	AGG_KEYS	mv_store_id	INT	INT	Yes	true	\N		true	`store_id`	
-		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	
+		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	
 
diff --git a/regression-test/data/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.out b/regression-test/data/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.out
@@ -7,7 +7,7 @@ test_materialized_view_hll_with_light_sc1	DUP_KEYS	record_id	INT	INT	Yes	true	\N
 		sale_amt	BIGINT	BIGINT	Yes	false	\N	NONE	true		
 
 amt_count1	AGG_KEYS	mv_store_id	INT	INT	Yes	true	\N		true	`store_id`	
-		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	
+		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	
 
 -- !sql --
 1	1

diff --git a/regression-test/data/rollup/test_materialized_view_hll.out b/regression-test/data/rollup/test_materialized_view_hll.out
@@ -7,7 +7,7 @@ test_materialized_view_hll	DUP_KEYS	record_id	INT	INT	Yes	true	\N		true
 		sale_amt	BIGINT	BIGINT	Yes	false	\N	NONE	true		
 
 amt_count	AGG_KEYS	mv_store_id	INT	INT	Yes	true	\N		true	`store_id`	
-		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	
+		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	
 
 -- !sql --
 1	1

diff --git a/regression-test/data/rollup/test_materialized_view_hll_with_light_sc.out b/regression-test/data/rollup/test_materialized_view_hll_with_light_sc.out
@@ -7,7 +7,7 @@ test_materialized_view_hll_with_light_sc	DUP_KEYS	record_id	INT	INT	Yes	true	\N
 		sale_amt	BIGINT	BIGINT	Yes	false	\N	NONE	true		
 
 amt_count1	AGG_KEYS	mv_store_id	INT	INT	Yes	true	\N		true	`store_id`	
-		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(*)))	
+		mva_HLL_UNION__hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	HLL	HLL	No	false	\N	HLL_UNION	true	hll_hash(CAST(`sale_amt` AS VARCHAR(65533)))	
 
 -- !sql --
 1	1

diff --git a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy
@@ -19,6 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite ("testNDVToHll") {
     sql """set enable_nereids_planner=true;"""
+    sql """set enable_fallback_to_original_planner=false;"""
     sql """ DROP TABLE IF EXISTS user_tags; """
 
     sql """ create table user_tags (

diff --git a/regression-test/suites/view_p0/view_p0.groovy b/regression-test/suites/view_p0/view_p0.groovy
@@ -37,7 +37,7 @@ suite("view_p0") {
     """
 
     qt_sql "select * from test_varchar_view;"
-    qt_sql "select cast( id as varchar(*)) from test_view_table;"
+    qt_sql "select cast( id as varchar(65533)) from test_view_table;"
 
     // array view
     sql """DROP TABLE IF EXISTS test_array_tbl_1"""