diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index a453520145a24f..fe32d97727786f 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -225,7 +225,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A KW_TABLE, KW_TABLES, KW_TABLET, KW_TERMINATED, KW_THAN, KW_THEN, KW_TIMESTAMP, KW_TINYINT, KW_TO, KW_TRANSACTION, KW_TRIGGERS, KW_TRIM, KW_TRUE, KW_TRUNCATE, KW_TYPE, KW_TYPES, KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, - KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VIEW, + KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VIEW, KW_WARNINGS, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE; terminal COMMA, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT; @@ -257,6 +257,7 @@ nonterminal opt_with_consistent_snapshot, opt_work, opt_chain, opt_release; // Single select statement. nonterminal SelectStmt select_stmt; +nonterminal ValueList value_clause; // No return. nonterminal describe_command, opt_full, opt_inner, opt_outer, from_or_in, keys_or_index, opt_storage, opt_wild_where, @@ -288,9 +289,9 @@ nonterminal RestoreStmt restore_stmt; nonterminal SelectList select_clause, select_list, select_sublist; nonterminal SelectListItem select_list_item, star_expr; -nonterminal Expr expr, non_pred_expr, arithmetic_expr, timestamp_arithmetic_expr; +nonterminal Expr expr, non_pred_expr, arithmetic_expr, timestamp_arithmetic_expr, expr_or_default; nonterminal LiteralExpr set_expr_or_default; -nonterminal ArrayList expr_list; +nonterminal ArrayList expr_list, values, row_value, opt_values; nonterminal ArrayList func_arg_list; nonterminal String select_alias, opt_table_alias; nonterminal ArrayList ident_list, opt_using_partition; @@ -403,7 +404,7 @@ nonterminal List alter_table_clause_list; // nonterminal String keyword, ident, ident_or_text, variable_name, text_or_password, charset_name_or_default, old_or_new_charset_name_or_default, opt_collate, - collation_name_or_default; + collation_name_or_default, type_func_name_keyword, type_function_name; nonterminal String opt_db, opt_partition_name, procedure_or_function, opt_default_value, opt_comment, opt_engine; nonterminal Boolean opt_if_exists, opt_if_not_exists; @@ -2690,8 +2691,65 @@ select_stmt ::= groupingExprs, havingPredicate, orderByClause, limitClause); :} + | value_clause:valueClause order_by_clause:orderByClause limit_clause:limitClause + {: + RESULT = new SelectStmt(valueClause, orderByClause, limitClause); + :} ; +value_clause ::= + KW_VALUES row_value:value + {: + RESULT = new ValueList(value); + :} + | value_clause:valueClause COMMA row_value:value + {: + valueClause.addRow(value); + RESULT = valueClause; + :} + ; + +row_value ::= + LPAREN opt_values:values RPAREN + {: + RESULT = values; + :} + ; + +opt_values ::= + values:valueList + {: + RESULT = valueList; + :} + | + {: + RESULT = Lists.newArrayList(); + :} + ; + +values ::= + expr_or_default:value + {: + RESULT = Lists.newArrayList(value); + :} + | values:valueList COMMA expr_or_default:value + {: + valueList.add(value); + RESULT = valueList; + :} + ; + +expr_or_default ::= + expr:expr + {: + RESULT = expr; + :} + | KW_DEFAULT + {: + RESULT = new DefaultValueExpr(); + :} + ; + select_clause ::= KW_SELECT select_list:l {: @@ -2803,12 +2861,19 @@ table_name ::= ; function_name ::= - ident:fn + type_function_name:fn {: RESULT = new FunctionName(null, fn); :} - | ident:db DOT ident:fn + | ident:db DOT type_function_name:fn {: RESULT = new FunctionName(db, fn); :} ; +type_function_name ::= + ident:id + {: RESULT = id; :} + | type_func_name_keyword:id + {: RESULT = id; :} + ; + from_clause ::= KW_FROM table_ref_list:l {: RESULT = new FromClause(l); :} @@ -3771,6 +3836,13 @@ opt_release ::= :} ; +type_func_name_keyword ::= + KW_LEFT:id + {: RESULT = id; :} + | KW_RIGHT:id + {: RESULT = id; :} + ; + // Keyword that we allow for identifiers keyword ::= KW_AFTER:id @@ -3859,8 +3931,6 @@ keyword ::= {: RESULT = id; :} | KW_LAST:id {: RESULT = id; :} - | KW_LEFT:id - {: RESULT = id; :} | KW_LESS:id {: RESULT = id; :} | KW_LEVEL:id @@ -3925,8 +3995,6 @@ keyword ::= {: RESULT = id; :} | KW_RETURNS:id {: RESULT = id; :} - | KW_RIGHT:id - {: RESULT = id; :} | KW_ROLLBACK:id {: RESULT = id; :} | KW_ROLLUP:id @@ -3971,6 +4039,8 @@ keyword ::= {: RESULT = id; :} | KW_VARIABLES:id {: RESULT = id; :} + | KW_VALUE:id + {: RESULT = id; :} | KW_VIEW:id {: RESULT = id; :} | KW_WARNINGS:id diff --git a/fe/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java b/fe/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java new file mode 100644 index 00000000000000..4552e6f84fcf59 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TExprNode; + +// +public class DefaultValueExpr extends Expr { + @Override + protected void analyzeImpl(Analyzer analyzer) throws AnalysisException { + } + + @Override + protected String toSqlImpl() { + return null; + } + + @Override + protected void toThrift(TExprNode msg) { + + } + + @Override + public Expr clone() { + return null; + } +} diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index 5c27198b4609a0..851f80a8028481 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -92,6 +92,8 @@ public class InsertStmt extends DdlStmt { private DataSink dataSink; private DataPartition dataPartition; + List targetColumns = Lists.newArrayList(); + public InsertStmt(InsertTarget target, List cols, InsertSource source, List hints) { this.tblName = target.getTblName(); List tmpPartitions = target.getPartitions(); @@ -318,12 +320,8 @@ private void analyzeTargetTable(Analyzer analyzer) throws AnalysisException { } } - private void checkColumnCoverage(Set mentionedCols, List baseColumns, List selectList) + private void checkColumnCoverage(Set mentionedCols, List baseColumns) throws AnalysisException { - // check if size of select item equal with columns mentioned in statement - if (mentionedCols.size() != selectList.size()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT); - } // check columns of target table for (Column col : baseColumns) { @@ -337,14 +335,8 @@ private void checkColumnCoverage(Set mentionedCols, List baseCol } public void analyzeSubquery(Analyzer analyzer) throws UserException { - queryStmt.setFromInsert(true); - // parse query statement - queryStmt.analyze(analyzer); - List selectList = Expr.cloneList(queryStmt.getBaseTblResultExprs()); - // Analyze columns mentioned in the statement. Set mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - List targetColumns = Lists.newArrayList(); if (targetColumnNames == null) { for (Column col : targetTable.getBaseSchema()) { mentionedColumns.add(col.getName()); @@ -363,10 +355,69 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { } } + // parse query statement + queryStmt.setFromInsert(true); + queryStmt.analyze(analyzer); + + // check if size of select item equal with columns mentioned in statement + if (mentionedColumns.size() != queryStmt.getResultExprs().size()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT); + } + // Check if all columns mentioned is enough - checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema(), selectList); + checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ; + if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) { + SelectStmt selectStmt = (SelectStmt) queryStmt; + if (selectStmt.getValueList() != null) { + List> rows = selectStmt.getValueList().getRows(); + for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) { + analyzeRow(analyzer, targetColumns, rows.get(rowIdx), rowIdx + 1); + } + for (int i = 0; i < selectStmt.getResultExprs().size(); ++i) { + selectStmt.getResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i)); + selectStmt.getBaseTblResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i)); + } + } else { + analyzeRow(analyzer, targetColumns, selectStmt.getResultExprs(), 1); + } + isStreaming = true; + } else { + for (int i = 0; i < targetColumns.size(); ++i) { + Column column = targetColumns.get(i); + if (column.getType().isHllType()) { + Expr expr = queryStmt.getResultExprs().get(i); + checkHllCompatibility(column, expr); + } + } + } + } + + private void analyzeRow(Analyzer analyzer, List targetColumns, ArrayList row, int rowIdx) + throws AnalysisException { + // 1. check number of fields if equal with first row + if (row.size() != targetColumns.size()) { + throw new AnalysisException("Column count doesn't match value count at row " + rowIdx); + } + for (int i = 0; i < row.size(); ++i) { + Expr expr = row.get(i); + Column col = targetColumns.get(i); + + // TargeTable's hll column must be hll_hash's result + if (col.getType().equals(Type.HLL)) { + checkHllCompatibility(col, expr); + } + + if (expr instanceof DefaultValueExpr) { + if (targetColumns.get(i).getDefaultValue() == null) { + throw new AnalysisException("Column has no default value, column=" + targetColumns.get(i).getName()); + } + expr = new StringLiteral(targetColumns.get(i).getDefaultValue()); + } + + expr.analyze(analyzer); - prepareExpressions(targetColumns, selectList, analyzer); + row.set(i, checkTypeCompatibility(col, expr)); + } } private void analyzePlanHints(Analyzer analyzer) throws AnalysisException { @@ -397,40 +448,37 @@ private void analyzePlanHints(Analyzer analyzer) throws AnalysisException { } } } - - private Expr checkTypeCompatibility(Column col, Expr expr) throws AnalysisException { - // TargeTable's hll column must be hll_hash's result - if (col.getType().equals(Type.HLL)) { - final String hllMismatchLog = "Column's type is HLL," - + " SelectList must contains HLL or hll_hash function's result, column=" + col.getName(); - if (expr instanceof SlotRef) { - final SlotRef slot = (SlotRef) expr; - if (!slot.getType().equals(Type.HLL)) { - throw new AnalysisException(hllMismatchLog); - } - } else if (expr instanceof FunctionCallExpr) { - final FunctionCallExpr functionExpr = (FunctionCallExpr) expr; - if (!functionExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { - throw new AnalysisException(hllMismatchLog); - } - } else { + private void checkHllCompatibility(Column col, Expr expr) throws AnalysisException { + final String hllMismatchLog = "Column's type is HLL," + + " SelectList must contains HLL or hll_hash function's result, column=" + col.getName(); + if (expr instanceof SlotRef) { + final SlotRef slot = (SlotRef) expr; + if (!slot.getType().equals(Type.HLL)) { throw new AnalysisException(hllMismatchLog); } - + } else if (expr instanceof FunctionCallExpr) { + final FunctionCallExpr functionExpr = (FunctionCallExpr) expr; + if (!functionExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { + throw new AnalysisException(hllMismatchLog); + } + } else { + throw new AnalysisException(hllMismatchLog); } + } - if (col.getDataType().equals(expr.getType())) { + private Expr checkTypeCompatibility(Column col, Expr expr) throws AnalysisException { + if (col.getDataType().equals(expr.getType().getPrimitiveType())) { return expr; } return expr.castTo(col.getType()); } - private void prepareExpressions(List targetCols, List selectList, Analyzer analyzer) - throws UserException { + public void prepareExpressions() throws UserException { + List selectList = Expr.cloneList(queryStmt.getBaseTblResultExprs()); // check type compatibility - int numCols = targetCols.size(); + int numCols = targetColumns.size(); for (int i = 0; i < numCols; ++i) { - Column col = targetCols.get(i); + Column col = targetColumns.get(i); Expr expr = checkTypeCompatibility(col, selectList.get(i)); selectList.set(i, expr); exprByName.put(col.getName(), expr); @@ -498,5 +546,6 @@ public void reset() { exprByName.clear(); dataSink = null; dataPartition = null; + targetColumns.clear(); } } diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java index 4a1869e4d6b4c3..5ea6e7054c8d7a 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -86,13 +86,23 @@ public class SelectStmt extends QueryStmt { // directly private ExprSubstitutionMap baseTblSmap = new ExprSubstitutionMap(); + private ValueList valueList; + // END: Members that need to be reset() // /////////////////////////////////////// // SQL string of this SelectStmt before inline-view expression substitution. // Set in analyze(). protected String sqlString_; - + + public SelectStmt(ValueList valueList, ArrayList orderByElement, LimitElement limitElement) { + super(orderByElement, limitElement); + this.valueList = valueList; + this.selectList = new SelectList(); + this.fromClause_ = new FromClause(); + this.colLabels = Lists.newArrayList(); + } + SelectStmt( SelectList selectList, FromClause fromClause, @@ -160,6 +170,10 @@ public SelectList getSelectList() { return selectList; } + public ValueList getValueList() { + return valueList; + } + /** * @return the HAVING clause post-analysis and with aliases resolved */ @@ -306,6 +320,20 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { colLabels.add(item.toColumnLabel()); } } + if (valueList != null) { + if (!fromInsert) { + valueList.analyzeForSelect(analyzer); + } + for (Expr expr : valueList.getFirstRow()) { + if (expr instanceof DefaultValueExpr) { + resultExprs.add(new IntLiteral(1)); + } else { + resultExprs.add(expr); + } + colLabels.add(expr.toColumnLabel()); + } + } + // analyze valueList if exists if (needToSql) { originalExpr = Expr.cloneList(resultExprs); } diff --git a/fe/src/main/java/org/apache/doris/analysis/ValueList.java b/fe/src/main/java/org/apache/doris/analysis/ValueList.java new file mode 100644 index 00000000000000..bb440059faf501 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/ValueList.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import com.google.common.collect.Lists; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import java.util.ArrayList; +import java.util.List; + +public class ValueList { + private List> rows; + + public ValueList(ArrayList row) { + rows = Lists.newArrayList(); + rows.add(row); + } + public ValueList(List> rows) { + this.rows = rows; + } + + public List> getRows() { return rows; } + public void addRow(ArrayList row) { rows.add(row); } + public ArrayList getFirstRow() { return rows.get(0); } + + public void analyzeForSelect(Analyzer analyzer) throws AnalysisException { + if (rows.isEmpty()) { + throw new AnalysisException("No row in value list"); + } + ArrayList firstRow = null; + int rowIdx = 0; + for (ArrayList row : rows) { + rowIdx++; + // 1. check number of fields if equal with first row + if (firstRow != null && row.size() != firstRow.size()) { + throw new AnalysisException("Column count doesn't match value count at row " + rowIdx); + } + for (int i = 0; i < row.size(); ++i) { + Expr expr = row.get(i); + if (expr instanceof DefaultValueExpr) { + throw new AnalysisException("Default expression can't exist in SELECT statement at row " + rowIdx); + } + expr.analyze(analyzer); + if (firstRow != null) { + Type dstType = firstRow.get(i).getType(); + if (!expr.getType().getPrimitiveType().equals(dstType.getPrimitiveType())) { + row.set(i, expr.castTo(dstType)); + } + } + } + if (firstRow == null) { + firstRow = row; + } + } + } +} diff --git a/fe/src/main/java/org/apache/doris/planner/Planner.java b/fe/src/main/java/org/apache/doris/planner/Planner.java index a0cb5fb82f50db..e1e901147ea109 100644 --- a/fe/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/src/main/java/org/apache/doris/planner/Planner.java @@ -148,6 +148,7 @@ public void createPlanFragments(StatementBase statment, Analyzer analyzer, TQuer List resultExprs = queryStmt.getResultExprs(); if (statment instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) statment; + insertStmt.prepareExpressions(); if (insertStmt.getOlapTuple() != null && !insertStmt.isStreaming()) { singleNodePlan = new OlapRewriteNode(plannerContext.getNextNodeId(), singleNodePlan, insertStmt); singleNodePlan.init(analyzer); @@ -183,6 +184,7 @@ public void createPlanFragments(StatementBase statment, Analyzer analyzer, TQuer if (statment instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) statment; + rootFragment = distributedPlanner.createInsertFragment(rootFragment, insertStmt, fragments); rootFragment.setSink(insertStmt.createDataSink()); insertStmt.finalize(); diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index b83a599e7c7fcf..6f0c8c5f56aa67 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -686,21 +686,29 @@ private PlanNode createAggregationPlan(SelectStmt selectStmt, Analyzer analyzer, * Returns a MergeNode that materializes the exprs of the constant selectStmt. Replaces the resultExprs of the * selectStmt with SlotRefs into the materialized tuple. */ - private PlanNode createConstantSelectPlan(SelectStmt selectStmt, Analyzer analyzer) - throws UserException { + private PlanNode createConstantSelectPlan(SelectStmt selectStmt, Analyzer analyzer) { Preconditions.checkState(selectStmt.getTableRefs().isEmpty()); ArrayList resultExprs = selectStmt.getResultExprs(); // Create tuple descriptor for materialized tuple. TupleDescriptor tupleDesc = createResultTupleDescriptor(selectStmt, "union", analyzer); UnionNode unionNode = new UnionNode(ctx_.getNextNodeId(), tupleDesc.getId()); + // Analysis guarantees that selects without a FROM clause only have constant exprs. - unionNode.addConstExprList(Lists.newArrayList(resultExprs)); + if (selectStmt.getValueList() != null) { + for (ArrayList row : selectStmt.getValueList().getRows()) { + unionNode.addConstExprList(row); + } + } else { + unionNode.addConstExprList(Lists.newArrayList(resultExprs)); + } // Replace the select stmt's resultExprs with SlotRefs into tupleDesc. for (int i = 0; i < resultExprs.size(); ++i) { SlotRef slotRef = new SlotRef(tupleDesc.getSlots().get(i)); resultExprs.set(i, slotRef); + selectStmt.getBaseTblResultExprs().set(i, slotRef); } + // UnionNode.init() needs tupleDesc to have been initialized unionNode.init(analyzer); return unionNode; diff --git a/fe/src/main/jflex/sql_scanner.flex b/fe/src/main/jflex/sql_scanner.flex index 1ad0c5326139ef..f82e39c21f51af 100644 --- a/fe/src/main/jflex/sql_scanner.flex +++ b/fe/src/main/jflex/sql_scanner.flex @@ -306,6 +306,7 @@ import org.apache.doris.common.util.SqlUtils; keywordMap.put("use", new Integer(SqlParserSymbols.KW_USE)); keywordMap.put("user", new Integer(SqlParserSymbols.KW_USER)); keywordMap.put("using", new Integer(SqlParserSymbols.KW_USING)); + keywordMap.put("value", new Integer(SqlParserSymbols.KW_VALUE)); keywordMap.put("values", new Integer(SqlParserSymbols.KW_VALUES)); keywordMap.put("varchar", new Integer(SqlParserSymbols.KW_VARCHAR)); keywordMap.put("variables", new Integer(SqlParserSymbols.KW_VARIABLES));