Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support insert values #1067

Merged
merged 3 commits into from
Apr 29, 2019
Merged

Support insert values #1067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0e5f2af
Support insert into values
Apr 28, 2019
90af2f5
Fix hll import bug
Apr 28, 2019
fc1d9ce
Fix insert subquery fail
Apr 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 80 additions & 10 deletions fe/src/main/cup/sql_parser.cup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A
KW_TABLE, KW_TABLES, KW_TABLET, KW_TERMINATED, KW_THAN, KW_THEN, KW_TIMESTAMP, KW_TINYINT,
KW_TO, KW_TRANSACTION, KW_TRIGGERS, KW_TRIM, KW_TRUE, KW_TRUNCATE, KW_TYPE, KW_TYPES,
KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNSIGNED, KW_USE, KW_USER, KW_USING,
KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VIEW,
KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VIEW,
KW_WARNINGS, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE;

terminal COMMA, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
Expand Down Expand Up @@ -257,6 +257,7 @@ nonterminal opt_with_consistent_snapshot, opt_work, opt_chain, opt_release;

// Single select statement.
nonterminal SelectStmt select_stmt;
nonterminal ValueList value_clause;

// No return.
nonterminal describe_command, opt_full, opt_inner, opt_outer, from_or_in, keys_or_index, opt_storage, opt_wild_where,
Expand Down Expand Up @@ -288,9 +289,9 @@ nonterminal RestoreStmt restore_stmt;

nonterminal SelectList select_clause, select_list, select_sublist;
nonterminal SelectListItem select_list_item, star_expr;
nonterminal Expr expr, non_pred_expr, arithmetic_expr, timestamp_arithmetic_expr;
nonterminal Expr expr, non_pred_expr, arithmetic_expr, timestamp_arithmetic_expr, expr_or_default;
nonterminal LiteralExpr set_expr_or_default;
nonterminal ArrayList<Expr> expr_list;
nonterminal ArrayList<Expr> expr_list, values, row_value, opt_values;
nonterminal ArrayList<Expr> func_arg_list;
nonterminal String select_alias, opt_table_alias;
nonterminal ArrayList<String> ident_list, opt_using_partition;
Expand Down Expand Up @@ -403,7 +404,7 @@ nonterminal List<AlterClause> alter_table_clause_list;
//
nonterminal String keyword, ident, ident_or_text, variable_name, text_or_password,
charset_name_or_default, old_or_new_charset_name_or_default, opt_collate,
collation_name_or_default;
collation_name_or_default, type_func_name_keyword, type_function_name;

nonterminal String opt_db, opt_partition_name, procedure_or_function, opt_default_value, opt_comment, opt_engine;
nonterminal Boolean opt_if_exists, opt_if_not_exists;
Expand Down Expand Up @@ -2690,8 +2691,65 @@ select_stmt ::=
groupingExprs, havingPredicate, orderByClause,
limitClause);
:}
| value_clause:valueClause order_by_clause:orderByClause limit_clause:limitClause
{:
RESULT = new SelectStmt(valueClause, orderByClause, limitClause);
:}
;

value_clause ::=
KW_VALUES row_value:value
{:
RESULT = new ValueList(value);
:}
| value_clause:valueClause COMMA row_value:value
{:
valueClause.addRow(value);
RESULT = valueClause;
:}
;

row_value ::=
LPAREN opt_values:values RPAREN
{:
RESULT = values;
:}
;

opt_values ::=
values:valueList
{:
RESULT = valueList;
:}
|
{:
RESULT = Lists.newArrayList();
:}
;

values ::=
expr_or_default:value
{:
RESULT = Lists.newArrayList(value);
:}
| values:valueList COMMA expr_or_default:value
{:
valueList.add(value);
RESULT = valueList;
:}
;

expr_or_default ::=
expr:expr
{:
RESULT = expr;
:}
| KW_DEFAULT
{:
RESULT = new DefaultValueExpr();
:}
;

select_clause ::=
KW_SELECT select_list:l
{:
Expand Down Expand Up @@ -2803,12 +2861,19 @@ table_name ::=
;

function_name ::=
ident:fn
type_function_name:fn
{: RESULT = new FunctionName(null, fn); :}
| ident:db DOT ident:fn
| ident:db DOT type_function_name:fn
{: RESULT = new FunctionName(db, fn); :}
;

type_function_name ::=
ident:id
{: RESULT = id; :}
| type_func_name_keyword:id
{: RESULT = id; :}
;

from_clause ::=
KW_FROM table_ref_list:l
{: RESULT = new FromClause(l); :}
Expand Down Expand Up @@ -3771,6 +3836,13 @@ opt_release ::=
:}
;

type_func_name_keyword ::=
KW_LEFT:id
{: RESULT = id; :}
| KW_RIGHT:id
{: RESULT = id; :}
;

// Keyword that we allow for identifiers
keyword ::=
KW_AFTER:id
Expand Down Expand Up @@ -3859,8 +3931,6 @@ keyword ::=
{: RESULT = id; :}
| KW_LAST:id
{: RESULT = id; :}
| KW_LEFT:id
{: RESULT = id; :}
| KW_LESS:id
{: RESULT = id; :}
| KW_LEVEL:id
Expand Down Expand Up @@ -3925,8 +3995,6 @@ keyword ::=
{: RESULT = id; :}
| KW_RETURNS:id
{: RESULT = id; :}
| KW_RIGHT:id
{: RESULT = id; :}
| KW_ROLLBACK:id
{: RESULT = id; :}
| KW_ROLLUP:id
Expand Down Expand Up @@ -3971,6 +4039,8 @@ keyword ::=
{: RESULT = id; :}
| KW_VARIABLES:id
{: RESULT = id; :}
| KW_VALUE:id
{: RESULT = id; :}
| KW_VIEW:id
{: RESULT = id; :}
| KW_WARNINGS:id
Expand Down
43 changes: 43 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.analysis;

import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;

//
public class DefaultValueExpr extends Expr {
@Override
protected void analyzeImpl(Analyzer analyzer) throws AnalysisException {
}

@Override
protected String toSqlImpl() {
return null;
}

@Override
protected void toThrift(TExprNode msg) {

}

@Override
public Expr clone() {
return null;
}
}
121 changes: 85 additions & 36 deletions fe/src/main/java/org/apache/doris/analysis/InsertStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ public class InsertStmt extends DdlStmt {
private DataSink dataSink;
private DataPartition dataPartition;

List<Column> targetColumns = Lists.newArrayList();

public InsertStmt(InsertTarget target, List<String> cols, InsertSource source, List<String> hints) {
this.tblName = target.getTblName();
List<String> tmpPartitions = target.getPartitions();
Expand Down Expand Up @@ -318,12 +320,8 @@ private void analyzeTargetTable(Analyzer analyzer) throws AnalysisException {
}
}

private void checkColumnCoverage(Set<String> mentionedCols, List<Column> baseColumns, List<Expr> selectList)
private void checkColumnCoverage(Set<String> mentionedCols, List<Column> baseColumns)
throws AnalysisException {
// check if size of select item equal with columns mentioned in statement
if (mentionedCols.size() != selectList.size()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT);
}

// check columns of target table
for (Column col : baseColumns) {
Expand All @@ -337,14 +335,8 @@ private void checkColumnCoverage(Set<String> mentionedCols, List<Column> baseCol
}

public void analyzeSubquery(Analyzer analyzer) throws UserException {
queryStmt.setFromInsert(true);
// parse query statement
queryStmt.analyze(analyzer);
List<Expr> selectList = Expr.cloneList(queryStmt.getBaseTblResultExprs());

// Analyze columns mentioned in the statement.
Set<String> mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
List<Column> targetColumns = Lists.newArrayList();
if (targetColumnNames == null) {
for (Column col : targetTable.getBaseSchema()) {
mentionedColumns.add(col.getName());
Expand All @@ -363,10 +355,69 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException {
}
}

// parse query statement
queryStmt.setFromInsert(true);
queryStmt.analyze(analyzer);

// check if size of select item equal with columns mentioned in statement
if (mentionedColumns.size() != queryStmt.getResultExprs().size()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The length has been checked in checkColumnConverage.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two checks are not the same. this checks if the number of query statement's result is equal with number of columns user assigned. And checkColumnConverage means to check if user assigned columns can covert the columns of target table.

ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT);
}

// Check if all columns mentioned is enough
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema(), selectList);
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ;
if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
List<ArrayList<Expr>> rows = selectStmt.getValueList().getRows();
for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) {
analyzeRow(analyzer, targetColumns, rows.get(rowIdx), rowIdx + 1);
}
for (int i = 0; i < selectStmt.getResultExprs().size(); ++i) {
selectStmt.getResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i));
selectStmt.getBaseTblResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i));
}
} else {
analyzeRow(analyzer, targetColumns, selectStmt.getResultExprs(), 1);
}
isStreaming = true;
} else {
for (int i = 0; i < targetColumns.size(); ++i) {
Column column = targetColumns.get(i);
if (column.getType().isHllType()) {
Expr expr = queryStmt.getResultExprs().get(i);
checkHllCompatibility(column, expr);
}
}
}
}

private void analyzeRow(Analyzer analyzer, List<Column> targetColumns, ArrayList<Expr> row, int rowIdx)
throws AnalysisException {
// 1. check number of fields if equal with first row
if (row.size() != targetColumns.size()) {
throw new AnalysisException("Column count doesn't match value count at row " + rowIdx);
}
for (int i = 0; i < row.size(); ++i) {
Expr expr = row.get(i);
Column col = targetColumns.get(i);

// TargeTable's hll column must be hll_hash's result
if (col.getType().equals(Type.HLL)) {
checkHllCompatibility(col, expr);
}

if (expr instanceof DefaultValueExpr) {
if (targetColumns.get(i).getDefaultValue() == null) {
throw new AnalysisException("Column has no default value, column=" + targetColumns.get(i).getName());
}
expr = new StringLiteral(targetColumns.get(i).getDefaultValue());
}

expr.analyze(analyzer);

prepareExpressions(targetColumns, selectList, analyzer);
row.set(i, checkTypeCompatibility(col, expr));
}
}

private void analyzePlanHints(Analyzer analyzer) throws AnalysisException {
Expand Down Expand Up @@ -397,40 +448,37 @@ private void analyzePlanHints(Analyzer analyzer) throws AnalysisException {
}
}
}

private Expr checkTypeCompatibility(Column col, Expr expr) throws AnalysisException {
// TargeTable's hll column must be hll_hash's result
if (col.getType().equals(Type.HLL)) {
final String hllMismatchLog = "Column's type is HLL,"
+ " SelectList must contains HLL or hll_hash function's result, column=" + col.getName();
if (expr instanceof SlotRef) {
final SlotRef slot = (SlotRef) expr;
if (!slot.getType().equals(Type.HLL)) {
throw new AnalysisException(hllMismatchLog);
}
} else if (expr instanceof FunctionCallExpr) {
final FunctionCallExpr functionExpr = (FunctionCallExpr) expr;
if (!functionExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) {
throw new AnalysisException(hllMismatchLog);
}
} else {
private void checkHllCompatibility(Column col, Expr expr) throws AnalysisException {
final String hllMismatchLog = "Column's type is HLL,"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why hll's compatibility check is s separated from normal compatibility check?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because HLL column will replace with SlotRef in Planner with a varchar type, and this will fail in this function. So we need check HLL column here.

+ " SelectList must contains HLL or hll_hash function's result, column=" + col.getName();
if (expr instanceof SlotRef) {
final SlotRef slot = (SlotRef) expr;
if (!slot.getType().equals(Type.HLL)) {
throw new AnalysisException(hllMismatchLog);
}

} else if (expr instanceof FunctionCallExpr) {
final FunctionCallExpr functionExpr = (FunctionCallExpr) expr;
if (!functionExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) {
throw new AnalysisException(hllMismatchLog);
}
} else {
throw new AnalysisException(hllMismatchLog);
}
}

if (col.getDataType().equals(expr.getType())) {
private Expr checkTypeCompatibility(Column col, Expr expr) throws AnalysisException {
if (col.getDataType().equals(expr.getType().getPrimitiveType())) {
return expr;
}
return expr.castTo(col.getType());
}

private void prepareExpressions(List<Column> targetCols, List<Expr> selectList, Analyzer analyzer)
throws UserException {
public void prepareExpressions() throws UserException {
List<Expr> selectList = Expr.cloneList(queryStmt.getBaseTblResultExprs());
// check type compatibility
int numCols = targetCols.size();
int numCols = targetColumns.size();
for (int i = 0; i < numCols; ++i) {
Column col = targetCols.get(i);
Column col = targetColumns.get(i);
Expr expr = checkTypeCompatibility(col, selectList.get(i));
selectList.set(i, expr);
exprByName.put(col.getName(), expr);
Expand Down Expand Up @@ -498,5 +546,6 @@ public void reset() {
exprByName.clear();
dataSink = null;
dataPartition = null;
targetColumns.clear();
}
}
Loading