Reconstruct the parsing for sqlTokens

sharding-core/sharding-core-optimize/src/test/java/org/apache/shardingsphere/core/optimize/engine/encrypt/EncryptInsertOptimizeEngineTest.java

@@ -84,8 +84,8 @@ public void assertInsertStatementWithValuesWithPlaceHolderWithEncrypt() {
     private InsertStatement createInsertStatementWithValuesWithPlaceHolderWithEncrypt() {
         InsertStatement result = new InsertStatement();
         result.getTables().add(new Table("t_encrypt", null));
-        result.addSQLToken(new TableToken(12, "t_encrypt", QuoteCharacter.NONE, 0));
-        result.addSQLToken(new InsertValuesToken(34));
+        result.addSQLToken(new TableToken(12, 20, "t_encrypt", QuoteCharacter.NONE));
+        result.addSQLToken(new InsertValuesToken(21, 46));
         result.getColumnNames().add("col1");
         result.getColumnNames().add("col2");
         result.getValues().add(new InsertValue(Arrays.<SQLExpression>asList(new SQLParameterMarkerExpression(0), new SQLParameterMarkerExpression(1))));
@@ -112,8 +112,8 @@ public void assertInsertStatementWithValuesWithoutPlaceHolderWithQueryEncrypt()
     private InsertStatement createInsertStatementWithValuesWithoutPlaceHolderWithQueryEncrypt() {
         InsertStatement result = new InsertStatement();
         result.getTables().add(new Table("t_query_encrypt", null));
-        result.addSQLToken(new TableToken(12, "t_query_encrypt", QuoteCharacter.NONE, 0));
-        result.addSQLToken(new InsertValuesToken(40));
+        result.addSQLToken(new TableToken(12, 27, "t_query_encrypt", QuoteCharacter.NONE));
+        result.addSQLToken(new InsertValuesToken(27, 52));
         result.getColumnNames().add("col1");
         result.getColumnNames().add("col2");
         result.getValues().add(new InsertValue(Arrays.<SQLExpression>asList(new SQLNumberExpression(1), new SQLNumberExpression(2))));
@@ -138,8 +138,8 @@ public void assertInsertStatementWithSetWithoutPlaceHolderWithEncrypt() {
     private InsertStatement createInsertStatementWithSetWithoutPlaceHolderWithEncrypt() {
         InsertStatement result = new InsertStatement();
         result.getTables().add(new Table("t_encrypt", null));
-        result.addSQLToken(new TableToken(12, "t_encrypt", QuoteCharacter.NONE, 0));
-        result.addSQLToken(new InsertSetToken(34));
+        result.addSQLToken(new TableToken(12, 20, "t_encrypt", QuoteCharacter.NONE));
+        result.addSQLToken(new InsertSetToken(12, 20));
         result.getColumnNames().add("col1");
         result.getColumnNames().add("col2");
         result.getValues().add(new InsertValue(Arrays.<SQLExpression>asList(new SQLNumberExpression(1), new SQLNumberExpression(2))));
@@ -166,8 +166,8 @@ public void assertInsertStatementWithSetWithPlaceHolderWithQueryEncrypt() {
     private InsertStatement createInsertStatementWithSetWithPlaceHolderWithQueryEncrypt() {
         InsertStatement result = new InsertStatement();
         result.getTables().add(new Table("t_query_encrypt", null));
-        result.addSQLToken(new TableToken(12, "t_query_encrypt", QuoteCharacter.NONE, 0));
-        result.addSQLToken(new InsertSetToken(40));
+        result.addSQLToken(new TableToken(12, 26, "t_query_encrypt", QuoteCharacter.NONE));
+        result.addSQLToken(new InsertSetToken(12, 27));
         result.getColumnNames().add("col1");
         result.getColumnNames().add("col2");
         result.getValues().add(new InsertValue(Arrays.<SQLExpression>asList(new SQLParameterMarkerExpression(0), new SQLParameterMarkerExpression(1))));

sharding-core/sharding-core-optimize/src/test/java/org/apache/shardingsphere/core/optimize/engine/sharding/InsertOptimizeEngineTest.java

@@ -118,8 +118,8 @@ private void initializeInsertValuesWithPlaceHolder() {
         insertValuesStatementWithPlaceHolder = new InsertStatement();
         insertValuesStatementWithPlaceHolder.getTables().add(new Table("t_order", null));
         insertValuesStatementWithPlaceHolder.setParametersIndex(4);
-        insertValuesStatementWithPlaceHolder.addSQLToken(new TableToken(12, "t_order", QuoteCharacter.NONE, 0));
-        insertValuesStatementWithPlaceHolder.addSQLToken(new InsertValuesToken(39));
+        insertValuesStatementWithPlaceHolder.addSQLToken(new TableToken(12, 18, "t_order", QuoteCharacter.NONE));
+        insertValuesStatementWithPlaceHolder.addSQLToken(new InsertValuesToken(21, 46));
         AndCondition andCondition1 = new AndCondition();
         andCondition1.getConditions().add(new Condition(new Column("user_id", "t_order"), new SQLParameterMarkerExpression(0)));
         insertValuesStatementWithPlaceHolder.getRouteConditions().getOrCondition().getAndConditions().add(andCondition1);
@@ -137,8 +137,8 @@ private void initializeInsertValuesWithPlaceHolderWithEncrypt() {
         insertValuesStatementWithPlaceHolderWithEncrypt = new InsertStatement();
         insertValuesStatementWithPlaceHolderWithEncrypt.getTables().add(new Table("t_encrypt", null));
         insertValuesStatementWithPlaceHolderWithEncrypt.setParametersIndex(4);
-        insertValuesStatementWithPlaceHolderWithEncrypt.addSQLToken(new TableToken(12, "t_encrypt", QuoteCharacter.NONE, 0));
-        insertValuesStatementWithPlaceHolderWithEncrypt.addSQLToken(new InsertValuesToken(39));
+        insertValuesStatementWithPlaceHolderWithEncrypt.addSQLToken(new TableToken(12, 20, "t_encrypt", QuoteCharacter.NONE));
+        insertValuesStatementWithPlaceHolderWithEncrypt.addSQLToken(new InsertValuesToken(21, 46));
         AndCondition andCondition1 = new AndCondition();
         andCondition1.getConditions().add(new Condition(new Column("user_id", "t_encrypt"), new SQLParameterMarkerExpression(0)));
         insertValuesStatementWithPlaceHolderWithEncrypt.getRouteConditions().getOrCondition().getAndConditions().add(andCondition1);
@@ -156,8 +156,8 @@ private void initializeInsertValuesWithoutPlaceHolder() {
         insertValuesStatementWithoutPlaceHolder = new InsertStatement();
         insertValuesStatementWithoutPlaceHolder.getTables().add(new Table("t_order", null));
         insertValuesStatementWithoutPlaceHolder.setParametersIndex(0);
-        insertValuesStatementWithoutPlaceHolder.addSQLToken(new TableToken(12, "t_order", QuoteCharacter.NONE, 0));
-        insertValuesStatementWithoutPlaceHolder.addSQLToken(new InsertValuesToken(42));
+        insertValuesStatementWithoutPlaceHolder.addSQLToken(new TableToken(12, 20, "t_order", QuoteCharacter.NONE));
+        insertValuesStatementWithoutPlaceHolder.addSQLToken(new InsertValuesToken(27, 52));
         ItemsToken itemsToken = new ItemsToken(34);
         itemsToken.getItems().add("order_id");
         insertValuesStatementWithoutPlaceHolder.addSQLToken(itemsToken);
@@ -170,8 +170,8 @@ private void initializeInsertValuesWithoutPlaceHolderWithQueryEncrypt() {
         insertValuesStatementWithoutPlaceHolderWithQueryEncrypt = new InsertStatement();
         insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.getTables().add(new Table("t_encrypt_query", null));
         insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.setParametersIndex(0);
-        insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.addSQLToken(new TableToken(12, "t_encrypt_query", QuoteCharacter.NONE, 0));
-        insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.addSQLToken(new InsertValuesToken(42));
+        insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.addSQLToken(new TableToken(12, 26, "t_encrypt_query", QuoteCharacter.NONE));
+        insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.addSQLToken(new InsertValuesToken(27, 52));
         ItemsToken itemsToken = new ItemsToken(34);
         itemsToken.getItems().add("order_id");
         insertValuesStatementWithoutPlaceHolderWithQueryEncrypt.addSQLToken(itemsToken);
@@ -184,8 +184,8 @@ private void initializeInsertSetWithPlaceHolder() {
         insertSetStatementWithPlaceHolder = new InsertStatement();
         insertSetStatementWithPlaceHolder.getTables().add(new Table("t_order", null));
         insertSetStatementWithPlaceHolder.setParametersIndex(0);
-        insertSetStatementWithPlaceHolder.addSQLToken(new TableToken(12, "t_order", QuoteCharacter.NONE, 0));
-        insertSetStatementWithPlaceHolder.addSQLToken(new InsertSetToken(24));
+        insertSetStatementWithPlaceHolder.addSQLToken(new TableToken(12, 18, "t_order", QuoteCharacter.NONE));
+        insertSetStatementWithPlaceHolder.addSQLToken(new InsertSetToken(12, 20));
         insertSetStatementWithPlaceHolder.getColumnNames().add("user_id");
         insertSetStatementWithPlaceHolder.getColumnNames().add("status");
         AndCondition andCondition = new AndCondition();
@@ -197,8 +197,8 @@ private void initializeInsertSetWithPlaceHolderWithQueryEncrypt() {
         insertSetStatementWithPlaceHolderWithQueryEncrypt = new InsertStatement();
         insertSetStatementWithPlaceHolderWithQueryEncrypt.getTables().add(new Table("t_encrypt_query", null));
         insertSetStatementWithPlaceHolderWithQueryEncrypt.setParametersIndex(0);
-        insertSetStatementWithPlaceHolderWithQueryEncrypt.addSQLToken(new TableToken(12, "t_encrypt_query", QuoteCharacter.NONE, 0));
-        insertSetStatementWithPlaceHolderWithQueryEncrypt.addSQLToken(new InsertSetToken(24));
+        insertSetStatementWithPlaceHolderWithQueryEncrypt.addSQLToken(new TableToken(12, 26, "t_encrypt_query", QuoteCharacter.NONE));
+        insertSetStatementWithPlaceHolderWithQueryEncrypt.addSQLToken(new InsertSetToken(12, 20));
         insertSetStatementWithPlaceHolderWithQueryEncrypt.getColumnNames().add("user_id");
         insertSetStatementWithPlaceHolderWithQueryEncrypt.getColumnNames().add("status");
         AndCondition andCondition = new AndCondition();
@@ -210,8 +210,8 @@ private void initializeInsertSetWithoutPlaceHolder() {
         insertSetStatementWithoutPlaceHolder = new InsertStatement();
         insertSetStatementWithoutPlaceHolder.getTables().add(new Table("t_order", null));
         insertSetStatementWithoutPlaceHolder.setParametersIndex(0);
-        insertSetStatementWithoutPlaceHolder.addSQLToken(new TableToken(12, "t_order", QuoteCharacter.NONE, 0));
-        insertSetStatementWithoutPlaceHolder.addSQLToken(new InsertSetToken(24));
+        insertSetStatementWithoutPlaceHolder.addSQLToken(new TableToken(12, 18, "t_order", QuoteCharacter.NONE));
+        insertSetStatementWithoutPlaceHolder.addSQLToken(new InsertSetToken(12, 20));
         insertSetStatementWithoutPlaceHolder.getColumnNames().add("user_id");
         insertSetStatementWithoutPlaceHolder.getColumnNames().add("status");
         AndCondition andCondition = new AndCondition();
@@ -223,8 +223,8 @@ private void initializeInsertSetWithoutPlaceHolderWithEncrypt() {
         insertSetStatementWithoutPlaceHolderWithEncrypt = new InsertStatement();
         insertSetStatementWithoutPlaceHolderWithEncrypt.getTables().add(new Table("t_encrypt", null));
         insertSetStatementWithoutPlaceHolderWithEncrypt.setParametersIndex(0);
-        insertSetStatementWithoutPlaceHolderWithEncrypt.addSQLToken(new TableToken(12, "t_encrypt", QuoteCharacter.NONE, 0));
-        insertSetStatementWithoutPlaceHolderWithEncrypt.addSQLToken(new InsertSetToken(24));
+        insertSetStatementWithoutPlaceHolderWithEncrypt.addSQLToken(new TableToken(12, 20, "t_encrypt", QuoteCharacter.NONE));
+        insertSetStatementWithoutPlaceHolderWithEncrypt.addSQLToken(new InsertSetToken(12, 20));
         insertSetStatementWithoutPlaceHolderWithEncrypt.getColumnNames().add("user_id");
         insertSetStatementWithoutPlaceHolderWithEncrypt.getColumnNames().add("status");
         AndCondition andCondition = new AndCondition();

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/extractor/impl/common/table/TableExtractor.java

@@ -54,7 +54,7 @@ public Optional<TableSegment> extract(final ParserRuleContext ancestorNode, fina
             tableName = nodeText;
             owner = Optional.absent();
         }
-        TableSegment result = new TableSegment(tableNameNode.get().getStart().getStartIndex(), tableName);
+        TableSegment result = new TableSegment(tableNameNode.get().getStart().getStartIndex(), tableNameNode.get().getStart().getStopIndex(), tableName);
         if (owner.isPresent()) {
             result.setOwner(owner.get());
         }

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/extractor/impl/dml/select/LimitExtractor.java

@@ -35,6 +35,7 @@
  * Limit extractor.
  *
  * @author duhongjun
+ * @author panjuan
  */
 public final class LimitExtractor implements OptionalSQLSegmentExtractor {
 
@@ -53,8 +54,12 @@ public Optional<LimitSegment> extract(final ParserRuleContext ancestorNode, fina
     }
 
     private LimitValueSegment createLimitValueSegment(final Map<ParserRuleContext, Integer> placeholderAndNodeIndexMap, final ParserRuleContext limitValueNode) {
-        return Symbol.QUESTION.getLiterals().equals(limitValueNode.getText()) 
-                ? new PlaceholderLimitValueSegment(placeholderAndNodeIndexMap.get(limitValueNode.getChild(0)), ((ParserRuleContext) limitValueNode.getChild(0)).getStart().getStartIndex())
-                : new LiteralLimitValueSegment(NumberUtil.getExactlyNumber(limitValueNode.getText(), 10).intValue(), limitValueNode.getStart().getStartIndex());
+        if (Symbol.QUESTION.getLiterals().equals(limitValueNode.getText())) {
+            ParserRuleContext placeholderLimitValueNode = (ParserRuleContext) limitValueNode.getChild(0);
+            return new PlaceholderLimitValueSegment(placeholderAndNodeIndexMap.get(placeholderLimitValueNode), 
+                    placeholderLimitValueNode.getStart().getStartIndex(), placeholderLimitValueNode.getStart().getStopIndex());
+
+        }
+        return new LiteralLimitValueSegment(NumberUtil.getExactlyNumber(limitValueNode.getText(), 10).intValue(), limitValueNode.getStart().getStartIndex(), limitValueNode.getStart().getStopIndex());
     }
 }

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/common/dml/InsertColumnsFiller.java

@@ -46,7 +46,7 @@ public void fill(final InsertColumnsSegment sqlSegment, final SQLStatement sqlSt
             } else {
                 fillFromSQL(sqlSegment, insertStatement);
             }
-            insertStatement.getSQLTokens().add(new InsertValuesToken(sqlSegment.getStartIndex()));
+            insertStatement.getSQLTokens().add(new InsertValuesToken(sqlSegment.getStartIndex(), sqlStatement.getLogicSQL().length() - 1));
         }
     }
 

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/common/dml/LimitFiller.java

@@ -50,7 +50,7 @@ private void setOffset(final LimitValueSegment offsetSegment, final SelectStatem
         if (offsetSegment instanceof LiteralLimitValueSegment) {
             int value = ((LiteralLimitValueSegment) offsetSegment).getValue();
             selectStatement.getLimit().setOffset(new LimitValue(value, -1, false));
-            selectStatement.getSQLTokens().add(new OffsetToken(offsetSegment.getStartIndex(), value));
+            selectStatement.getSQLTokens().add(new OffsetToken(offsetSegment.getStartIndex(), offsetSegment.getStopIndex(), value));
         } else {
             selectStatement.getLimit().setOffset(new LimitValue(-1, ((PlaceholderLimitValueSegment) offsetSegment).getParameterIndex(), false));
         }
@@ -60,7 +60,7 @@ private void setRowCount(final LimitValueSegment rowCountSegment, final SelectSt
         if (rowCountSegment instanceof LiteralLimitValueSegment) {
             int value = ((LiteralLimitValueSegment) rowCountSegment).getValue();
             selectStatement.getLimit().setRowCount(new LimitValue(value, -1, false));
-            selectStatement.getSQLTokens().add(new RowCountToken(rowCountSegment.getStartIndex(), value));
+            selectStatement.getSQLTokens().add(new RowCountToken(rowCountSegment.getStartIndex(), rowCountSegment.getStopIndex(), value));
         } else {
             selectStatement.getLimit().setRowCount(new LimitValue(-1, ((PlaceholderLimitValueSegment) rowCountSegment).getParameterIndex(), false));
         }

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/common/dml/SelectItemFiller.java

@@ -20,7 +20,6 @@
 import com.google.common.base.Optional;
 import lombok.RequiredArgsConstructor;
 import org.apache.shardingsphere.core.metadata.table.ShardingTableMetaData;
-import org.apache.shardingsphere.core.parse.antlr.constant.QuoteCharacter;
 import org.apache.shardingsphere.core.parse.antlr.filler.api.SQLSegmentFiller;
 import org.apache.shardingsphere.core.parse.antlr.sql.segment.SQLSegment;
 import org.apache.shardingsphere.core.parse.antlr.sql.segment.dml.expr.complex.SubquerySegment;
@@ -86,7 +85,7 @@ private void fillShorthandSelectItemSegment(final ShorthandSelectItemSegment sel
             Optional<Table> table = selectStatement.getTables().find(owner.get());
             if (table.isPresent() && !table.get().getAlias().isPresent() && shardingTableMetaData.containsTable(table.get().getName())) {
                 // FIXME for QuoteCharacter.getQuoteCharacter(owner), if order by `xxx`.xx, has problem
-                selectStatement.addSQLToken(new TableToken(selectItemSegment.getStartIndex(), owner.get(), QuoteCharacter.getQuoteCharacter(owner.get()), 0));
+                selectStatement.addSQLToken(new TableToken(selectItemSegment.getStartIndex(), selectItemSegment.getStopIndexOfOwner(), owner.get(), selectItemSegment.getOwnerQuoteCharacter()));
             }
         }
     }

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/encrypt/dml/EncryptSetAssignmentsFiller.java

@@ -63,7 +63,7 @@ private void fillInsert(final SetAssignmentsSegment sqlSegment, final InsertStat
         InsertValue insertValue = getInsertValue(sqlSegment, insertStatement.getLogicSQL());
         insertStatement.getValues().add(insertValue);
         insertStatement.setParametersIndex(insertValue.getParametersCount());
-        insertStatement.getSQLTokens().add(new InsertSetToken(sqlSegment.getStartIndex()));
+        insertStatement.getSQLTokens().add(new InsertSetToken(sqlSegment.getStartIndex(), insertStatement.getLogicSQL().length() - 1));
     }
 
     private InsertValue getInsertValue(final SetAssignmentsSegment sqlSegment, final String sql) {

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/sharding/ShardingColumnFiller.java

@@ -32,6 +32,7 @@
  *
  * @author duhongjun
  * @author zhangliang
+ * @author panjuan
  */
 @Setter
 public final class ShardingColumnFiller implements SQLSegmentFiller<ColumnSegment>, ShardingTableMetaDataAwareFiller {
@@ -45,7 +46,7 @@ public void fill(final ColumnSegment sqlSegment, final SQLStatement sqlStatement
         }
         Optional<Table> logicTable = sqlStatement.getTables().find(sqlSegment.getOwner().get());
         if (logicTable.isPresent() && !logicTable.get().getAlias().isPresent() && shardingTableMetaData.containsTable(logicTable.get().getName())) {
-            sqlStatement.addSQLToken(new TableToken(sqlSegment.getStartIndex(), logicTable.get().getName(), sqlSegment.getOwnerQuoteCharacter(), 0));
+            sqlStatement.addSQLToken(new TableToken(sqlSegment.getStartIndex(), sqlSegment.getStopIndexOfOwner(), logicTable.get().getName(), sqlSegment.getOwnerQuoteCharacter()));
         }
     }
 }

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/sharding/ShardingTableFiller.java

@@ -34,6 +34,7 @@
  *
  * @author duhongjun
  * @author zhangliang
+ * @author panjuan
  */
 @Setter
 public final class ShardingTableFiller implements SQLSegmentFiller<TableSegment>, ShardingRuleAwareFiller {
@@ -44,7 +45,7 @@ public final class ShardingTableFiller implements SQLSegmentFiller<TableSegment>
     public void fill(final TableSegment sqlSegment, final SQLStatement sqlStatement) {
         if (isTableInShardingRule(sqlSegment.getName()) || !(sqlStatement instanceof SelectStatement)) {
             sqlStatement.getTables().add(new Table(sqlSegment.getName(), sqlSegment.getAlias().orNull()));
-            sqlStatement.getSQLTokens().add(new TableToken(sqlSegment.getStartIndex(), sqlSegment.getName(), sqlSegment.getQuoteCharacter(), sqlSegment.getOwnerLength()));
+            sqlStatement.getSQLTokens().add(new TableToken(sqlSegment.getStartIndex(), sqlSegment.getStopIndex(), sqlSegment.getName(), sqlSegment.getQuoteCharacter()));
         }
         if (sqlStatement instanceof DMLStatement && !sqlStatement.getTables().isSingleTable()) {
             throw new SQLParsingUnsupportedException("Cannot support Multiple-Table.");

sharding-core/sharding-core-parse/sharding-core-parse-common/src/main/java/org/apache/shardingsphere/core/parse/antlr/filler/sharding/dml/update/ShardingSetAssignmentsFiller.java

@@ -84,7 +84,7 @@ private void fillInsert(final SetAssignmentsSegment sqlSegment, final InsertStat
         insertStatement.getValues().add(insertValue);
         insertStatement.getRouteConditions().getOrCondition().getAndConditions().add(andCondition);
         insertStatement.setParametersIndex(insertValue.getParametersCount());
-        insertStatement.getSQLTokens().add(new InsertSetToken(sqlSegment.getStartIndex()));
+        insertStatement.getSQLTokens().add(new InsertSetToken(sqlSegment.getStartIndex(), insertStatement.getLogicSQL().length() - 1));
     }
 
     private int getColumnCountExcludeAssistedQueryColumns(final InsertStatement insertStatement) {