From ef3276f99761f8b06705c09d059c6393b16e31fb Mon Sep 17 00:00:00 2001 From: "junjie.cheng" Date: Wed, 29 Dec 2021 18:30:51 +0800 Subject: [PATCH] add the mqtt source and sink, fix the udf issue and the window size problem --- docs/SUMMARY.md | 2 + .../alibaba/rsqldb/clients/sql/SQLStream.java | 9 +- .../rsqldb/clients/sql/SQLStreamClient.java | 99 ---------- .../rsqldb/clients/strategy/SQLStrategy.java | 5 +- .../clients/sql/PrefingerprintTest.java | 1 - .../rsqldb/clients/sql/TableStreamTest.java | 78 -------- .../parser/builder/HomologousSQLBuilder.java | 5 +- .../rsqldb/parser/builder/SQLBuilder.java | 140 -------------- .../rsqldb/parser/builder/SQLTreeBuilder.java | 27 ++- .../alibaba/rsqldb/parser/entity/SqlTask.java | 13 +- .../parser/parser/SQLNodeParserFactory.java | 28 ++- .../alibaba/rsqldb/parser/parser/SQLTree.java | 11 +- .../parser/builder/AbstractSQLBuilder.java | 2 +- .../parser/parser/builder/BlinkUDFScan.java | 60 ++++-- .../parser/builder/CreateSQLBuilder.java | 5 +- .../parser/builder/FunctionSQLBuilder.java | 15 +- .../parser/parser/builder/ISQLBuilder.java | 4 +- .../parser/builder/InsertSQLBuilder.java | 13 +- .../builder/JoinConditionSQLBuilder.java | 12 +- .../parser/parser/builder/JoinSQLBuilder.java | 54 +++--- .../parser/builder/LateralTableBuilder.java | 10 +- .../parser/builder/SelectSQLBuilder.java | 128 +++++++------ .../parser/builder/SnapshotBuilder.java | 180 ++++++++---------- .../parser/builder/TableNodeBuilder.java | 4 +- .../parser/builder/UnionSQLBuilder.java | 18 +- .../parser/parser/builder/WindowBuilder.java | 104 +++++----- .../parser/parser/expression/BlinkRule.java | 3 +- .../parser/expression/BlinkRuleV2Parser.java | 12 +- .../parser/expression/CompareParser.java | 8 +- .../parser/parser/expression/InParser.java | 6 +- .../parser/expression/IsTrueFunction.java | 7 +- .../parser/expression/LikeFunction.java | 4 +- .../parser/expression/RegexpParser.java | 4 +- .../parser/function/CastFunctionParser.java | 6 +- .../parser/parser/function/CountParser.java | 4 +- .../parser/function/FilterFunction.java | 9 +- .../parser/function/GroupFunctionParser.java | 4 +- .../parser/parser/function/HopParser.java | 4 +- .../parser/function/IFFunctionParser.java | 4 +- .../parser/parser/function/ItemParser.java | 7 +- .../parser/function/JsonConcatParser.java | 4 +- .../parser/parser/function/LowerParser.java | 4 +- .../parser/function/OverWindowParser.java | 12 +- .../parser/function/ScriptFunctionParser.java | 12 +- .../parser/parser/function/SessionParser.java | 8 +- .../parser/function/SqlAsOperatorParser.java | 4 +- .../function/SqlDataTypeSpecParser.java | 4 +- .../parser/function/SqlIndentifierParser.java | 2 +- .../parser/function/SqlLiteralParser.java | 11 +- .../parser/function/TransJsonArrayParser.java | 4 +- .../parser/parser/function/TumbleParser.java | 23 +-- .../parser/result/ScriptParseResult.java | 6 +- .../parser/result/SqlNodeParseResult.java | 10 +- .../sqlnode/AbstractSelectNodeParser.java | 2 +- .../parser/sqlnode/AbstractSqlNodeParser.java | 2 +- .../parser/parser/sqlnode/CreateParser.java | 4 +- .../parser/parser/sqlnode/InsertParser.java | 5 +- .../parser/parser/sqlnode/JoinParser.java | 6 +- .../parser/parser/sqlnode/SelectParser.java | 14 +- .../parser/parser/sqlnode/UnionParser.java | 14 +- .../parser/parser/sqlnode/ViewParser.java | 4 +- .../rsqldb/parser/util/ColumnUtil.java | 17 +- .../rsqldb/parser/util/SqlDataTypeUtil.java | 6 +- rsqldb-parser/src/main/resources/log4j.xml | 51 ----- .../rsqldb/parser/builder/JoinWindowTest.java | 173 ----------------- .../rsqldb/parser/builder/SQLBuilderTest.java | 49 ----- .../apache/rsqldb/runner/RecoverAction.java | 8 +- .../org/apache/rsqldb/runner/SqlAction.java | 2 - .../org/apache/rsqldb/runner/StartAction.java | 13 +- .../src/main/resources/dipper.properties | 9 +- .../com/alibaba/rsqldb/udf/BlinkDataType.java | 2 +- .../alibaba/rsqldb/udf/FunctionUDFScript.java | 3 +- .../rsqldb/udf/udaf/BlinkUDAFScript.java | 57 ++++++ .../rsqldb/udf/udf/BlinkUDFScript.java | 7 +- .../rsqldb/udf/udtf/BlinkUDTFScript.java | 56 ++++++ .../udf/udtf/collector/BlinkCollector.java | 9 +- .../udf/udtf/collector/BlinkRowCollector.java | 23 ++- .../udtf/collector/BlinkTupleCollector.java | 7 +- rsqldb-udf/src/test/resources/log4j.xml | 20 -- 79 files changed, 657 insertions(+), 1119 deletions(-) delete mode 100644 rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStreamClient.java delete mode 100644 rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/TableStreamTest.java delete mode 100644 rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLBuilder.java delete mode 100644 rsqldb-parser/src/main/resources/log4j.xml delete mode 100644 rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/JoinWindowTest.java delete mode 100644 rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/SQLBuilderTest.java delete mode 100755 rsqldb-udf/src/test/resources/log4j.xml diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6a3acc0..6e501bc 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -7,6 +7,7 @@ * [创建metaq源表](stream_source/metaq/README.md) * [创建RocketMQ源表](stream_source/rocketmq/README.md) * [创建文件源表](stream_source/file/README.md) + * [创建Mqtt源表](stream_source/mqtt/README.md) * [创建自定义源表](stream_source/custom/README.md) * [创建结果表](stream_sink/README.md) * [创建metaq结果表](stream_sink/metaq/README.md) @@ -15,6 +16,7 @@ * [创建打印结果表](stream_sink/print/README.md) * [创建DB结果表](stream_sink/db/README.md) * [创建ES结果表](stream_sink/es/README.md) + * [创建Mqtt结果表](stream_sink/mqtt/README.md) * [创建自定义结果表](stream_sink/custom/README.md) * [创建维表](stream_dim/README.md) * [创建mysql维表](stream_dim/mysql/README.md) diff --git a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStream.java b/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStream.java index cb292e9..2cb6f29 100644 --- a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStream.java +++ b/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStream.java @@ -1,16 +1,15 @@ package com.alibaba.rsqldb.clients.sql; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - +import com.alibaba.rsqldb.parser.entity.SqlTask; import org.apache.rocketmq.streams.client.strategy.Strategy; import org.apache.rocketmq.streams.common.component.ComponentCreator; import org.apache.rocketmq.streams.common.configurable.IConfigurableService; import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; import org.apache.rocketmq.streams.configurable.ConfigurableComponent; -import com.alibaba.rsqldb.parser.entity.SqlTask; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; /** * can execute sql directly can submit sql to server support sql assemble diff --git a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStreamClient.java b/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStreamClient.java deleted file mode 100644 index f7aefe1..0000000 --- a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/sql/SQLStreamClient.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.alibaba.rsqldb.clients.sql; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import org.apache.rocketmq.streams.client.strategy.Strategy; -import org.apache.rocketmq.streams.common.component.ComponentCreator; -import org.apache.rocketmq.streams.common.configurable.IConfigurableService; -import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; -import org.apache.rocketmq.streams.configurable.ConfigurableComponent; -import com.alibaba.rsqldb.parser.builder.SQLBuilder; - -/** - * can execute sql directly - * can submit sql to server - * support sql assemble - */ -public class SQLStreamClient { - - // protected SQLStreamClient parent; - protected String namespace; - protected String pipelineName; - protected String sql; - protected volatile boolean isStop=false; - protected List sameSourceSQL=new ArrayList<>(); - - public SQLStreamClient(String namespace, String pipelineName, String sql) { - this.namespace = namespace; - this.pipelineName = pipelineName; - this.sql=sql; - } - - - public void start(){ - startSQL(false); - } - - public void asynStart(){ - startSQL(true); - } - - protected void startSQL(boolean isAsyn){ - if(sql==null){ - return; - } - SQLBuilder sqlBuilder = build(ConfigurableComponent.getInstance(namespace)); - sqlBuilder.startSQL(); - if(isAsyn){ - while (!isStop){ - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - } - - public SQLStreamClient with(Strategy... strategies) { - Properties properties = new Properties(); - for (Strategy strategy : strategies) { - properties.putAll(strategy.getStrategyProperties()); - } - ComponentCreator.createProperties(properties); - return this; - } - - - public void addSameSourceSQL(String sql){ - sameSourceSQL.add(sql); - } - - public void submit(String serverUrl,String userName,String password, boolean isSubmitParent){ - String[] propertys=new String[4]; - propertys[0]= ConfigureFileKey.CONNECT_TYPE + ":"+ IConfigurableService.DEFAULT_SERVICE_NAME; - propertys[1]=ConfigureFileKey.JDBC_URL+":"+serverUrl; - propertys[2]=ConfigureFileKey.JDBC_USERNAME+":"+userName; - propertys[3]=ConfigureFileKey.JDBC_PASSWORD+":"+password; -// if(isSubmitParent&&parent!=null){ -// parent.submit(serverUrl,userName,password,false); -// } - ConfigurableComponent configurableComponent= ComponentCreator.getComponent(namespace,ConfigurableComponent.class,propertys); - build(configurableComponent); - } - - protected SQLBuilder build(ConfigurableComponent configurableComponent){ - SQLBuilder sqlBuilder = new SQLBuilder(namespace, pipelineName, sql); - sqlBuilder.build(configurableComponent); - if(sameSourceSQL!=null&&sameSourceSQL.size()>0){ - for(String sql:sameSourceSQL){ - SQLBuilder builder = new SQLBuilder(namespace, pipelineName, sql); - builder.build(configurableComponent); - } - } - return sqlBuilder; - } - -} diff --git a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/strategy/SQLStrategy.java b/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/strategy/SQLStrategy.java index 2e98fce..c0756c7 100644 --- a/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/strategy/SQLStrategy.java +++ b/rsqldb-clients/src/main/java/com/alibaba/rsqldb/clients/strategy/SQLStrategy.java @@ -16,13 +16,14 @@ */ package com.alibaba.rsqldb.clients.strategy; -import java.util.Properties; +import com.alibaba.rsqldb.parser.parser.builder.BlinkUDFScan; import org.apache.rocketmq.streams.client.strategy.Strategy; import org.apache.rocketmq.streams.common.classloader.IsolationClassLoader; import org.apache.rocketmq.streams.common.component.AbstractComponent; import org.apache.rocketmq.streams.common.configurable.IConfigurableService; import org.apache.rocketmq.streams.script.ScriptComponent; -import com.alibaba.rsqldb.parser.parser.builder.BlinkUDFScan; + +import java.util.Properties; public class SQLStrategy implements Strategy { diff --git a/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/PrefingerprintTest.java b/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/PrefingerprintTest.java index c0665ec..2ebb59b 100644 --- a/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/PrefingerprintTest.java +++ b/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/PrefingerprintTest.java @@ -17,7 +17,6 @@ package com.alibaba.rsqldb.clients.sql; import com.alibaba.rsqldb.parser.entity.SqlTask; - import org.apache.rocketmq.streams.common.optimization.fingerprint.FingerprintCache; import org.apache.rocketmq.streams.common.topology.ChainPipeline; import org.apache.rocketmq.streams.common.utils.FileUtil; diff --git a/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/TableStreamTest.java b/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/TableStreamTest.java deleted file mode 100644 index ca2bbbe..0000000 --- a/rsqldb-clients/src/test/java/com/alibaba/rsqldb/clients/sql/TableStreamTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.rsqldb.clients.sql; - -import org.junit.Test; - -public class TableStreamTest { - protected String sql="CREATE FUNCTION now as 'com.sql.Function';\n" - + "CREATE TABLE graph_vertex_proc (\n" - + " `time` varchar,\n" - + " `uuid` varchar,\n" - + " aliuid varchar,\n" - + " pid varchar,\n" - + " file_path varchar,\n" - + " cmdline varchar,\n" - + " tty varchar,\n" - + " cwd varchar,\n" - + " perm varchar\n" - + ") WITH (\n" - + " type='metaq',\n" - + " topic='blink_dXXXXXXX',\n" - + " pullIntervalMs='100',\n" - + " consumerGroup='CID_BLINK_SOURCE_001',\n" - + " fieldDelimiter='#'\n" - + ");\n" - + "CREATE TABLE graph_proc_label_extend (\n" - + " `time` varchar,\n" - + " `uuid` varchar,\n" - + " aliuid varchar,\n" - + " pid varchar,\n" - + " file_path varchar,\n" - + " cmdline varchar,\n" - + " tty varchar,\n" - + " cwd varchar,\n" - + " perm varchar\n" - + ") WITH (type = 'print');\n" - + "INSERT\n" - + " INTO graph_proc_label_extend\n" - + "SELECT\n" - + " `time`,\n" - + " `uuid`,\n" - + " aliuid,\n" - + " pid,\n" - + " file_path,\n" - + " cmdline,\n" - + " tty,\n" - + " cwd,\n" - + " perm\n" - + "FROM\n" - + " graph_vertex_proc;"; - - - /** - * start in jar - */ - @Test - public void testSQL(){ - SQLStreamClient sqlStreamClient= new SQLStreamClient("test_namespace", "test_pipeline",sql); - sqlStreamClient.start(); - } - - - -} diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/HomologousSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/HomologousSQLBuilder.java index 0f4fece..1c437f1 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/HomologousSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/HomologousSQLBuilder.java @@ -16,12 +16,11 @@ */ package com.alibaba.rsqldb.parser.builder; -import java.util.Map; - import com.alibaba.rsqldb.parser.entity.SqlTask; - import org.apache.rocketmq.streams.configurable.ConfigurableComponent; +import java.util.Map; + /** * 同源,相同数据源的sql会动态装配在一起 可以通过getInsertSql 插入数据库完成任务发布，会自动和正在运行的数据源完成装配 */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLBuilder.java deleted file mode 100644 index d335205..0000000 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLBuilder.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.rsqldb.parser.builder; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.rocketmq.streams.common.component.ComponentCreator; -import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable; -import org.apache.rocketmq.streams.common.configurable.IConfigurable; -import org.apache.rocketmq.streams.common.topology.ChainPipeline; -import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; -import org.apache.rocketmq.streams.common.utils.StringUtil; -import org.apache.rocketmq.streams.configurable.ConfigurableComponent; -import com.alibaba.rsqldb.parser.parser.builder.BlinkUDFScan; - -public class SQLBuilder { - protected String namespace; - protected String pipelineName; - protected String sql; - protected volatile boolean isStop = false;//设置这个值，停止主线程运行 - protected AtomicBoolean hasBuilder = new AtomicBoolean(false);//可重入保障 - protected List pipelineBuilders;// - protected List chainPipelines;//编译后的拓扑结构，可以直接运行。一个sql一个，如果有join场景，会有多个 - - public SQLBuilder(String namespace, String pipelineName, String sql) { - this.namespace = namespace; - this.pipelineName = pipelineName; - this.sql = sql; - } - - /** - * 扫描某个目录下jar包的包名 - * - * @param jarDir 如果为null，在类路径扫描 - * @param packageNames - */ - public void registerUDFPackage(String jarDir, String... packageNames) { - if (packageNames != null) { - for (String packageName : packageNames) { - BlinkUDFScan.getInstance().scan(jarDir, packageName); - } - } - } - - /** - * 编译成pipeline - * - * @return - */ - public void build(ConfigurableComponent component) { - if (hasBuilder.compareAndSet(false, true)) { - SQLTreeBuilder sqlTreeBuilder = createSQLTreeBuilder(component); - List chainPipelines = sqlTreeBuilder.build(); - pipelineBuilders = sqlTreeBuilder.getPipelineBuilders(); - this.chainPipelines = chainPipelines; - - } - - } - /** - * 编译成pipeline - * - * @return - */ - public void build() { - build(null); - - } - - - protected SQLTreeBuilder createSQLTreeBuilder( - ConfigurableComponent component) { - SQLTreeBuilder sqlTreeBuilder= new SQLTreeBuilder(namespace, pipelineName, sql); - if(component!=null){ - sqlTreeBuilder.setConfigurableComponent(component); - } - return sqlTreeBuilder; - } - - /** - * 把这个sql生成的所有元数据保存在configurables，并把元数据生成sqlList，保存在sql中 - */ - public List getInsertSql() { - build(); - List allConfigurables = new ArrayList<>(); - for (PipelineBuilder pipelineBuilder : pipelineBuilders) { - allConfigurables.addAll(pipelineBuilder.getAllConfigurables()); - } - - List sqlList = new ArrayList<>(); - String tableName = ComponentCreator.getProperties().getProperty("dipper.rds.table.name"); - for (IConfigurable configurable : allConfigurables) { - AbstractConfigurable abstractConfigurable = (AbstractConfigurable)configurable; - String sql = null; - if (StringUtil.isNotEmpty(tableName)) { - sql = AbstractConfigurable.createSQL(abstractConfigurable, tableName); - } else { - sql = abstractConfigurable.createSQL(); - } - sqlList.add(sql); - } - return sqlList; - } - - /** - * 开始跑任务 - */ - public void startSQL() { - build(); - for (ChainPipeline pipeline : chainPipelines) { - pipeline.startChannel(); - } - } - - public List getPipelineBuilders() { - build(); - return pipelineBuilders; - } - - public List getChainPipelines() { - build(); - return chainPipelines; - } -} diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLTreeBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLTreeBuilder.java index 70a7bb8..aaa35ec 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLTreeBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/builder/SQLTreeBuilder.java @@ -16,6 +16,19 @@ */ package com.alibaba.rsqldb.parser.builder; +import com.alibaba.rsqldb.parser.parser.ISqlParser; +import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; +import com.alibaba.rsqldb.parser.parser.SQLParserContext; +import com.alibaba.rsqldb.parser.parser.SQLTree; +import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.CreateSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.FunctionSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.ISQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.InsertSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.NotSupportSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.SQLCreateTables; +import com.alibaba.rsqldb.parser.parser.builder.ViewSQLBuilder; +import com.alibaba.rsqldb.parser.parser.sqlnode.IBuilderCreator; import org.apache.calcite.sql.SqlNode; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,14 +43,12 @@ import org.apache.rocketmq.streams.common.utils.StringUtil; import org.apache.rocketmq.streams.configurable.ConfigurableComponent; -import com.alibaba.rsqldb.parser.parser.ISqlParser; -import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; -import com.alibaba.rsqldb.parser.parser.SQLParserContext; -import com.alibaba.rsqldb.parser.parser.SQLTree; -import com.alibaba.rsqldb.parser.parser.builder.*; -import com.alibaba.rsqldb.parser.parser.sqlnode.IBuilderCreator; - -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class SQLTreeBuilder { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/entity/SqlTask.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/entity/SqlTask.java index 5c54985..37e1f83 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/entity/SqlTask.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/entity/SqlTask.java @@ -16,12 +16,8 @@ */ package com.alibaba.rsqldb.parser.entity; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicBoolean; - +import com.alibaba.rsqldb.parser.builder.SQLTreeBuilder; +import com.alibaba.rsqldb.parser.parser.builder.BlinkUDFScan; import org.apache.rocketmq.streams.common.component.ComponentCreator; import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable; import org.apache.rocketmq.streams.common.configurable.IConfigurable; @@ -31,8 +27,9 @@ import org.apache.rocketmq.streams.common.utils.StringUtil; import org.apache.rocketmq.streams.configurable.ConfigurableComponent; -import com.alibaba.rsqldb.parser.builder.SQLTreeBuilder; -import com.alibaba.rsqldb.parser.parser.builder.BlinkUDFScan; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; public class SqlTask { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLNodeParserFactory.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLNodeParserFactory.java index 30c3e49..ef6ab46 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLNodeParserFactory.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLNodeParserFactory.java @@ -16,30 +16,24 @@ */ package com.alibaba.rsqldb.parser.parser; -import com.alibaba.rsqldb.parser.parser.builder.FunctionSQLBuilder; -import com.alibaba.rsqldb.parser.parser.expression.BlinkRuleV2Parser; - -import java.lang.reflect.Modifier; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.collections.map.HashedMap; -import org.apache.rocketmq.streams.common.model.NameCreator; -import org.apache.rocketmq.streams.common.calssscaner.AbstractScan; -import org.apache.rocketmq.streams.common.utils.ReflectUtil; - import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.FunctionSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.IBuilderCreator; - import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import org.apache.commons.collections.map.HashedMap; +import org.apache.rocketmq.streams.common.calssscaner.AbstractScan; +import org.apache.rocketmq.streams.common.model.NameCreator; +import org.apache.rocketmq.streams.common.utils.ReflectUtil; + +import java.lang.reflect.Modifier; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * 可以通过扩展来增加sql解析能力，主体框架不变化 diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLTree.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLTree.java index 26b2664..f12d74f 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLTree.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/SQLTree.java @@ -16,15 +16,9 @@ */ package com.alibaba.rsqldb.parser.parser; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.CreateSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.FunctionSQLBuilder; - import org.apache.rocketmq.streams.common.configurable.IConfigurable; import org.apache.rocketmq.streams.common.configurable.IConfigurableService; import org.apache.rocketmq.streams.common.topology.ChainPipeline; @@ -34,6 +28,11 @@ import org.apache.rocketmq.streams.common.utils.CollectionUtil; import org.apache.rocketmq.streams.common.utils.PrintUtil; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class SQLTree { protected String namespace; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/AbstractSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/AbstractSQLBuilder.java index 63f5b60..aa88c7a 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/AbstractSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/AbstractSQLBuilder.java @@ -16,8 +16,8 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; import java.util.ArrayList; import java.util.HashMap; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/BlinkUDFScan.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/BlinkUDFScan.java index 5d5c5ad..633e335 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/BlinkUDFScan.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/BlinkUDFScan.java @@ -16,26 +16,10 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Method; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - import com.alibaba.rsqldb.udf.FunctionUDFScript; import com.alibaba.rsqldb.udf.udaf.BlinkUDAFScript; import com.alibaba.rsqldb.udf.udf.BlinkUDFScript; import com.alibaba.rsqldb.udf.udtf.BlinkUDTFScript; - import org.apache.flink.table.functions.AggregateFunction; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.functions.TableFunction; @@ -50,6 +34,21 @@ import org.apache.rocketmq.streams.script.service.udf.UDFScript; import sun.misc.JarFilter; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + /** * 支持blink udf的扫描，指定扫描路径完成udf函数扫描，会把jar包中所有udf扫描出来进行处处，目标把blink udf转化成dipper函数 */ @@ -104,6 +103,7 @@ public static void main(String[] args) { @Deprecated public void scan(String jarFilePath, String classname, String functionName) { scanInnerBlinkUDF(); + String localJarFielPath = ComponentCreator.getProperties().getProperty(BLINK_UDF_JAR_PATH); if (localJarFielPath == null || "".equalsIgnoreCase(localJarFielPath)) { localJarFielPath = "./udflib"; @@ -130,9 +130,13 @@ public void scanInnerBlinkUDF() { scan(null, "com.aliyun.yundun.dipper.sql.udf"); scan(null, "com.aliyun.isec.seraph.udtf"); scan(null, "com.lyra.udf.ext"); + scan(null, "org.apache.rocketmq.streams.script.function.impl.flatmap"); } } + + + /** * 扫描某个目录下jar包的包名 * @@ -233,14 +237,15 @@ public void registerJarUDF(String dir, String className, String methodName) { @Override protected void doProcessor(Class clazz, String functionName) { try { - if (ScalarFunction.class.isAssignableFrom(clazz) || TableFunction.class.isAssignableFrom(clazz) || AggregateFunction.class.isAssignableFrom(clazz)) { + String blinkSuperClassName = getBlinkSuperClassName(clazz); + if (!"".equalsIgnoreCase(blinkSuperClassName)) { if (notSupportUDF.contains(clazz.getSimpleName())) { return; } UDFScript script = null; - if (TableFunction.class.isAssignableFrom(clazz)) { + if (TableFunction.class.isAssignableFrom(clazz) || TableFunction.class.getSimpleName().equalsIgnoreCase(blinkSuperClassName)) { script = new BlinkUDTFScript(); - } else if (AggregateFunction.class.isAssignableFrom(clazz)) { + } else if (AggregateFunction.class.isAssignableFrom(clazz) || AggregateFunction.class.getSimpleName().equalsIgnoreCase(blinkSuperClassName)) { script = new BlinkUDAFScript(); } else { script = new BlinkUDFScript(); @@ -263,6 +268,19 @@ protected void doProcessor(Class clazz, String functionName) { } } + private String getBlinkSuperClassName(Class clazz) { + if (Object.class.getSimpleName().equalsIgnoreCase(clazz.getSimpleName())) { + return ""; + } + if (ScalarFunction.class.getSimpleName().equalsIgnoreCase(clazz.getSimpleName()) || + TableFunction.class.getSimpleName().equalsIgnoreCase(clazz.getSimpleName()) || + AggregateFunction.class.getSimpleName().equalsIgnoreCase(clazz.getSimpleName())) { + return clazz.getSimpleName(); + } else { + return this.getBlinkSuperClassName(clazz.getSuperclass()); + } + } + /** * 将带有@FunctionMethod注解的方法注册为udfscript * @@ -293,8 +311,7 @@ public void registerAnnotationFunction(Class clazz) { } /** - * 根据class和函数名字注册udf，如果方法列表中包含eval方法，则将functionname全部与eval方法绑定并注册UDFScript - * 如果class中不包含eval方法，则将functionname与对应的方法名进行绑定并注册 + * 根据class和函数名字注册udf，如果方法列表中包含eval方法，则将functionname全部与eval方法绑定并注册UDFScript 如果class中不包含eval方法，则将functionname与对应的方法名进行绑定并注册 * * @param clazz * @param functionName @@ -347,6 +364,7 @@ private List getMethodList(Class clazz) { /** * 提取class中方法名为eval的函数列表 + * * @param clazz * @return */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/CreateSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/CreateSQLBuilder.java index 9e2758f..955da18 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/CreateSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/CreateSQLBuilder.java @@ -68,7 +68,10 @@ public void build() { return; } this.source = createSource(); - this.source.setGroupName(getPipelineBuilder().getPipelineName()); + if(StringUtil.isEmpty(this.source.getGroupName())){ + this.source.setGroupName(StringUtil.getUUID()); + } + getPipelineBuilder().setSource(source); getPipelineBuilder().setChannelMetaData(metaData); if(this.getScripts()!=null&&this.getScripts().size()>0){ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/FunctionSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/FunctionSQLBuilder.java index 92f648c..0017988 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/FunctionSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/FunctionSQLBuilder.java @@ -16,15 +16,15 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.HashSet; -import java.util.Set; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.rocketmq.streams.common.model.NameCreator; import org.apache.rocketmq.streams.common.utils.MapKeyUtil; import org.apache.rocketmq.streams.script.service.udf.UDFScript; +import java.util.HashSet; +import java.util.Set; + /** * UDX's SQL Builder */ @@ -42,16 +42,19 @@ public void build() { if (blinkUDFScript == null) { blinkUDFScan.scan(className, null, functionName); blinkUDFScript = blinkUDFScan.getScript(className, functionName); + if (blinkUDFScript == null) { + blinkUDFScript = blinkUDFScan.getScript(className, null); + } if (blinkUDFScript == null) { LOG.error("can not find udf, the udf is " + className); return; } } -// blinkUDFScan.scan(null); + blinkUDFScript.setFunctionName(functionName); + // blinkUDFScan.scan(null); blinkUDFScript.setFunctionName(functionName); blinkUDFScript.setNameSpace(getPipelineBuilder().getPipelineNameSpace()); - String name = MapKeyUtil.createKey(getPipelineBuilder().getPipelineName(), - NameCreator.createNewName(functionName)); + String name = MapKeyUtil.createKey(getPipelineBuilder().getPipelineName(), NameCreator.createNewName(functionName)); blinkUDFScript.setConfigureName(name); getPipelineBuilder().addConfigurables(blinkUDFScript); } diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/ISQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/ISQLBuilder.java index 996aadf..84cf058 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/ISQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/ISQLBuilder.java @@ -16,10 +16,10 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.Set; - import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; +import java.util.Set; + /** * 把一条独立sql对应的描述信息保存下来，并能够builder成dipper的pipeline的节点 */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/InsertSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/InsertSQLBuilder.java index c0bd0b1..51e506a 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/InsertSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/InsertSQLBuilder.java @@ -16,21 +16,16 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.apache.rocketmq.streams.common.channel.sink.ISink; -import org.apache.rocketmq.streams.common.component.ComponentCreator; -import org.apache.rocketmq.streams.common.configure.StreamsConfigure; import org.apache.rocketmq.streams.common.topology.stages.OutputChainStage; -import org.apache.rocketmq.streams.common.metadata.MetaData; import org.apache.rocketmq.streams.common.utils.ContantsUtil; import org.apache.rocketmq.streams.common.utils.PrintUtil; -import com.alibaba.rsqldb.parser.util.ColumnUtil; -import org.apache.calcite.sql.SqlNodeList; import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + public class InsertSQLBuilder extends AbstractSQLBuilder { protected AbstractSQLBuilder builder; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinConditionSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinConditionSQLBuilder.java index 7e09a49..626fc6a 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinConditionSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinConditionSQLBuilder.java @@ -16,18 +16,18 @@ */ package com.alibaba.rsqldb.parser.parser.builder; +import org.apache.rocketmq.streams.script.operator.expression.ScriptExpression; +import org.apache.rocketmq.streams.script.operator.expression.ScriptParameter; +import org.apache.rocketmq.streams.script.operator.impl.FunctionScript; +import org.apache.rocketmq.streams.script.service.IScriptExpression; +import org.apache.rocketmq.streams.script.service.IScriptParamter; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.rocketmq.streams.script.operator.impl.FunctionScript; -import org.apache.rocketmq.streams.script.service.IScriptExpression; -import org.apache.rocketmq.streams.script.service.IScriptParamter; -import org.apache.rocketmq.streams.script.operator.expression.ScriptExpression; -import org.apache.rocketmq.streams.script.operator.expression.ScriptParameter; - public class JoinConditionSQLBuilder extends SelectSQLBuilder { protected Set dimFieldNames; protected String dimAsAlias; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinSQLBuilder.java index 2287999..283d15c 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/JoinSQLBuilder.java @@ -16,6 +16,9 @@ */ package com.alibaba.rsqldb.parser.parser.builder; +import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; +import com.alibaba.rsqldb.parser.parser.result.IParseResult; +import com.alibaba.rsqldb.parser.parser.result.NotSupportParseResult; import org.apache.calcite.sql.SqlNode; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,13 +35,15 @@ import org.apache.rocketmq.streams.filter.operator.expression.Expression; import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression; import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; -import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; -import com.alibaba.rsqldb.parser.parser.result.IParseResult; -import com.alibaba.rsqldb.parser.parser.result.NotSupportParseResult; import org.apache.rocketmq.streams.window.builder.WindowBuilder; import org.apache.rocketmq.streams.window.operator.join.JoinWindow; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -60,8 +65,7 @@ public class JoinSQLBuilder extends SelectSQLBuilder { protected String rightPiplineName;//默认是空，在双流join场景，左右流都可能填充这个值 - - protected boolean needWhereToCondition=false;//need where as onCondition + protected boolean needWhereToCondition = false;//need where as onCondition @Override public void build() { @@ -97,8 +101,7 @@ public void build() { } getPipelineBuilder().addChainStage(new ScriptOperator(stringBuilder.toString())); } - if (right != null && BuilderParseResult.class.isInstance(right) && NotSupportParseResult.class.isInstance(right) - == false) { + if (right != null && BuilderParseResult.class.isInstance(right) && NotSupportParseResult.class.isInstance(right) == false) { BuilderParseResult result = (BuilderParseResult)right; AbstractSQLBuilder builder = result.getBuilder(); builder.setPipelineBuilder(pipelineBuilder); @@ -108,8 +111,8 @@ public void build() { if (SnapshotBuilder.class.isInstance(builder)) { SnapshotBuilder snapshotBuilder = (SnapshotBuilder)builder; //snapshotBuilder.setExpression(onCondition); - snapshotBuilder.buildDimCondition(conditionSQLNode,joinType,onCondition); - }else { + snapshotBuilder.buildDimCondition(conditionSQLNode, joinType, onCondition); + } else { builder.buildSQL(); } @@ -202,9 +205,9 @@ public void addConfigurables(PipelineBuilder pipelineBuilder) { public ChainStage createStageChain(PipelineBuilder pipelineBuilder) { JoinChainStage joinChainStage = new JoinChainStage(); joinChainStage.setWindow(joinWindow); - joinChainStage.setLeftPipline(leftPipelineBuilder.getPipeline()); - joinChainStage.setRightPipline(rightBuilder.getPipeline()); - joinChainStage.setRigthDependentTableName(((BuilderParseResult)right).getBuilder().getTableName()); + joinChainStage.setLeftPipeline(leftPipelineBuilder.getPipeline()); + joinChainStage.setRightPipeline(rightBuilder.getPipeline()); + joinChainStage.setRightDependentTableName(((BuilderParseResult)right).getBuilder().getTableName()); return joinChainStage; } @@ -229,13 +232,13 @@ protected boolean isRightBranch(String parentName) { if (this.rootTableNames.size() <= 1) { return false; } - BuilderParseResult rightResult = (BuilderParseResult) getRight(); - if( rightResult.getBuilder().getTableName().equals(parentName)){ + BuilderParseResult rightResult = (BuilderParseResult)getRight(); + if (rightResult.getBuilder().getTableName().equals(parentName)) { return true; } return false; -// BuilderParseResult rightResult = (BuilderParseResult)getRight(); -// return rightResult.getBuilder().getTableName().equals(parentName); + // BuilderParseResult rightResult = (BuilderParseResult)getRight(); + // return rightResult.getBuilder().getTableName().equals(parentName); } /** @@ -332,8 +335,7 @@ public boolean isJoin() { * @return */ private AbstractSQLBuilder getJoinBuilder(IParseResult result) { - if (result != null && BuilderParseResult.class.isInstance(result) && NotSupportParseResult.class.isInstance(result) - == false) { + if (result != null && BuilderParseResult.class.isInstance(result) && NotSupportParseResult.class.isInstance(result) == false) { BuilderParseResult parseResult = (BuilderParseResult)result; return parseResult.getBuilder(); } else { @@ -441,12 +443,12 @@ public String getFieldName(String fieldName) { asName = fieldName.substring(0, index); fieldName = fieldName.substring(index + 1); } - String tableAsName=null; + String tableAsName = null; if (BuilderParseResult.class.isInstance(getLeft())) { BuilderParseResult builderParseResult = (BuilderParseResult)getLeft(); - tableAsName=builderParseResult.getBuilder().getAsName(); - if(asName!=null&&tableAsName==null){ - tableAsName=builderParseResult.getBuilder().getTableName(); + tableAsName = builderParseResult.getBuilder().getAsName(); + if (asName != null && tableAsName == null) { + tableAsName = builderParseResult.getBuilder().getTableName(); } if ((asName != null && asName.equals(tableAsName)) | StringUtil.isEmpty(asName)) { if (SelectSQLBuilder.class.isInstance(builderParseResult.getBuilder())) { @@ -460,9 +462,9 @@ public String getFieldName(String fieldName) { } if (BuilderParseResult.class.isInstance(getRight())) { BuilderParseResult builderParseResult = (BuilderParseResult)getRight(); - tableAsName=builderParseResult.getBuilder().getAsName(); - if(asName!=null&&tableAsName==null){ - tableAsName=builderParseResult.getBuilder().getTableName(); + tableAsName = builderParseResult.getBuilder().getAsName(); + if (asName != null && tableAsName == null) { + tableAsName = builderParseResult.getBuilder().getTableName(); } if ((asName != null && asName.equals(tableAsName)) | StringUtil.isEmpty(asName)) { if (SelectSQLBuilder.class.isInstance(builderParseResult.getBuilder())) { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/LateralTableBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/LateralTableBuilder.java index f417e74..70fd151 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/LateralTableBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/LateralTableBuilder.java @@ -16,15 +16,15 @@ */ package com.alibaba.rsqldb.parser.parser.builder; +import com.alibaba.rsqldb.parser.parser.result.VarParseResult; +import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.script.function.model.FunctionType; +import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; + import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.rocketmq.streams.script.function.model.FunctionType; -import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; -import com.alibaba.rsqldb.parser.parser.result.VarParseResult; -import org.apache.calcite.sql.SqlNode; - public class LateralTableBuilder extends SelectSQLBuilder { /** diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SelectSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SelectSQLBuilder.java index f68edd9..ca0f944 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SelectSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SelectSQLBuilder.java @@ -16,25 +16,23 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import org.apache.rocketmq.streams.common.model.NameCreator; -import org.apache.rocketmq.streams.filter.operator.FilterOperator; -import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; -import org.apache.rocketmq.streams.common.utils.MapKeyUtil; -import org.apache.rocketmq.streams.common.utils.PrintUtil; -import org.apache.rocketmq.streams.common.utils.StringUtil; -import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; import com.alibaba.rsqldb.parser.parser.SQLParserContext; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.SelectParser; - import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlSelect; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.rocketmq.streams.common.model.NameCreator; +import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder; +import org.apache.rocketmq.streams.common.utils.MapKeyUtil; +import org.apache.rocketmq.streams.common.utils.PrintUtil; +import org.apache.rocketmq.streams.common.utils.StringUtil; +import org.apache.rocketmq.streams.filter.operator.FilterOperator; +import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; import java.util.ArrayList; - import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -49,38 +47,60 @@ public class SelectSQLBuilder extends AbstractSQLBuilder { private static final Log LOG = LogFactory.getLog(SelectParser.class); - protected String expression;//where部分逻辑判断的转换成表达式字符串(varName,functionName,value)&((varName,functionName,value)|(varName,functionName,int,value)) - - protected Set allFieldNames = null;//如果做过字段解析，把所有的字段放到这里面，在做sub select 取值时，直接取 - /** - * 在sql select部分出现的字段 + * where部分逻辑判断的转换成表达式字符串(varName,functionName,value)&((varName,functionName,value)|(varName,functionName,int,value)) + */ + protected String expression; + /** + * 如果做过字段解析，把所有的字段放到这里面，在做sub select 取值时，直接取 + */ + protected Set allFieldNames = null; + /** + * 在sql select部分出现的字段,每个字段和对应解析器，在必要的时候完成解析 */ - protected Map fieldName2ParseResult = new HashMap<>();//每个字段和对应解析器，在必要的时候完成解析 - protected List fieldNamesOrderByDeclare=new ArrayList<>();//field name order by sql select,use in insert table(field,field) select - protected int parseStage = 0;//0:select,1:from,2:where + protected Map fieldName2ParseResult = new HashMap<>(); + /** + * field name order by sql select,use in insert table(field,field) select + */ + protected List fieldNamesOrderByDeclare = new ArrayList<>(); + /** + * 0:select,1:from,2:where + */ + protected int parseStage = 0; protected List selectScripts = new ArrayList<>(); - - protected JoinSQLBuilder joinSQLBuilder;// select场景中有join的场景 - protected SelectSQLBuilder parentSelect;//如果有嵌套查询，则表示外侧查询 - protected SelectSQLBuilder subSelect;//如果有嵌套查询，则表示子查询 - protected boolean isDistinct=false; + /** + * select场景中有join的场景 + */ + protected JoinSQLBuilder joinSQLBuilder; + /** + * 如果有嵌套查询，则表示外侧查询 + */ + protected SelectSQLBuilder parentSelect; + /** + * 如果有嵌套查询，则表示子查询 + */ + protected SelectSQLBuilder subSelect; + protected boolean isDistinct = false; /** * group by 相关参数 */ protected WindowBuilder windowBuilder; protected String overName; - - //有union的场景 + /** + * 有union的场景 + */ protected UnionSQLBuilder unionSQLBuilder; - - //主要是lower或trim等函数，如果同一字段添加多次函数，最终只返回一个名字。使用时需要谨慎 - //string：包含函数名;变量名 + /** + * 主要是lower或trim等函数，如果同一字段添加多次函数，最终只返回一个名字。使用时需要谨慎 string：包含函数名;变量名 + */ protected Map innerVarNames = new HashMap(); + /** + * use in create table + */ + protected boolean closeFieldCheck = false; - protected boolean closeFieldCheck=false;//use in create table @Override public void buildSQL() { if (pipelineBuilder == null) { @@ -101,14 +121,14 @@ public void buildSQL() { } if (unionSQLBuilder != null) { - if(!unionSQLBuilder.getTableName().equals(pipelineBuilder.getParentTableName())){ - if(unionSQLBuilder.containsTableName(pipelineBuilder.getParentTableName())){ + if (!unionSQLBuilder.getTableName().equals(pipelineBuilder.getParentTableName())) { + if (unionSQLBuilder.containsTableName(pipelineBuilder.getParentTableName())) { unionSQLBuilder.setTableName(pipelineBuilder.getParentTableName()); this.setTableName(unionSQLBuilder.getTableName()); - SelectSQLBuilder parent=this.parentSelect; - while (parent!=null){ + SelectSQLBuilder parent = this.parentSelect; + while (parent != null) { parent.setTableName(this.getTableName()); - parent=parent.parentSelect; + parent = parent.parentSelect; } } unionSQLBuilder.setTableName(pipelineBuilder.getParentTableName()); @@ -175,8 +195,8 @@ protected void bulidExpression() { if (StringUtil.isNotEmpty(expression)) { SqlSelect sqlSelect = (SqlSelect)sqlNode; sqlSelect.setWhere(null); - String ruleName= NameCreator.createOrGet(this.getPipelineBuilder().getPipelineName()).createName(this.getPipelineBuilder().getPipelineName(),"rule"); - pipelineBuilder.addChainStage(new FilterOperator(getNamespace(),ruleName,expression)); + String ruleName = NameCreator.createOrGet(this.getPipelineBuilder().getPipelineName()).createName(this.getPipelineBuilder().getPipelineName(), "rule"); + pipelineBuilder.addChainStage(new FilterOperator(getNamespace(), ruleName, expression)); } } @@ -187,7 +207,7 @@ protected void bulidExpression() { protected void buildGroup() { if (windowBuilder != null) { windowBuilder.setPipelineBuilder(pipelineBuilder); - // windowBuilder.setTreeSQLBulider(getTreeSQLBulider()); + //windowBuilder.setTreeSQLBulider(getTreeSQLBulider()); windowBuilder.setOwner(this); windowBuilder.build(); } @@ -205,8 +225,8 @@ protected void buildSelect() { */ String retainScript = "retainField("; boolean isFirst = true; - String[] fieldNames=new String[fieldName2ParseResult.size()]; - int i=0; + String[] fieldNames = new String[fieldName2ParseResult.size()]; + int i = 0; if (fieldName2ParseResult != null) { Iterator> it = fieldName2ParseResult.entrySet().iterator(); @@ -219,9 +239,9 @@ protected void buildSelect() { retainScript += ","; } if (entry.getKey().indexOf("*") != -1) { - String fieldName=doAsteriskTrimAliasName(entry.getKey(), stringBuilder, allFieldNames); + String fieldName = doAsteriskTrimAliasName(entry.getKey(), stringBuilder, allFieldNames); retainScript += fieldName; - fieldNames[i]=fieldName; + fieldNames[i] = fieldName; } else { String name = entry.getKey(); if (name.indexOf(".") != -1) { @@ -233,7 +253,7 @@ protected void buildSelect() { } allFieldNames.add(name); retainScript += name; - fieldNames[i]=name; + fieldNames[i] = name; } IParseResult parseResult = entry.getValue(); @@ -265,9 +285,9 @@ protected void buildSelect() { String scriptValue = stringBuilder.toString(); this.allFieldNames = allFieldNames; if (isFirst == false) { - if(isDistinct){ - String distinctScript=scriptValue.replace("retainField","distinct"); - scriptValue+=distinctScript; + if (isDistinct) { + String distinctScript = scriptValue.replace("retainField", "distinct"); + scriptValue += distinctScript; } pipelineBuilder.addChainStage(new ScriptOperator(scriptValue)); //optimizer.put(scriptValue,true); @@ -400,7 +420,7 @@ protected String doAsteriskTrimAliasName(String key, StringBuilder stringBuilder if (index != -1) { String name = fieldName.substring(index + 1); if (fieldNames.contains(name)) { - stringBuilder.append(name + "=" + fieldName + ";" + PrintUtil.LINE); + stringBuilder.append(name).append("=").append(fieldName).append(";").append(PrintUtil.LINE); list.add(name); } } else { @@ -423,7 +443,7 @@ protected boolean inSelectField(String fieldName) { Entry entry = it.next(); String key = entry.getKey(); String value = null; - if (ScriptParseResult.class.isInstance(entry.getValue())) { + if (entry.getValue() instanceof ScriptParseResult) { ScriptParseResult scriptParseResult = (ScriptParseResult)entry.getValue(); value = scriptParseResult.getScript(); if (value.endsWith("=" + fieldName + ";")) { @@ -431,11 +451,7 @@ protected boolean inSelectField(String fieldName) { } } else { value = entry.getValue().getReturnValue(); - if (key.equals(value)) { - continue; - } } - } return false; } @@ -461,8 +477,8 @@ public Set parseDependentTables() { if (join != null) { dependentTables.addAll(join.parseDependentTables()); } - UnionSQLBuilder union=this.unionSQLBuilder; - if(union!=null){ + UnionSQLBuilder union = this.unionSQLBuilder; + if (union != null) { dependentTables.addAll(union.parseDependentTables()); } return dependentTables; @@ -608,9 +624,9 @@ public String getFieldName(String fieldName, boolean containsSelf) { } else { String ailasName = fieldName.substring(0, index); fieldName = fieldName.substring(index + 1); - String tableAilasName=getAsName(); - if(ailasName!=null&&tableAilasName==null){ - tableAilasName=getTableName(); + String tableAilasName = getAsName(); + if (ailasName != null && tableAilasName == null) { + tableAilasName = getTableName(); } if (ailasName.equals(tableAilasName)) { if (fieldNames.contains(fieldName)) { @@ -631,7 +647,7 @@ public String getFieldName(String fieldName, boolean containsSelf) { @Override @Deprecated public String getFieldName(String fieldName) { - if(isCloseFieldCheck()){ + if (isCloseFieldCheck()) { return fieldName; } return getFieldName(fieldName, false); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SnapshotBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SnapshotBuilder.java index 875b277..a6d4078 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SnapshotBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/SnapshotBuilder.java @@ -16,17 +16,20 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; - +import com.alibaba.rsqldb.parser.parser.ISqlParser; +import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; +import com.alibaba.rsqldb.parser.parser.SQLParserContext; +import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; +import com.alibaba.rsqldb.parser.parser.result.IParseResult; +import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; +import org.apache.rocketmq.streams.common.metadata.MetaData; +import org.apache.rocketmq.streams.common.metadata.MetaDataField; +import org.apache.rocketmq.streams.common.utils.ContantsUtil; +import org.apache.rocketmq.streams.common.utils.MapKeyUtil; +import org.apache.rocketmq.streams.common.utils.ReflectUtil; +import org.apache.rocketmq.streams.common.utils.StringUtil; +import org.apache.rocketmq.streams.db.driver.JDBCDriver; import org.apache.rocketmq.streams.dim.builder.IDimSQLParser; import org.apache.rocketmq.streams.dim.builder.SQLParserFactory; import org.apache.rocketmq.streams.dim.intelligence.AbstractIntelligenceCache; @@ -38,27 +41,23 @@ import org.apache.rocketmq.streams.dim.model.DBDim; import org.apache.rocketmq.streams.dim.model.FileDim; import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder; - import org.apache.rocketmq.streams.filter.function.expression.Equals; import org.apache.rocketmq.streams.filter.operator.Rule; import org.apache.rocketmq.streams.filter.operator.expression.Expression; import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression; import org.apache.rocketmq.streams.filter.operator.expression.SimpleExpression; -import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; -import org.apache.rocketmq.streams.common.metadata.MetaData; -import org.apache.rocketmq.streams.common.metadata.MetaDataField; -import org.apache.rocketmq.streams.common.utils.ContantsUtil; -import org.apache.rocketmq.streams.common.utils.MapKeyUtil; -import org.apache.rocketmq.streams.common.utils.ReflectUtil; -import org.apache.rocketmq.streams.common.utils.StringUtil; import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator; -import com.alibaba.rsqldb.parser.parser.SQLParserContext; -import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; -import com.alibaba.rsqldb.parser.parser.ISqlParser; -import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; -import com.alibaba.rsqldb.parser.parser.result.IParseResult; -import org.apache.calcite.sql.SqlNode; -import org.apache.rocketmq.streams.db.driver.JDBCDriver; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; /** * dimension table join builder specially for url, ip and domain @@ -66,9 +65,7 @@ public class SnapshotBuilder extends SelectSQLBuilder { protected static Map INTELLIGENCE = new HashMap<>(); - protected static Map dims=new HashMap<>(); - - + protected static Map dims = new HashMap<>(); static { DomainIntelligenceCache domainIntelligenceCache = new DomainIntelligenceCache(); @@ -88,10 +85,11 @@ public void buildSQL() { /** * 在join sql build中调用，一个sql如果有对一个维表多次join，数据只存一份。通过建立不同索引实现 + * * @param condition * @param joinType */ - public void buildDimCondition(SqlNode condition, String joinType,String onCondition) { + public void buildDimCondition(SqlNode condition, String joinType, String onCondition) { CreateSQLBuilder builder = SQLCreateTables.getInstance().get().get(getTableName()); Properties properties = builder.createProperty(); @@ -101,9 +99,9 @@ public void buildDimCondition(SqlNode condition, String joinType,String onCondit long pollingTime = 30;//默认更新时间是30分钟 if (StringUtil.isNotEmpty(cacheTTLMs)) { - long ms=Long.valueOf(cacheTTLMs); - if(ms<1000*60){ - ms=1000*60; + long ms = Long.valueOf(cacheTTLMs); + if (ms < 1000 * 60) { + ms = 1000 * 60; } pollingTime = (Long.valueOf(ms) / 1000 / 60); } @@ -112,16 +110,16 @@ public void buildDimCondition(SqlNode condition, String joinType,String onCondit */ AbstractIntelligenceCache intelligenceCache = INTELLIGENCE.get(tableName); if (intelligenceCache != null) { - buildIntelligence(pollingTime, intelligenceCache,joinType,onCondition); + buildIntelligence(pollingTime, intelligenceCache, joinType, onCondition); } else { - AbstractDim dim=dims.get(builder); + AbstractDim dim = dims.get(builder); /** * 通用维表builder */ - if(dim==null){ - dim=buildDim(builder, properties); + if (dim == null) { + dim = buildDim(builder, properties); getPipelineBuilder().addConfigurables(dim); - dims.put(builder,dim); + dims.put(builder, dim); } /** @@ -130,50 +128,51 @@ public void buildDimCondition(SqlNode condition, String joinType,String onCondit String selectFields = createSelectFields(builder); Set fieldNames = createFieldNames(selectFields); JoinConditionSQLBuilder conditionSQLBuilder = new JoinConditionSQLBuilder(fieldNames, asName); - String expressionStr = convertExpression(conditionSQLBuilder,condition, selectFields, this.getAsName(), selectFields); + String expressionStr = convertExpression(conditionSQLBuilder, condition, selectFields, this.getAsName(), selectFields); //编译join条件 - createIndexByJoinCondition(dim,expressionStr,builder); - addDimJoib2Pipeline(dim,conditionSQLBuilder,expressionStr,joinType,selectFields); + createIndexByJoinCondition(dim, expressionStr, builder); + addDimJoib2Pipeline(dim, conditionSQLBuilder, expressionStr, joinType, selectFields); } } private String getDimType(Properties properties) { - String type= properties.getProperty("type"); - if(StringUtil.isEmpty(type)){ - type= properties.getProperty("TYPE"); + String type = properties.getProperty("type"); + if (StringUtil.isEmpty(type)) { + type = properties.getProperty("TYPE"); } - if(StringUtil.isEmpty(type)){ - type= properties.getProperty("connector"); + if (StringUtil.isEmpty(type)) { + type = properties.getProperty("connector"); } - if(StringUtil.isEmpty(type)){ - type= properties.getProperty("CONNECTOR"); + if (StringUtil.isEmpty(type)) { + type = properties.getProperty("CONNECTOR"); } return type; } + /** * 创建dim对象，相同的create table ，只创建一个对象 + * * @param builder * @param properties * @return */ - protected AbstractDim buildDim( CreateSQLBuilder builder, Properties properties){ - String type=getDimType(properties).toLowerCase(); - IDimSQLParser dimSQLParser= SQLParserFactory.getInstance().create(type); - return dimSQLParser.parseDim(namespace,properties,builder.getMetaData()); + protected AbstractDim buildDim(CreateSQLBuilder builder, Properties properties) { + String type = getDimType(properties).toLowerCase(); + IDimSQLParser dimSQLParser = SQLParserFactory.getInstance().create(type); + return dimSQLParser.parseDim(namespace, properties, builder.getMetaData()); } - - /** * 创建dim对象，相同的create table ，只创建一个对象 + * * @param pollingTime * @param tableName * @param builder * @param properties * @return */ - protected AbstractDim buildDBDim(long pollingTime, String tableName, CreateSQLBuilder builder, Properties properties){ + protected AbstractDim buildDBDim(long pollingTime, String tableName, CreateSQLBuilder builder, Properties properties) { /** * 创建namelist，要起必须有pco rimary key，，否则抛出错误 */ @@ -185,9 +184,7 @@ protected AbstractDim buildDBDim(long pollingTime, String tableName, CreateSQLBu dbNameList.setUrl(url); dbNameList.setUserName(userName); dbNameList.setPassword(password); - dbNameList.setPollingTimeMintue(pollingTime); - - + dbNameList.setPollingTimeMinute(pollingTime); String selectFields = createSelectFields(builder); String sql = "select " + selectFields + " from " + tableName + " limit 1000000"; @@ -202,23 +199,24 @@ protected AbstractDim buildDBDim(long pollingTime, String tableName, CreateSQLBu } protected AbstractDim buildFileDim(long time, String name, CreateSQLBuilder builder, Properties properties) { - String filePath=properties.getProperty("filePath"); - if(StringUtil.isEmpty(filePath)){ - filePath=properties.getProperty("file_path"); + String filePath = properties.getProperty("filePath"); + if (StringUtil.isEmpty(filePath)) { + filePath = properties.getProperty("file_path"); } - FileDim fileDim=new FileDim(); + FileDim fileDim = new FileDim(); fileDim.setFilePath(filePath); return fileDim; } + /** * 根据join条件设置索引 + * * @param dbNameList */ - protected void createIndexByJoinCondition(AbstractDim dbNameList,String expressionStr, CreateSQLBuilder createSQLBuilder) { + protected void createIndexByJoinCondition(AbstractDim dbNameList, String expressionStr, CreateSQLBuilder createSQLBuilder) { List expressions = new ArrayList<>(); List relationExpressions = new ArrayList<>(); - Expression expression = ExpressionBuilder.createOptimizationExpression("tmp", "tmp", expressionStr, - expressions, relationExpressions); + Expression expression = ExpressionBuilder.createOptimizationExpression("tmp", "tmp", expressionStr, expressions, relationExpressions); RelationExpression relationExpression = null; if (RelationExpression.class.isInstance(expression)) { @@ -242,7 +240,7 @@ protected void createIndexByJoinCondition(AbstractDim dbNameList,String expressi relationExpression.setValue(new ArrayList<>()); for (String expressionName : expressionNames) { Expression subExpression = map.get(expressionName); - if (subExpression != null && !RelationExpression.class.isInstance(subExpression)&&isDimField(subExpression.getValue(),createSQLBuilder)) { + if (subExpression != null && !RelationExpression.class.isInstance(subExpression) && isDimField(subExpression.getValue(), createSQLBuilder)) { indexExpressions.add(subExpression); } else { otherExpressions.add(subExpression); @@ -261,35 +259,35 @@ protected void createIndexByJoinCondition(AbstractDim dbNameList,String expressi continue; } String indexName = expre.getValue().toString(); - if (Equals.isEqualFunction(expre.getFunctionName()) &&isDimField(expre.getValue(),createSQLBuilder)) { + if (Equals.isEqualFunction(expre.getFunctionName()) && isDimField(expre.getValue(), createSQLBuilder)) { fieldNames.add(indexName); } } - String[] indexFieldNameArray=new String[fieldNames.size()]; - int i=0; - for(String fieldName:fieldNames){ - indexFieldNameArray[i]=fieldName; + String[] indexFieldNameArray = new String[fieldNames.size()]; + int i = 0; + for (String fieldName : fieldNames) { + indexFieldNameArray[i] = fieldName; i++; } Arrays.sort(indexFieldNameArray); String index = MapKeyUtil.createKey(indexFieldNameArray); - if(dbNameList.getIndexs().contains(index)){ + if (dbNameList.getIndexs().contains(index)) { return; } - if(indexFieldNameArray.length>0){ + if (indexFieldNameArray.length > 0) { dbNameList.addIndex(indexFieldNameArray); } } - protected boolean isDimField(Object value,CreateSQLBuilder createSQLBuilder){ - if(!String.class.isInstance(value)){ + protected boolean isDimField(Object value, CreateSQLBuilder createSQLBuilder) { + if (!String.class.isInstance(value)) { return false; } - MetaData metaData= createSQLBuilder.getMetaData(); - if(metaData.getMetaDataField((String)value)!=null){ + MetaData metaData = createSQLBuilder.getMetaData(); + if (metaData.getMetaDataField((String)value) != null) { return true; } return false; @@ -297,13 +295,14 @@ protected boolean isDimField(Object value,CreateSQLBuilder createSQLBuilder){ /** * 生成dim的脚本，并设置pipeline的stage + * * @param dbNameList * @param conditionSQLBuilder * @param expression * @param joinType * @param selectFields */ - protected void addDimJoib2Pipeline(AbstractDim dbNameList,JoinConditionSQLBuilder conditionSQLBuilder,String expression,String joinType,String selectFields){ + protected void addDimJoib2Pipeline(AbstractDim dbNameList, JoinConditionSQLBuilder conditionSQLBuilder, String expression, String joinType, String selectFields) { List scriptValue = conditionSQLBuilder.getScripts(); String dimScript = conditionSQLBuilder.getDimScriptValue(); if (dimScript == null) { @@ -323,7 +322,6 @@ protected void addDimJoib2Pipeline(AbstractDim dbNameList,JoinConditionSQLBuilde getPipelineBuilder().addChainStage(new ScriptOperator(conditionSQLBuilder.createScript(scriptValue))); } - private Set createFieldNames(String selectFields) { Set fieldNames = new HashSet<>(); if (selectFields != null) { @@ -334,13 +332,14 @@ private Set createFieldNames(String selectFields) { } return fieldNames; } + /** * 创建情报维表，和对应的脚本。情报的表达式，支持等值操作 * * @param pollingTime 多长时间加载一次 * @param intelligenceCache 情报对应的对象 */ - protected void buildIntelligence(long pollingTime, AbstractIntelligenceCache intelligenceCache,String joinType,String expressionStr) { + protected void buildIntelligence(long pollingTime, AbstractIntelligenceCache intelligenceCache, String joinType, String expressionStr) { /** * 创建维表连接对象， 默认情报的数据连接是单独配置好的，不依赖sql中create语句 */ @@ -356,13 +355,11 @@ protected void buildIntelligence(long pollingTime, AbstractIntelligenceCache int String intelligenceKey = null; Rule rule = ExpressionBuilder.createRule("tmp", "tmp", expressionStr); if (rule.getExpressionMap().size() > 1) { - throw new RuntimeException( - "can not support expression in intelligence . the expression is " + expression); + throw new RuntimeException("can not support expression in intelligence . the expression is " + expression); } Expression expression = rule.getExpressionMap().values().iterator().next(); if (!SimpleExpression.class.isInstance(expression)) { - throw new RuntimeException( - "can not support expression in intelligence . the expression is " + expressionStr); + throw new RuntimeException("can not support expression in intelligence . the expression is " + expressionStr); } if (expression.getVarName().equals(asName + "." + intelligence.getKeyName()) || expression.getVarName().equals(intelligence.getKeyName())) { intelligenceKey = expression.getValue().toString(); @@ -374,19 +371,13 @@ protected void buildIntelligence(long pollingTime, AbstractIntelligenceCache int * */ if (joinType.toUpperCase().equals("INNER")) { - getPipelineBuilder().addChainStage(new ScriptOperator( - "intelligence('" + intelligence.getNameSpace() + "','" + intelligence.getConfigureName() + "'," - + intelligenceKey + ",'" + getAsName() + "');")); + getPipelineBuilder().addChainStage(new ScriptOperator("intelligence('" + intelligence.getNameSpace() + "','" + intelligence.getConfigureName() + "'," + intelligenceKey + ",'" + getAsName() + "');")); } else if (joinType.toUpperCase().equals("LEFT")) { - getPipelineBuilder().addChainStage(new ScriptOperator( - "left_join_intelligence('" + intelligence.getNameSpace() + "','" + intelligence.getConfigureName() + "'," - + intelligenceKey + ",'" + getAsName() + "');")); + getPipelineBuilder().addChainStage(new ScriptOperator("left_join_intelligence('" + intelligence.getNameSpace() + "','" + intelligence.getConfigureName() + "'," + intelligenceKey + ",'" + getAsName() + "');")); } } - - /** * 根据字段名，创建sql，最大加载10 w条数据，超过10w会被截断 * @@ -447,11 +438,10 @@ public Set parseDependentTables() { return tables; } - /** * 维表不识别别名，需要做去除。维表join，要求维表字段必须在value字段，如果sql写反了，需要转换过来 */ - protected String convertExpression(JoinConditionSQLBuilder conditionSQLBuilder,SqlNode expressionSQLNode, String fieldNames, String aliasName, String selectFields) { + protected String convertExpression(JoinConditionSQLBuilder conditionSQLBuilder, SqlNode expressionSQLNode, String fieldNames, String aliasName, String selectFields) { ISqlParser sqlParser = SQLNodeParserFactory.getParse(expressionSQLNode); conditionSQLBuilder.switchWhere(); @@ -526,6 +516,4 @@ protected String convertExpression(JoinConditionSQLBuilder conditionSQLBuilder,S return expression.toExpressionString(map); } - - } diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/TableNodeBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/TableNodeBuilder.java index 5b7968c..56bf426 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/TableNodeBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/TableNodeBuilder.java @@ -16,11 +16,11 @@ */ package com.alibaba.rsqldb.parser.parser.builder; +import com.alibaba.rsqldb.parser.parser.SQLParserContext; + import java.util.HashSet; import java.util.Set; -import com.alibaba.rsqldb.parser.parser.SQLParserContext; - public class TableNodeBuilder extends SelectSQLBuilder { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/UnionSQLBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/UnionSQLBuilder.java index 102d175..0c45e3d 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/UnionSQLBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/UnionSQLBuilder.java @@ -16,13 +16,8 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.rocketmq.streams.common.model.NameCreator; import org.apache.rocketmq.streams.common.topology.ChainPipeline; import org.apache.rocketmq.streams.common.topology.ChainStage; @@ -31,8 +26,13 @@ import org.apache.rocketmq.streams.common.topology.stages.UnionChainStage; import org.apache.rocketmq.streams.common.utils.NameCreatorUtil; import org.apache.rocketmq.streams.common.utils.StringUtil; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class UnionSQLBuilder extends SelectSQLBuilder { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/WindowBuilder.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/WindowBuilder.java index 8fc2a2e..fb621e7 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/WindowBuilder.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/builder/WindowBuilder.java @@ -16,31 +16,31 @@ */ package com.alibaba.rsqldb.parser.parser.builder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; - +import com.alibaba.rsqldb.parser.parser.result.IParseResult; +import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import org.apache.rocketmq.streams.common.configure.StreamsConfigure; +import org.apache.rocketmq.streams.common.utils.CollectionUtil; +import org.apache.rocketmq.streams.common.utils.MapKeyUtil; import org.apache.rocketmq.streams.common.utils.StringUtil; import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder; import org.apache.rocketmq.streams.filter.operator.expression.Expression; import org.apache.rocketmq.streams.filter.optimization.dependency.ScriptDependent; import org.apache.rocketmq.streams.script.service.IScriptExpression; import org.apache.rocketmq.streams.window.operator.AbstractWindow; -import org.apache.rocketmq.streams.common.utils.CollectionUtil; -import org.apache.rocketmq.streams.common.utils.MapKeyUtil; -import com.alibaba.rsqldb.parser.parser.result.IParseResult; -import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import org.apache.rocketmq.streams.window.operator.impl.SessionOperator; import org.apache.rocketmq.streams.window.operator.impl.ShuffleOverWindow; import org.apache.rocketmq.streams.window.operator.impl.WindowOperator; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + public class WindowBuilder extends SelectSQLBuilder { /** @@ -83,52 +83,54 @@ public class WindowBuilder extends SelectSQLBuilder { */ protected String type; - /** * 时间字段 */ protected String timeFieldName; - protected int timeUnitAdjust=60; + protected int timeUnitAdjust = 60; protected List groupByFieldNames = new ArrayList<>(); protected SelectSQLBuilder owner; - protected boolean isLocalStorageOnly = true;//是否只用本地存储 + /** + * 是否只用本地存储 + */ + protected boolean isLocalStorageOnly = true; protected String having; protected List havingScript; /** - * over window + * over window, 值为over时，是over window */ - protected String overWindowName;//值为over时，是over window - protected boolean isShuffleOverWindow=false; + protected String overWindowName; + protected boolean isShuffleOverWindow = false; protected List shuffleOverWindowOrderByFieldNames; - protected int overWindowTopN=100; + protected int overWindowTopN = 100; + @Override protected void build() { AbstractWindow window; if (overWindowName != null) { - if(!isShuffleOverWindow){ + if (!isShuffleOverWindow) { buildOverWindow(); return; } - return; } - window = org.apache.rocketmq.streams.window.builder.WindowBuilder.createWindow(type); - if(window.getEmitBeforeValue()==null||window.getEmitBeforeValue()==0){ - window.setEmitBeforeValue( StreamsConfigure.getEmitBeforeValue()); + if (window.getEmitBeforeValue() == null || window.getEmitBeforeValue() == 0) { + window.setEmitBeforeValue(StreamsConfigure.getEmitBeforeValue()); } - if(window.getEmitAfterValue()==null||window.getEmitAfterValue()==0){ + if (window.getEmitAfterValue() == null || window.getEmitAfterValue() == 0) { window.setEmitAfterValue(StreamsConfigure.getEmitAfterValue()); - if(StreamsConfigure.getEmitMaxDelay()!=null){ + if (StreamsConfigure.getEmitMaxDelay() != null) { window.setMaxDelay(StreamsConfigure.getEmitMaxDelay()); } } window.setLocalStorageOnly(isLocalStorageOnly); window.setTimeFieldName(timeFieldName); window.setWindowType(type); + window.setTimeUnitAdjust(1); if (window instanceof WindowOperator) { window.setSizeInterval(Optional.ofNullable(size).orElse(AbstractWindow.DEFAULT_WINDOW_SIZE)); @@ -138,17 +140,13 @@ protected void build() { window.setSlideInterval(Optional.ofNullable(slide).orElse(AbstractWindow.DEFAULT_WINDOW_SLIDE)); window.setSlideVariable(slideVariable); window.setSlideAdjust(slideAdjust); - - - window.setTimeUnitAdjust(Optional.ofNullable(timeUnitAdjust).orElse(60)); } if (window instanceof SessionOperator) { - SessionOperator theWindow = (SessionOperator) window; + SessionOperator theWindow = (SessionOperator)window; theWindow.setSessionTimeOut(Optional.ofNullable(timeout).orElse(AbstractWindow.DEFAULT_WINDOW_SESSION_TIMEOUT)); } - Map selectMap = new HashMap<>(32); if (owner.getFieldName2ParseResult() != null) { //select部分处理，map：key字段名，value：脚本或字段名 @@ -173,18 +171,18 @@ protected void build() { } protected void buildHaving(AbstractWindow window) { - if(StringUtil.isEmpty(having)){ + if (StringUtil.isEmpty(having)) { return; } window.setHavingExpression(having); - if(havingScript!=null){ - List expressions=new ArrayList<>(); - ExpressionBuilder.createExpression("tmp","tmp",having,expressions,new ArrayList<>()); - for(Expression expression:expressions){ - String varName=expression.getVarName(); - List dependentScripts=getDependentScripts(varName,havingScript); - if(CollectionUtil.isNotEmpty(dependentScripts)){ - window.getSelectMap().put(varName,MapKeyUtil.createKey(";",dependentScripts)+";"); + if (havingScript != null) { + List expressions = new ArrayList<>(); + ExpressionBuilder.createExpression("tmp", "tmp", having, expressions, new ArrayList<>()); + for (Expression expression : expressions) { + String varName = expression.getVarName(); + List dependentScripts = getDependentScripts(varName, havingScript); + if (CollectionUtil.isNotEmpty(dependentScripts)) { + window.getSelectMap().put(varName, MapKeyUtil.createKey(";", dependentScripts) + ";"); } } } @@ -192,28 +190,28 @@ protected void buildHaving(AbstractWindow window) { } private List getDependentScripts(String varName, List script) { - ScriptDependent scriptDependent=new ScriptDependent(null,MapKeyUtil.createKey(";",script)+";"); - List scriptExpressions= scriptDependent.getDependencyExpression(varName); - List expressions=new ArrayList<>(); - for(IScriptExpression scriptExpression:scriptExpressions){ + ScriptDependent scriptDependent = new ScriptDependent(null, MapKeyUtil.createKey(";", script) + ";"); + List scriptExpressions = scriptDependent.getDependencyExpression(varName); + List expressions = new ArrayList<>(); + for (IScriptExpression scriptExpression : scriptExpressions) { expressions.add(scriptExpression.toString()); } return expressions; } protected void buildOverWindow() { - AbstractWindow overWindow=null; - String groupBy=MapKeyUtil.createKey(";", groupByFieldNames); - if(!isShuffleOverWindow){ - overWindow = org.apache.rocketmq.streams.window.builder.WindowBuilder.createOvertWindow(groupBy, overWindowName); - }else { - ShuffleOverWindow shuffleOverWindow=new ShuffleOverWindow(); + AbstractWindow overWindow = null; + String groupBy = MapKeyUtil.createKey(";", groupByFieldNames); + if (!isShuffleOverWindow) { + overWindow = org.apache.rocketmq.streams.window.builder.WindowBuilder.createOvertWindow(groupBy, overWindowName); + } else { + ShuffleOverWindow shuffleOverWindow = new ShuffleOverWindow(); shuffleOverWindow.setTimeFieldName(""); shuffleOverWindow.setGroupByFieldName(groupBy); shuffleOverWindow.setRowNumerName(overWindowName); shuffleOverWindow.setTopN(overWindowTopN); shuffleOverWindow.setOrderFieldNames(shuffleOverWindowOrderByFieldNames); - overWindow=shuffleOverWindow; + overWindow = shuffleOverWindow; } getPipelineBuilder().addChainStage(overWindow); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRule.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRule.java index 23f8c9f..b0038b8 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRule.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRule.java @@ -17,13 +17,14 @@ package com.alibaba.rsqldb.parser.parser.expression; import com.alibaba.fastjson.JSONObject; +import org.apache.rocketmq.streams.common.utils.MapKeyUtil; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.rocketmq.streams.common.utils.MapKeyUtil; public class BlinkRule { /** diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRuleV2Parser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRuleV2Parser.java index a757b85..5d879f5 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRuleV2Parser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/BlinkRuleV2Parser.java @@ -24,6 +24,12 @@ import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.utils.FileUtil; +import org.apache.rocketmq.streams.common.utils.ReflectUtil; +import org.apache.rocketmq.streams.script.utils.FunctionUtils; + import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; @@ -32,12 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlNode; -import org.apache.rocketmq.streams.common.utils.FileUtil; -import org.apache.rocketmq.streams.common.utils.MapKeyUtil; -import org.apache.rocketmq.streams.common.utils.ReflectUtil; -import org.apache.rocketmq.streams.script.utils.FunctionUtils; /** * create by udf diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/CompareParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/CompareParser.java index df1d18e..6c2bc3c 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/CompareParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/CompareParser.java @@ -16,10 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.expression; -import java.util.List; - -import org.apache.rocketmq.streams.common.datatype.DataType; -import org.apache.rocketmq.streams.common.datatype.StringDataType; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; @@ -28,6 +24,10 @@ import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.datatype.DataType; +import org.apache.rocketmq.streams.common.datatype.StringDataType; + +import java.util.List; public class CompareParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/InParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/InParser.java index 6fb37a2..144a231 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/InParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/InParser.java @@ -16,9 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.expression; -import java.util.List; - -import org.apache.rocketmq.streams.common.datatype.StringDataType; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; @@ -28,6 +25,9 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.rocketmq.streams.common.datatype.StringDataType; + +import java.util.List; public class InParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/IsTrueFunction.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/IsTrueFunction.java index a9138f4..67c0896 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/IsTrueFunction.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/IsTrueFunction.java @@ -16,13 +16,14 @@ */ package com.alibaba.rsqldb.parser.parser.expression; -import java.util.List; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlNode; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlNode; + +import java.util.List; public class IsTrueFunction extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/LikeFunction.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/LikeFunction.java index b3c731d..c752ad0 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/LikeFunction.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/LikeFunction.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.expression; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -27,6 +25,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.fun.SqlLikeOperator; +import java.util.List; + public class LikeFunction extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/RegexpParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/RegexpParser.java index c534b39..40806ec 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/RegexpParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/expression/RegexpParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.expression; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -26,6 +24,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class RegexpParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CastFunctionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CastFunctionParser.java index 2ced460..0d83cdb 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CastFunctionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CastFunctionParser.java @@ -16,9 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - -import org.apache.rocketmq.streams.common.datatype.DataType; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -27,6 +24,9 @@ import com.alibaba.rsqldb.parser.util.SqlDataTypeUtil; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.datatype.DataType; + +import java.util.List; public class CastFunctionParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CountParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CountParser.java index 6655362..015e2d2 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CountParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/CountParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -26,6 +24,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class CountParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/FilterFunction.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/FilterFunction.java index bebb187..857d90c 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/FilterFunction.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/FilterFunction.java @@ -16,14 +16,15 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.ArrayList; -import java.util.List; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlNode; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlNode; + +import java.util.ArrayList; +import java.util.List; public class FilterFunction extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/GroupFunctionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/GroupFunctionParser.java index a29e12d..750291e 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/GroupFunctionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/GroupFunctionParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -26,6 +24,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class GroupFunctionParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/HopParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/HopParser.java index 5a18b45..269373f 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/HopParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/HopParser.java @@ -16,14 +16,14 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; -import org.apache.rocketmq.streams.common.topology.model.IWindow; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.VarParseResult; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIntervalLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.configure.ConfigureFileKey; +import org.apache.rocketmq.streams.common.topology.model.IWindow; import org.apache.rocketmq.streams.window.builder.WindowBuilder; public class HopParser extends TumbleParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/IFFunctionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/IFFunctionParser.java index a6f3fbc..20f4c15 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/IFFunctionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/IFFunctionParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -26,6 +24,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class IFFunctionParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ItemParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ItemParser.java index 9446736..1568e22 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ItemParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ItemParser.java @@ -16,14 +16,15 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlNode; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlNode; + +import java.util.List; public class ItemParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/JsonConcatParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/JsonConcatParser.java index 90c38d8..515c5a7 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/JsonConcatParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/JsonConcatParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; @@ -25,6 +23,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class JsonConcatParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/LowerParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/LowerParser.java index 94699e8..70889f4 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/LowerParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/LowerParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -26,6 +24,8 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class LowerParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/OverWindowParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/OverWindowParser.java index fa4a1a8..4f0bae6 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/OverWindowParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/OverWindowParser.java @@ -16,22 +16,22 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.ArrayList; -import java.util.List; - -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; -import org.apache.rocketmq.streams.common.model.NameCreator; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.WindowBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.VarParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlWindow; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.rocketmq.streams.common.model.NameCreator; + +import java.util.ArrayList; +import java.util.List; public class OverWindowParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ScriptFunctionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ScriptFunctionParser.java index 32145ff..84f2cdd 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ScriptFunctionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/ScriptFunctionParser.java @@ -16,12 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.rocketmq.streams.filter.function.expression.ScriptFunction; -import org.apache.rocketmq.streams.common.utils.StringUtil; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -29,6 +23,12 @@ import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.common.utils.StringUtil; +import org.apache.rocketmq.streams.filter.function.expression.ScriptFunction; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class ScriptFunctionParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SessionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SessionParser.java index 38ee3e0..41f40e3 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SessionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SessionParser.java @@ -1,14 +1,14 @@ package com.alibaba.rsqldb.parser.parser.function; -import org.apache.calcite.avatica.util.TimeUnit; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlIntervalLiteral; -import org.apache.calcite.sql.SqlNode; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.WindowBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.VarParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIntervalLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.rocketmq.streams.window.operator.AbstractWindow; /** diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlAsOperatorParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlAsOperatorParser.java index 0f65ee1..caa53cd 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlAsOperatorParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlAsOperatorParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.NotSupportParseResult; @@ -29,6 +27,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.List; + public class SqlAsOperatorParser extends AbstractSelectNodeParser { private static final Log LOG = LogFactory.getLog(SqlAsOperatorParser.class); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlDataTypeSpecParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlDataTypeSpecParser.java index e140983..c859482 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlDataTypeSpecParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlDataTypeSpecParser.java @@ -16,12 +16,12 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import org.apache.rocketmq.streams.common.datatype.StringDataType; -import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.AbstractSqlParser; +import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.rocketmq.streams.common.datatype.StringDataType; public class SqlDataTypeSpecParser extends AbstractSqlParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlIndentifierParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlIndentifierParser.java index 45a25bc..dc1ee9d 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlIndentifierParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlIndentifierParser.java @@ -16,8 +16,8 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.AbstractSqlParser; +import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlLiteralParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlLiteralParser.java index 449b282..4697a7b 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlLiteralParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/SqlLiteralParser.java @@ -16,18 +16,19 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.ArrayList; -import java.util.List; -import org.apache.rocketmq.streams.common.datatype.DataType; -import org.apache.rocketmq.streams.common.utils.ContantsUtil; -import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.AbstractSqlParser; +import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.namecreator.ParserNameCreator; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; import com.alibaba.rsqldb.parser.util.SqlDataTypeUtil; import org.apache.calcite.sql.SqlLiteral; +import org.apache.rocketmq.streams.common.datatype.DataType; +import org.apache.rocketmq.streams.common.utils.ContantsUtil; + +import java.util.ArrayList; +import java.util.List; public class SqlLiteralParser extends AbstractSqlParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TransJsonArrayParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TransJsonArrayParser.java index 2ae17be..56ec291 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TransJsonArrayParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TransJsonArrayParser.java @@ -16,14 +16,14 @@ */ package com.alibaba.rsqldb.parser.parser.function; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlNode; +import java.util.List; + public class TransJsonArrayParser extends AbstractSelectNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TumbleParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TumbleParser.java index e012b87..1ed41b0 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TumbleParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/function/TumbleParser.java @@ -21,12 +21,12 @@ import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.VarParseResult; import com.alibaba.rsqldb.parser.parser.sqlnode.AbstractSelectNodeParser; -import org.apache.rocketmq.streams.window.operator.AbstractWindow; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIntervalLiteral; import org.apache.calcite.sql.SqlIntervalLiteral.IntervalValue; import org.apache.calcite.sql.SqlNode; +import org.apache.rocketmq.streams.window.operator.AbstractWindow; public class TumbleParser extends AbstractSelectNodeParser { @@ -83,7 +83,7 @@ protected int getWindowPeriod(SqlIntervalLiteral sqlIntervalLiteral) { //TODO default value int interval = 10; try { - interval = Integer.valueOf(intervalValue.getIntervalLiteral()); + interval = Integer.parseInt(intervalValue.getIntervalLiteral()); } catch (Exception e) { } @@ -97,11 +97,11 @@ protected static int getDiff2Minute(TimeUnit timeUnit) { case SECOND: return 1; case MINUTE: - return 1; - case DAY: - return 24 * 60; - case HOUR: return 60; + case HOUR: + return 60 * 60; + case DAY: + return 24 * 60 * 60; default: throw new RuntimeException("can not this time unit :" + timeUnit.toString() + ", support second,minute,houre,day, millsecond"); @@ -118,15 +118,16 @@ protected static int getDiff2Minute(TimeUnit timeUnit) { protected static int convert2Minute(int interval, TimeUnit timeUnit) { int tumblePeriod = interval; if (timeUnit != null) { - if (TimeUnit.MINUTE == timeUnit || TimeUnit.SECOND == timeUnit) { + if (TimeUnit.SECOND == timeUnit) { tumblePeriod = interval; - } else if (TimeUnit.HOUR == timeUnit) { + } else if (TimeUnit.MINUTE == timeUnit) { tumblePeriod = interval * 60; + } else if (TimeUnit.HOUR == timeUnit) { + tumblePeriod = interval * 60 * 60; } else if (TimeUnit.DAY == timeUnit) { - tumblePeriod = interval * 24 * 60; + tumblePeriod = interval * 24 * 60 * 60; } else { - throw new RuntimeException("can not this time unit :" + timeUnit.toString() - + ", support second,minute,houre,day, millsecond"); + throw new RuntimeException("can not this time unit :" + timeUnit + ", support second,minute,houre,day, millsecond"); } } return tumblePeriod; diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/ScriptParseResult.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/ScriptParseResult.java index 2c075ca..7a8baf0 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/ScriptParseResult.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/ScriptParseResult.java @@ -16,12 +16,12 @@ */ package com.alibaba.rsqldb.parser.parser.result; +import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; +import org.apache.rocketmq.streams.common.utils.StringUtil; + import java.util.ArrayList; import java.util.List; -import org.apache.rocketmq.streams.common.utils.StringUtil; -import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; - /** * 每个解析的返回结果。需要考虑变量，常量和脚本的情况 */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/SqlNodeParseResult.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/SqlNodeParseResult.java index 28c39ac..b2e8444 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/SqlNodeParseResult.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/result/SqlNodeParseResult.java @@ -16,17 +16,17 @@ */ package com.alibaba.rsqldb.parser.parser.result; +import com.alibaba.rsqldb.parser.parser.ISqlParser; +import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; +import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; +import org.apache.calcite.sql.SqlNode; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; -import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; -import com.alibaba.rsqldb.parser.parser.ISqlParser; -import org.apache.calcite.sql.SqlNode; - /** * 每个解析的返回结果。需要考虑变量，常量和脚本的情况 */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSelectNodeParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSelectNodeParser.java index 7e00a7b..2787193 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSelectNodeParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSelectNodeParser.java @@ -16,11 +16,11 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import org.apache.rocketmq.streams.common.datatype.StringDataType; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.ConstantParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; import com.alibaba.rsqldb.parser.parser.result.ScriptParseResult; +import org.apache.rocketmq.streams.common.datatype.StringDataType; public abstract class AbstractSelectNodeParser extends AbstractSqlNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSqlNodeParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSqlNodeParser.java index b6666fc..c87a8bc 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSqlNodeParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/AbstractSqlNodeParser.java @@ -16,8 +16,8 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.AbstractSqlParser; +import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; public abstract class AbstractSqlNodeParser extends AbstractSqlParser implements diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/CreateParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/CreateParser.java index 62abc2b..569e43e 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/CreateParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/CreateParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import java.util.List; - import com.alibaba.rsqldb.parser.parser.builder.CreateSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -27,6 +25,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.flink.sql.parser.ddl.SqlCreateTable; +import java.util.List; + /** * Create Table Parser */ diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/InsertParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/InsertParser.java index 29f3d5a..71e2305 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/InsertParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/InsertParser.java @@ -21,14 +21,15 @@ import com.alibaba.rsqldb.parser.parser.builder.InsertSQLBuilder; import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; -import java.util.ArrayList; -import java.util.List; import org.apache.calcite.sql.SqlEmit; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.rocketmq.streams.common.configure.StreamsConfigure; +import java.util.ArrayList; +import java.util.List; + public class InsertParser extends AbstractSqlNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/JoinParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/JoinParser.java index 4018a2d..467e882 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/JoinParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/JoinParser.java @@ -16,15 +16,13 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import java.util.List; - +import com.alibaba.rsqldb.parser.parser.ISqlParser; import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.JoinSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.LateralTableBuilder; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.TableNodeBuilder; -import com.alibaba.rsqldb.parser.parser.ISqlParser; import com.alibaba.rsqldb.parser.parser.function.SqlIndentifierParser; import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -37,6 +35,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.List; + public class JoinParser extends AbstractSqlNodeParser { private static final Log LOG = LogFactory.getLog(JoinParser.class); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/SelectParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/SelectParser.java index b237825..f2d9536 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/SelectParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/SelectParser.java @@ -16,17 +16,11 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import java.util.ArrayList; -import java.util.List; - -import org.apache.rocketmq.streams.common.utils.StringUtil; -import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder; -import org.apache.rocketmq.streams.filter.operator.expression.Expression; +import com.alibaba.rsqldb.parser.parser.ISqlParser; import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; import com.alibaba.rsqldb.parser.parser.builder.JoinSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.UnionSQLBuilder; -import com.alibaba.rsqldb.parser.parser.ISqlParser; import com.alibaba.rsqldb.parser.parser.function.HopParser; import com.alibaba.rsqldb.parser.parser.result.BuilderParseResult; import com.alibaba.rsqldb.parser.parser.result.IParseResult; @@ -38,6 +32,12 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.rocketmq.streams.common.utils.StringUtil; +import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder; +import org.apache.rocketmq.streams.filter.operator.expression.Expression; + +import java.util.ArrayList; +import java.util.List; public class SelectParser extends AbstractSelectNodeParser { diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/UnionParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/UnionParser.java index 390d618..3f8babc 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/UnionParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/UnionParser.java @@ -16,13 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.UnionSQLBuilder; @@ -33,6 +26,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + public class UnionParser extends AbstractSqlNodeParser { private static final Log LOG = LogFactory.getLog(UnionParser.class); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/ViewParser.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/ViewParser.java index e1866f7..26dae43 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/ViewParser.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/parser/sqlnode/ViewParser.java @@ -16,8 +16,6 @@ */ package com.alibaba.rsqldb.parser.parser.sqlnode; -import java.util.Set; - import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; import com.alibaba.rsqldb.parser.parser.builder.AbstractSQLBuilder; import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; @@ -27,6 +25,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.sql.parser.ddl.SqlCreateView; +import java.util.Set; + public class ViewParser extends AbstractSqlNodeParser { @Override diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/ColumnUtil.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/ColumnUtil.java index ba3f183..45f8045 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/ColumnUtil.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/ColumnUtil.java @@ -16,20 +16,21 @@ */ package com.alibaba.rsqldb.parser.util; -import java.util.List; +import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; +import com.alibaba.rsqldb.parser.parser.builder.CreateSQLBuilder; +import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.rocketmq.streams.common.datatype.DataType; -import org.apache.rocketmq.streams.common.datatype.StringDataType; -import org.apache.rocketmq.streams.common.metadata.MetaData; -import org.apache.rocketmq.streams.common.metadata.MetaDataField; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.flink.sql.parser.ddl.SqlTableColumn; -import com.alibaba.rsqldb.parser.parser.SQLNodeParserFactory; -import com.alibaba.rsqldb.parser.parser.builder.CreateSQLBuilder; -import com.alibaba.rsqldb.parser.parser.builder.SelectSQLBuilder; +import org.apache.rocketmq.streams.common.datatype.DataType; +import org.apache.rocketmq.streams.common.datatype.StringDataType; +import org.apache.rocketmq.streams.common.metadata.MetaData; +import org.apache.rocketmq.streams.common.metadata.MetaDataField; + +import java.util.List; public class ColumnUtil { private static final Log LOG = LogFactory.getLog(ColumnUtil.class); diff --git a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/SqlDataTypeUtil.java b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/SqlDataTypeUtil.java index bdc283b..586c7fe 100644 --- a/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/SqlDataTypeUtil.java +++ b/rsqldb-parser/src/main/java/com/alibaba/rsqldb/parser/util/SqlDataTypeUtil.java @@ -16,9 +16,6 @@ */ package com.alibaba.rsqldb.parser.util; -import java.util.HashMap; -import java.util.Map; - import org.apache.rocketmq.streams.common.datatype.BooleanDataType; import org.apache.rocketmq.streams.common.datatype.DataType; import org.apache.rocketmq.streams.common.datatype.DateDataType; @@ -29,6 +26,9 @@ import org.apache.rocketmq.streams.common.datatype.StringDataType; import org.apache.rocketmq.streams.script.utils.FunctionUtils; +import java.util.HashMap; +import java.util.Map; + public class SqlDataTypeUtil { private static Map dataTypeMap = new HashMap<>(); diff --git a/rsqldb-parser/src/main/resources/log4j.xml b/rsqldb-parser/src/main/resources/log4j.xml deleted file mode 100644 index 1e3786b..0000000 --- a/rsqldb-parser/src/main/resources/log4j.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/JoinWindowTest.java b/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/JoinWindowTest.java deleted file mode 100644 index 9d5b641..0000000 --- a/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/JoinWindowTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.rsqldb.parser.builder; - -import com.alibaba.fastjson.JSONObject; -import org.apache.rocketmq.streams.common.channel.source.systemmsg.NewSplitMessage; -import org.apache.rocketmq.streams.common.channel.source.systemmsg.RemoveSplitMessage; -import org.apache.rocketmq.streams.common.checkpoint.CheckPointMessage; -import org.apache.rocketmq.streams.common.context.AbstractContext; -import org.apache.rocketmq.streams.common.context.Context; -import org.apache.rocketmq.streams.common.context.IMessage; -import org.apache.rocketmq.streams.common.context.Message; -import org.apache.rocketmq.streams.common.topology.ChainStage; -import org.apache.rocketmq.streams.common.topology.model.IStageHandle; -import org.apache.rocketmq.streams.window.operator.join.JoinWindow; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -public class JoinWindowTest { - private static final String WINDOW_NAMESPACE = "chris_tmp_10"; - private static final String WINDOW_NAME = "tmp10"; - - @Test - public void testJoinWindow() throws InterruptedException { - Map> result = new ConcurrentHashMap<>(); - JoinWindow joinWindow = createJoinWindow(); - - joinWindow.setFireReceiver(new Receiver(new ChainStage() { - @Override - public void checkpoint(IMessage message, AbstractContext context, CheckPointMessage checkPointMessage) { - - } - - @Override - public void addNewSplit(IMessage message, AbstractContext context, NewSplitMessage newSplitMessage) { - - } - - @Override - public void removeSplit(IMessage message, AbstractContext context, RemoveSplitMessage removeSplitMessage) { - - } - - @Override - public boolean isAsyncNode() { - return false; - } - - @Override - protected IStageHandle selectHandle(IMessage iMessage, AbstractContext context) { - return null; - } - })); - for (int i = 0; i < 1; i++) { - batchAddMsg(joinWindow); - // joinWindow.run(); - Thread.sleep(150000); - } - - for (Map.Entry> tmp : result.entrySet()) { - for (IMessage msg : tmp.getValue()) { - System.out.println("joinwindow==" + tmp.getKey() + "==" + msg.getMessageBody().toJSONString()); - } - } - - Thread.sleep(100000000); - } - - private JoinWindow createJoinWindow() { - - JoinWindow window = new JoinWindow(); - window.setNameSpace(WINDOW_NAMESPACE); - window.setConfigureName(WINDOW_NAME); - window.setSlideInterval(5); - // window.setFireDelaySecond(5); - window.setSizeInterval(5); - window.setWaterMarkMinute(1); - window.setJoinType("INNER"); - List fields = new ArrayList<>(); - fields.add("name"); - window.setLeftJoinFieldNames(fields); - window.setRightJoinFieldNames(fields); - window.setExpression(null); - window.setRightAsName("a"); - window.setRetainWindowCount(4); - - window.toObject(window.toJson()); - window.init(); - return window; - } - - /** - * 插入测试数据 - * - * @param window - */ - protected void batchAddMsg(JoinWindow window) { - AtomicInteger count = new AtomicInteger(100); - for (int i = 0; i < 10; i++) { - JSONObject msg1 = new JSONObject(); - msg1.put("name", "chris" + i % 5); - msg1.put("age", 18); - msg1.put("time", System.currentTimeMillis()); - msg1.put("numid", count.incrementAndGet()); - Message message1 = new Message(msg1); - message1.getHeader().setQueueId(String.valueOf(System.currentTimeMillis())); - message1.getHeader().setOffset(i + "left"); - message1.getHeader().setMsgRouteFromLable("left"); - Context context1 = new Context(message1); - window.doMessage(message1, context1); - - JSONObject msg = new JSONObject(); - msg.put("name", "chris" + i % 5); - msg.put("age", 20); - msg.put("time", System.currentTimeMillis()); - msg.put("numid", count.incrementAndGet()); - Message message = new Message(msg); - message.getHeader().setQueueId(String.valueOf(System.currentTimeMillis())); - message.getHeader().setOffset(i + "right"); - message.getHeader().setMsgRouteFromLable("right"); - Context context = new Context(message); - window.doMessage(message, context); - } - - // for(int i=0;i<10;i++){ - // JSONObject msg=new JSONObject(); - // msg.put("name","chris" + i%5); - // msg.put("age",20); - // msg.put("time", System.currentTimeMillis()); - // msg.put("numid", count.incrementAndGet()); - // Message message=new Message(msg); - // message.getHeader().setQueueId(String.valueOf(System.currentTimeMillis())); - // message.getHeader().setOffset(i+"right"); - // message.getHeader().setMsgRouteFromLable("right"); - // Context context=new Context(message); - // window.doMessage(message,context); - // } - // window.flush(); - } - - public class Receiver extends ChainStage.PiplineRecieverAfterCurrentNode { - - public Receiver(ChainStage stage) { - stage.super(); - } - - @Override - public AbstractContext doMessage(IMessage message, AbstractContext context) { - System.out.println("************" + message.getMessageBody().toJSONString()); - return null; - } - - } -} diff --git a/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/SQLBuilderTest.java b/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/SQLBuilderTest.java deleted file mode 100644 index e8be05a..0000000 --- a/rsqldb-parser/src/test/java/com/alibaba/rsqldb/parser/builder/SQLBuilderTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.rsqldb.parser.builder; - -import com.alibaba.rsqldb.parser.entity.SqlTask; - -import org.apache.rocketmq.streams.common.utils.FileUtil; -import org.junit.Test; - -public class SQLBuilderTest { - @Test - public void testSQL() { - String sqlPath = "/Users/yuanxiaodong/Desktop/dipper_sql/aegis_proc/windows_proc_alert.sql"; - String sql = FileUtil.loadFileContentContainLineSign(sqlPath); - SqlTask sqlBuilder = new SqlTask("namespace", "name", sql); - sqlBuilder.startSQL(); - } - - @Test - public void testHomologousSQL() { - String sqlPath = "/Users/yuanxiaodong/Desktop/dipper_sql/aegis_proc/windows_proc_alert.sql"; - String sql = FileUtil.loadFileContentContainLineSign(sqlPath); - HomologousSQLBuilder sqlBuilder = new HomologousSQLBuilder("namespace", "name", sql, "aegis_proc"); - sqlBuilder.startSQL(); - } - - @Test - public void testStreamSQL() { - - HomologousSQLBuilder sqlBuilder = new HomologousSQLBuilder("namespace", "name", "1212", "aegis_proc"); - sqlBuilder.startSQL(); - - } - -} diff --git a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/RecoverAction.java b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/RecoverAction.java index 4b0b752..3a685d9 100644 --- a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/RecoverAction.java +++ b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/RecoverAction.java @@ -16,10 +16,6 @@ */ package org.apache.rsqldb.runner; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - import com.google.common.collect.Lists; import org.apache.rocketmq.streams.common.component.ComponentCreator; import org.apache.rocketmq.streams.common.topology.ChainPipeline; @@ -27,6 +23,10 @@ import org.apache.rocketmq.streams.common.topology.task.StreamsTask; import org.apache.rocketmq.streams.configurable.ConfigurableComponent; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + /** * @author junjie.cheng * @date 2021/11/25 diff --git a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/SqlAction.java b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/SqlAction.java index 4c331d7..fdda26a 100644 --- a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/SqlAction.java +++ b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/SqlAction.java @@ -17,8 +17,6 @@ package org.apache.rsqldb.runner; import com.alibaba.rsqldb.clients.sql.SQLStream; -import com.alibaba.rsqldb.parser.entity.SqlTask; - import org.apache.rocketmq.streams.common.utils.FileUtil; /** diff --git a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/StartAction.java b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/StartAction.java index 7502231..c760a47 100644 --- a/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/StartAction.java +++ b/rsqldb-runner/src/main/java/org/apache/rsqldb/runner/StartAction.java @@ -16,14 +16,7 @@ */ package org.apache.rsqldb.runner; -import java.io.File; -import java.util.List; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.stream.Collectors; - import com.alibaba.rsqldb.parser.entity.SqlTask; - import com.google.common.collect.Lists; import org.apache.rocketmq.streams.common.utils.FileUtil; import org.apache.rocketmq.streams.common.utils.PropertiesUtils; @@ -31,6 +24,12 @@ import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; +import java.io.File; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.stream.Collectors; + /** * @author junjie.cheng * @date 2021/11/19 diff --git a/rsqldb-runner/src/main/resources/dipper.properties b/rsqldb-runner/src/main/resources/dipper.properties index 4621090..4cd733e 100644 --- a/rsqldb-runner/src/main/resources/dipper.properties +++ b/rsqldb-runner/src/main/resources/dipper.properties @@ -1,6 +1,6 @@ ## checkpoint存储配置，可以是memory, DB 或者file， 除了checkpoint外， 任务序列化的内容也会被缓存在该存储 # dipper.configurable.service.type=memory - +window.shuffle.channel.type=memory ## 当checkpoint为DB时 # dipper.rds.jdbc.type= # dipper.rds.jdbc.url= @@ -11,10 +11,13 @@ ## 任务从存储反序列化的频次 # dipper.configurable.polling.time=60 #单位秒(s) - +pipeline.qps.print=true +homologous.expression.cache.size=100000 +homologous.pre.fingerprint.cache.size=100000 +homologous.optimization.switch=true ## 监控日志的相关配置 # dipper.monitor.output.level=INFO #日志等级，有三种INFO，SLOW，ERROR -# dipper.monitor.slow.timeout=60000 #慢查询超时时间 + dipper.monitor.slow.timeout=60000 # dipper.monitor.logs.dir=./logs #日志目录 ## 窗口配置 diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/BlinkDataType.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/BlinkDataType.java index 9907f18..0caed2b 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/BlinkDataType.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/BlinkDataType.java @@ -16,9 +16,9 @@ */ package com.alibaba.rsqldb.udf; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.rocketmq.streams.common.datatype.DataType; import org.apache.rocketmq.streams.common.utils.DataTypeUtil; -import org.apache.flink.api.common.typeinfo.TypeInformation; public class BlinkDataType { diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/FunctionUDFScript.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/FunctionUDFScript.java index 09ed9bb..3a8bbc9 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/FunctionUDFScript.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/FunctionUDFScript.java @@ -1,6 +1,5 @@ package com.alibaba.rsqldb.udf; -import java.io.File; import org.apache.flink.api.common.accumulators.DoubleCounter; import org.apache.flink.api.common.accumulators.Histogram; import org.apache.flink.api.common.accumulators.IntCounter; @@ -9,6 +8,8 @@ import org.apache.flink.table.functions.FunctionContext; import org.apache.rocketmq.streams.script.service.udf.UDFScript; +import java.io.File; + public class FunctionUDFScript extends UDFScript { protected transient FunctionContext functionContext = new FunctionContext() { diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udaf/BlinkUDAFScript.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udaf/BlinkUDAFScript.java index a2e2898..84db69b 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udaf/BlinkUDAFScript.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udaf/BlinkUDAFScript.java @@ -16,10 +16,66 @@ */ package com.alibaba.rsqldb.udf.udaf; +import java.io.File; + +import org.apache.flink.api.common.accumulators.DoubleCounter; +import org.apache.flink.api.common.accumulators.Histogram; +import org.apache.flink.api.common.accumulators.IntCounter; +import org.apache.flink.api.common.accumulators.LongCounter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.table.functions.FunctionContext; import org.apache.rocketmq.streams.script.service.udf.UDAFScript; public class BlinkUDAFScript extends UDAFScript { + protected transient FunctionContext functionContext = new FunctionContext() { + + @Override + public MetricGroup getMetricGroup() { + return null; + } + + @Override + public File getCachedFile(String name) { + return null; + } + + @Override + public int getNumberOfParallelSubtasks() { + return 0; + } + + @Override + public int getIndexOfThisSubtask() { + return 0; + } + + @Override + public IntCounter getIntCounter(String name) { + return null; + } + + @Override + public LongCounter getLongCounter(String name) { + return null; + } + + @Override + public DoubleCounter getDoubleCounter(String name) { + return null; + } + + @Override + public Histogram getHistogram(String name) { + return null; + } + + @Override + public String getJobParameter(String key, String defaultValue) { + return "0"; + } + }; + public BlinkUDAFScript() { this.accumulateMethodName = "accumulate"; this.createAccumulatorMethodName = "createAccumulator"; @@ -28,6 +84,7 @@ public BlinkUDAFScript() { this.mergeMethodName = "merge"; this.methodName = "eval"; this.initMethodName = "open"; + this.initParameters = new Object[] {functionContext}; } @Override diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udf/BlinkUDFScript.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udf/BlinkUDFScript.java index 836954e..5dfa5a2 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udf/BlinkUDFScript.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udf/BlinkUDFScript.java @@ -16,16 +16,15 @@ */ package com.alibaba.rsqldb.udf.udf; -import java.io.File; - -import org.apache.rocketmq.streams.script.service.udf.UDFScript; import org.apache.flink.api.common.accumulators.DoubleCounter; import org.apache.flink.api.common.accumulators.Histogram; import org.apache.flink.api.common.accumulators.IntCounter; import org.apache.flink.api.common.accumulators.LongCounter; - import org.apache.flink.metrics.MetricGroup; import org.apache.flink.table.functions.FunctionContext; +import org.apache.rocketmq.streams.script.service.udf.UDFScript; + +import java.io.File; public class BlinkUDFScript extends UDFScript { diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/BlinkUDTFScript.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/BlinkUDTFScript.java index e2262c5..14d345e 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/BlinkUDTFScript.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/BlinkUDTFScript.java @@ -17,14 +17,70 @@ package com.alibaba.rsqldb.udf.udtf; import com.alibaba.rsqldb.udf.udtf.collector.BlinkCollector; +import java.io.File; +import org.apache.flink.api.common.accumulators.DoubleCounter; +import org.apache.flink.api.common.accumulators.Histogram; +import org.apache.flink.api.common.accumulators.IntCounter; +import org.apache.flink.api.common.accumulators.LongCounter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.table.functions.FunctionContext; import org.apache.rocketmq.streams.script.service.udf.UDTFScript; import org.apache.flink.util.Collector; public class BlinkUDTFScript extends UDTFScript { + protected transient FunctionContext functionContext = new FunctionContext() { + + @Override + public MetricGroup getMetricGroup() { + return null; + } + + @Override + public File getCachedFile(String name) { + return null; + } + + @Override + public int getNumberOfParallelSubtasks() { + return 0; + } + + @Override + public int getIndexOfThisSubtask() { + return 0; + } + + @Override + public IntCounter getIntCounter(String name) { + return null; + } + + @Override + public LongCounter getLongCounter(String name) { + return null; + } + + @Override + public DoubleCounter getDoubleCounter(String name) { + return null; + } + + @Override + public Histogram getHistogram(String name) { + return null; + } + + @Override + public String getJobParameter(String key, String defaultValue) { + return "0"; + } + }; + public BlinkUDTFScript() { this.methodName = "eval"; this.initMethodName = "open"; + this.initParameters = new Object[] {functionContext}; this.setSetCollectorMethodName("setCollector"); } diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkCollector.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkCollector.java index 347833d..5c689e3 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkCollector.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkCollector.java @@ -18,14 +18,13 @@ import com.alibaba.fastjson.JSONObject; import com.alibaba.rsqldb.udf.udtf.BlinkUDTFScript; -import org.apache.rocketmq.streams.common.context.IMessage; - -import org.apache.rocketmq.streams.common.context.Message; -import org.apache.rocketmq.streams.script.context.FunctionContext; -import org.apache.rocketmq.streams.common.model.ThreadContext; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.apache.rocketmq.streams.common.context.IMessage; +import org.apache.rocketmq.streams.common.context.Message; +import org.apache.rocketmq.streams.common.model.ThreadContext; +import org.apache.rocketmq.streams.script.context.FunctionContext; import org.apache.rocketmq.streams.script.function.model.FunctionType; public class BlinkCollector implements Collector { diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkRowCollector.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkRowCollector.java index b642b7c..9961bf8 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkRowCollector.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkRowCollector.java @@ -19,22 +19,27 @@ import com.alibaba.fastjson.JSONObject; import com.alibaba.rsqldb.udf.BlinkDataType; import com.alibaba.rsqldb.udf.udtf.BlinkUDTFScript; - +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.types.DataTypes; +import org.apache.flink.table.types.RowType; +import org.apache.flink.table.types.TypeInfoWrappedType; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.apache.rocketmq.streams.common.context.AbstractContext; import org.apache.rocketmq.streams.common.context.IMessage; import org.apache.rocketmq.streams.common.context.Message; -import org.apache.rocketmq.streams.common.datatype.*; +import org.apache.rocketmq.streams.common.datatype.BooleanDataType; +import org.apache.rocketmq.streams.common.datatype.ByteDataType; import org.apache.rocketmq.streams.common.datatype.DataType; +import org.apache.rocketmq.streams.common.datatype.DateDataType; +import org.apache.rocketmq.streams.common.datatype.DoubleDataType; +import org.apache.rocketmq.streams.common.datatype.FloatDataType; +import org.apache.rocketmq.streams.common.datatype.IntDataType; +import org.apache.rocketmq.streams.common.datatype.LongDataType; +import org.apache.rocketmq.streams.common.datatype.ShortDataType; import org.apache.rocketmq.streams.common.datatype.StringDataType; import org.apache.rocketmq.streams.common.model.ThreadContext; import org.apache.rocketmq.streams.script.context.FunctionContext; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.table.types.DataTypes; -import org.apache.flink.table.types.RowType; -import org.apache.flink.table.types.TypeInfoWrappedType; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.apache.rocketmq.streams.script.function.model.FunctionType; import java.lang.reflect.InvocationTargetException; diff --git a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkTupleCollector.java b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkTupleCollector.java index 29b242a..45c5586 100644 --- a/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkTupleCollector.java +++ b/rsqldb-udf/src/main/java/com/alibaba/rsqldb/udf/udtf/collector/BlinkTupleCollector.java @@ -17,13 +17,12 @@ package com.alibaba.rsqldb.udf.udtf.collector; import com.alibaba.fastjson.JSONObject; - +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.util.Collector; import org.apache.rocketmq.streams.common.context.IMessage; import org.apache.rocketmq.streams.common.context.Message; -import org.apache.rocketmq.streams.script.context.FunctionContext; import org.apache.rocketmq.streams.common.model.ThreadContext; -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.util.Collector; +import org.apache.rocketmq.streams.script.context.FunctionContext; import org.apache.rocketmq.streams.script.function.model.FunctionType; public class BlinkTupleCollector implements Collector { diff --git a/rsqldb-udf/src/test/resources/log4j.xml b/rsqldb-udf/src/test/resources/log4j.xml deleted file mode 100755 index 7812fe7..0000000 --- a/rsqldb-udf/src/test/resources/log4j.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file