diff --git a/.repo/pom.template.xml b/.repo/pom.template.xml index 62a2a52c..d9ed694e 100644 --- a/.repo/pom.template.xml +++ b/.repo/pom.template.xml @@ -33,6 +33,11 @@ mlsql-cli byzer-yaml-visualization byzer-openmldb + byzer-objectstore-cos + byzer-objectstore-obs + byzer-objectstore-oss + byzer-objectstore-blob + byzer-objectstore-s3 byzer-xgboost MLSQL Plugins @@ -75,18 +80,16 @@ {{spark_version}} {{spark_binary_version}} {{spark_binary_version}} - + 3.2.13 + 2.4.0-SNAPSHOT + 0.4.0 16.0 4.5.3 - - 0.3.9.4 - 2.0.5 + 0.3.9.5 + 2.0.9 provided 2.6.5 - 2.3.0-SNAPSHOT - 0.2.0 0.8.0 - 3.0.3 @@ -128,7 +131,7 @@ tech.mlsql - streamingpro-spark-3.0.0-adaptor_${scala.binary.version} + streamingpro-spark-${spark.version}-adaptor_${scala.binary.version} ${mlsql.version} ${scope} @@ -208,24 +211,6 @@ - - spark-2.4 - - 2.4.3 - 2.4 - 2.4 - - - - - spark-3.0 - - 3.0.0-preview2 - 3.0 - 3.0 - - - disable-java8-doclint diff --git a/binlog2delta/.repo/desc.template.plugin b/binlog2delta/.repo/desc.template.plugin index da247052..8014a447 100644 --- a/binlog2delta/.repo/desc.template.plugin +++ b/binlog2delta/.repo/desc.template.plugin @@ -3,6 +3,7 @@ version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/binlog2delta -scala_version=2.11 +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} mlsqlPluginType=script desc=wow diff --git a/binlog2delta/.repo/pom.template.xml b/binlog2delta/.repo/pom.template.xml index 476f3860..1f8d4c2f 100644 --- a/binlog2delta/.repo/pom.template.xml +++ b/binlog2delta/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - binlog2delta_${scala.binary.version} + binlog2delta-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/binlog2delta/desc.plugin b/binlog2delta/desc.plugin index fda95e19..06d32d39 100644 --- a/binlog2delta/desc.plugin +++ b/binlog2delta/desc.plugin @@ -3,6 +3,7 @@ version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/binlog2delta -scala_version=2.11 +scala_version=2.12 +spark_version=3.3 mlsqlPluginType=script desc=wow \ No newline at end of file diff --git a/binlog2delta/pom.xml b/binlog2delta/pom.xml index 7fb218d9..3037d0d2 100644 --- a/binlog2delta/pom.xml +++ b/binlog2delta/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - binlog2delta_${scala.binary.version} + binlog2delta-3.3_2.12 \ No newline at end of file diff --git a/byzer-exception-render/.repo/desc.template.plugin b/byzer-exception-render/.repo/desc.template.plugin index 30132f30..013d49e3 100644 --- a/byzer-exception-render/.repo/desc.template.plugin +++ b/byzer-exception-render/.repo/desc.template.plugin @@ -4,6 +4,7 @@ version=0.1.0-SNAPSHOT author=jiachuan.zhu mlsqlVersions=">=2.0.0" githubUrl=https://github.com/byzer-org/byzer-extension/tree/master/byzer-exception-render -scala_version=2.11 +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} mlsqlPluginType=script desc=Renders Byzer-lang /run/script api's exception diff --git a/byzer-exception-render/.repo/pom.template.xml b/byzer-exception-render/.repo/pom.template.xml index 047858f9..35408712 100644 --- a/byzer-exception-render/.repo/pom.template.xml +++ b/byzer-exception-render/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - byzer-exception-render-${spark.binary.version}_${scala.binary.version} + byzer-exception-render-{{spark_binary_version}}_{{scala_binary_version}} 8 8 @@ -19,7 +19,7 @@ tech.mlsql - byzer-extension-tests-3.0_2.12 + byzer-extension-tests-{{spark_binary_version}}_{{scala_binary_version}} 0.1.0-SNAPSHOT test diff --git a/byzer-exception-render/desc.plugin b/byzer-exception-render/desc.plugin index ff4f8691..f669e35f 100644 --- a/byzer-exception-render/desc.plugin +++ b/byzer-exception-render/desc.plugin @@ -1,9 +1,10 @@ -moduleName=byzer-exception-render-3.0_2.12 +moduleName=byzer-exception-render-3.3_2.12 mainClass=tech.mlsql.plugins.render.ExceptionReplaceRenderApp version=0.1.0-SNAPSHOT author=jiachuan.zhu mlsqlVersions=">=2.0.0" githubUrl=https://github.com/byzer-org/byzer-extension/tree/master/byzer-exception-render -scala_version=2.11 +scala_version=2.12 +spark_version=3.3 mlsqlPluginType=script desc=Renders Byzer-lang /run/script api's exception \ No newline at end of file diff --git a/byzer-exception-render/pom.xml b/byzer-exception-render/pom.xml index 6a2922f0..23daa1f2 100644 --- a/byzer-exception-render/pom.xml +++ b/byzer-exception-render/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - byzer-exception-render-${spark.binary.version}_${scala.binary.version} + byzer-exception-render-3.3_2.12 8 8 @@ -19,7 +19,7 @@ tech.mlsql - byzer-extension-tests-3.0_2.12 + byzer-extension-tests-3.3_2.12 0.1.0-SNAPSHOT test diff --git a/byzer-exception-render/src/main/scala/tech/mlsql/plugins/render/ExceptionReplaceRender.scala b/byzer-exception-render/src/main/scala/tech/mlsql/plugins/render/ExceptionReplaceRender.scala index 36f7552a..1af47922 100644 --- a/byzer-exception-render/src/main/scala/tech/mlsql/plugins/render/ExceptionReplaceRender.scala +++ b/byzer-exception-render/src/main/scala/tech/mlsql/plugins/render/ExceptionReplaceRender.scala @@ -5,7 +5,7 @@ import org.apache.spark.streaming.SparkOperationUtil import streaming.core.strategy.platform.PlatformManager import streaming.dsl.ScriptSQLExec import tech.mlsql.app.ExceptionRender -import tech.mlsql.common.JsonUtils +import tech.mlsql.common.utils.serder.json.JsonUtils import tech.mlsql.common.utils.log.Logging import tech.mlsql.log.LogUtils import tech.mlsql.plugins.render.ExceptionReplaceTemplate.getMsgTemplate diff --git a/byzer-extension-tests/desc.plugin b/byzer-extension-tests/desc.plugin index 5a89d482..bb5d13cf 100644 --- a/byzer-extension-tests/desc.plugin +++ b/byzer-extension-tests/desc.plugin @@ -1,7 +1,7 @@ -moduleName=byzer-extension-tests-3.0 +moduleName=byzer-extension-tests-3.3 mainClass=tech.mlsql.plugins.ext.ets.app.MLSQLETApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/byzer-extension-tests/pom.xml b/byzer-extension-tests/pom.xml index 209b9012..52b621ba 100644 --- a/byzer-extension-tests/pom.xml +++ b/byzer-extension-tests/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - byzer-extension-tests-3.0_2.12 + byzer-extension-tests-3.3_2.12 1.2.32 diff --git a/byzer-hive312/.repo/desc.template.plugin b/byzer-hive312/.repo/desc.template.plugin new file mode 100644 index 00000000..2aa0743a --- /dev/null +++ b/byzer-hive312/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-hive312-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.hive.ByzerHive312 +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/byzer-hive312 +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-hive312/.repo/pom.template.xml b/byzer-hive312/.repo/pom.template.xml new file mode 100644 index 00000000..26d0b8b1 --- /dev/null +++ b/byzer-hive312/.repo/pom.template.xml @@ -0,0 +1,81 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-hive312-{{spark_binary_version}}_{{scala_binary_version}} + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.jetty + + + com.fasterxml.jackson.core + shadeio.com.fasterxml.jackson.core + + + com.google.guava + shadeio.azure.com.google.guava + + + org.apache.commons + shadeio.azure.org.apache.commons + + + + com.fasterxml.jackson.databind + shadeio.com.fasterxml.jackson.databind + + + commons-lang + shadeio.azure.commons-lang + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-hive312/desc.plugin b/byzer-hive312/desc.plugin new file mode 100644 index 00000000..0303985a --- /dev/null +++ b/byzer-hive312/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-hive312-3.3 +mainClass=tech.mlsql.plugins.hive.ByzerHive312 +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/byzer-hive312 +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-hive312/pom.xml b/byzer-hive312/pom.xml new file mode 100644 index 00000000..292c31d5 --- /dev/null +++ b/byzer-hive312/pom.xml @@ -0,0 +1,81 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-hive312-3.3_2.12 + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.jetty + + + com.fasterxml.jackson.core + shadeio.com.fasterxml.jackson.core + + + com.google.guava + shadeio.azure.com.google.guava + + + org.apache.commons + shadeio.azure.org.apache.commons + + + + com.fasterxml.jackson.databind + shadeio.com.fasterxml.jackson.databind + + + commons-lang + shadeio.azure.commons-lang + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-hive312/src/main/java/tech/mlsql/plugins/hive/ByzerHive312.scala b/byzer-hive312/src/main/java/tech/mlsql/plugins/hive/ByzerHive312.scala new file mode 100644 index 00000000..fd869b80 --- /dev/null +++ b/byzer-hive312/src/main/java/tech/mlsql/plugins/hive/ByzerHive312.scala @@ -0,0 +1,8 @@ +package tech.mlsql.plugins.hive + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerHive312 { + +} diff --git a/byzer-objectstore-blob/.repo/desc.template.plugin b/byzer-objectstore-blob/.repo/desc.template.plugin new file mode 100644 index 00000000..f80251df --- /dev/null +++ b/byzer-objectstore-blob/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-objectstore-blob-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.objectstore.blob.ByzerAzureBlob +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-blob +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-objectstore-blob/.repo/pom.template.xml b/byzer-objectstore-blob/.repo/pom.template.xml new file mode 100644 index 00000000..68b3a753 --- /dev/null +++ b/byzer-objectstore-blob/.repo/pom.template.xml @@ -0,0 +1,96 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-blob-{{spark_binary_version}}_{{scala_binary_version}} + + + org.apache.hadoop + hadoop-azure + 3.2.0 + + + com.fasterxml.jackson.core + jackson-core + 2.9.5 + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.5 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.jetty + + + com.fasterxml.jackson.core + shadeio.com.fasterxml.jackson.core + + + com.google.guava + shadeio.azure.com.google.guava + + + org.apache.commons + shadeio.azure.org.apache.commons + + + + com.fasterxml.jackson.databind + shadeio.com.fasterxml.jackson.databind + + + commons-lang + shadeio.azure.commons-lang + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-blob/desc.plugin b/byzer-objectstore-blob/desc.plugin new file mode 100644 index 00000000..06b1023b --- /dev/null +++ b/byzer-objectstore-blob/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-objectstore-blob-3.3 +mainClass=tech.mlsql.plugins.objectstore.blob.ByzerAzureBlob +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-blob +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-objectstore-blob/pom.xml b/byzer-objectstore-blob/pom.xml new file mode 100644 index 00000000..a643e196 --- /dev/null +++ b/byzer-objectstore-blob/pom.xml @@ -0,0 +1,96 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-blob-3.3_2.12 + + + org.apache.hadoop + hadoop-azure + 3.2.0 + + + com.fasterxml.jackson.core + jackson-core + 2.9.5 + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.5 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.jetty + + + com.fasterxml.jackson.core + shadeio.com.fasterxml.jackson.core + + + com.google.guava + shadeio.azure.com.google.guava + + + org.apache.commons + shadeio.azure.org.apache.commons + + + + com.fasterxml.jackson.databind + shadeio.com.fasterxml.jackson.databind + + + commons-lang + shadeio.azure.commons-lang + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-blob/src/main/java/tech/mlsql/plugins/objectstore/blob/ByzerAzureBlob.scala b/byzer-objectstore-blob/src/main/java/tech/mlsql/plugins/objectstore/blob/ByzerAzureBlob.scala new file mode 100644 index 00000000..6e77d517 --- /dev/null +++ b/byzer-objectstore-blob/src/main/java/tech/mlsql/plugins/objectstore/blob/ByzerAzureBlob.scala @@ -0,0 +1,29 @@ +package tech.mlsql.plugins.objectstore.blob + +import tech.mlsql.common.utils.log.Logging +import tech.mlsql.version.VersionCompatibility + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerAzureBlob extends tech.mlsql.app.App with VersionCompatibility with Logging { + override def run(args: Seq[String]): Unit = { + try { + Class.forName("org.apache.hadoop.fs.azure.NativeAzureFileSystem") + logInfo("Success to load jar for native Azure FileSystem ") + } catch { + case e: Exception => + logError("Fail to load jar for native Azure FileSystem") + } + + } + + + override def supportedVersions: Seq[String] = { + ByzerAzureBlob.versions + } +} + +object ByzerAzureBlob { + val versions = Seq(">=2.0.1") +} \ No newline at end of file diff --git a/byzer-objectstore-cos/.repo/desc.template.plugin b/byzer-objectstore-cos/.repo/desc.template.plugin new file mode 100644 index 00000000..eb3134a7 --- /dev/null +++ b/byzer-objectstore-cos/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-objectstore-cos-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.objectstore.cos.ByzerCos +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-cos +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-objectstore-cos/.repo/pom.template.xml b/byzer-objectstore-cos/.repo/pom.template.xml new file mode 100644 index 00000000..4505fd8a --- /dev/null +++ b/byzer-objectstore-cos/.repo/pom.template.xml @@ -0,0 +1,76 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-cos-{{spark_binary_version}}_{{scala_binary_version}} + + + org.apache.hadoop + hadoop-cos + 3.3.4 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.apache.poi + shadeio.poi + + + com.norbitltd.spoiwo + shadeio.spoiwo + + + com.github.pjfanning + shadeio.pjfanning + + + org.apache.commons.compress + shadeio.commons.compress + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-cos/README.md b/byzer-objectstore-cos/README.md new file mode 100644 index 00000000..6072c8fb --- /dev/null +++ b/byzer-objectstore-cos/README.md @@ -0,0 +1,26 @@ +# Byzer-objectstore-cos + +## Install + +``` +!plugin app install - "byzer-objectstore-cos-3.3" +``` + +## Uninstall + +``` +!plugin app remove "byzer-objectstore-cos-3.3" +``` + +## Usage + +``` +load FS.`cosn://xxxx-xxxx/test/a.parquet` +where `fs.cosn.impl`="org.apache.hadoop.fs.CosFileSystem" +and `fs.AbstractFileSystem.cosn.impl`="org.apache.hadoop.fs.CosN" +and `fs.cosn.bucket.endpoint_suffix`="cos.ap-xxx.myqcloud.com" +and `fs.cosn.userinfo.secretId`="xxxx" +and `fs.cosn.userinfo.secretKey`="xxxxx" +and `fs.cosn.tmp.dir`="/tmp/hadoop_cos" +and implClass="parquet" as output; +``` \ No newline at end of file diff --git a/byzer-objectstore-cos/desc.plugin b/byzer-objectstore-cos/desc.plugin new file mode 100644 index 00000000..8a12eaed --- /dev/null +++ b/byzer-objectstore-cos/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-objectstore-cos-3.3 +mainClass=tech.mlsql.plugins.objectstore.cos.ByzerCos +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-cos +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-objectstore-cos/pom.xml b/byzer-objectstore-cos/pom.xml new file mode 100644 index 00000000..566d5b93 --- /dev/null +++ b/byzer-objectstore-cos/pom.xml @@ -0,0 +1,76 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-cos-3.3_2.12 + + + org.apache.hadoop + hadoop-cos + 3.3.4 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.apache.poi + shadeio.poi + + + com.norbitltd.spoiwo + shadeio.spoiwo + + + com.github.pjfanning + shadeio.pjfanning + + + org.apache.commons.compress + shadeio.commons.compress + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-cos/src/main/java/tech/mlsql/plugins/objectstore/cos/ByzerCos.scala b/byzer-objectstore-cos/src/main/java/tech/mlsql/plugins/objectstore/cos/ByzerCos.scala new file mode 100644 index 00000000..d2b3eb86 --- /dev/null +++ b/byzer-objectstore-cos/src/main/java/tech/mlsql/plugins/objectstore/cos/ByzerCos.scala @@ -0,0 +1,29 @@ +package tech.mlsql.plugins.objectstore.cos + +import tech.mlsql.common.utils.log.Logging +import tech.mlsql.version.VersionCompatibility + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerCos extends tech.mlsql.app.App with VersionCompatibility with Logging { + override def run(args: Seq[String]): Unit = { + try { + Class.forName("org.apache.hadoop.fs.CosFileSystem") + logInfo("Success to load jar for Cos FileSystem ") + } catch { + case e: Exception => + logError("Fail to load jar for Cos FileSystem") + } + + } + + + override def supportedVersions: Seq[String] = { + ByzerCos.versions + } +} + +object ByzerCos { + val versions = Seq(">=2.0.1") +} \ No newline at end of file diff --git a/byzer-objectstore-obs/.repo/desc.template.plugin b/byzer-objectstore-obs/.repo/desc.template.plugin new file mode 100644 index 00000000..7016dce6 --- /dev/null +++ b/byzer-objectstore-obs/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-objectstore-obs-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.objectstore.obs.ByzerObs +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-obs +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-objectstore-obs/.repo/pom.template.xml b/byzer-objectstore-obs/.repo/pom.template.xml new file mode 100644 index 00000000..ffb3efe3 --- /dev/null +++ b/byzer-objectstore-obs/.repo/pom.template.xml @@ -0,0 +1,76 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-obs-{{spark_binary_version}}_{{scala_binary_version}} + + + + + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.apache.poi + shadeio.poi + + + com.norbitltd.spoiwo + shadeio.spoiwo + + + com.github.pjfanning + shadeio.pjfanning + + + org.apache.commons.compress + shadeio.commons.compress + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-obs/desc.plugin b/byzer-objectstore-obs/desc.plugin new file mode 100644 index 00000000..7213deeb --- /dev/null +++ b/byzer-objectstore-obs/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-objectstore-obs-3.3 +mainClass=tech.mlsql.plugins.objectstore.obs.ByzerObs +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-obs +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-objectstore-obs/pom.xml b/byzer-objectstore-obs/pom.xml new file mode 100644 index 00000000..e870519c --- /dev/null +++ b/byzer-objectstore-obs/pom.xml @@ -0,0 +1,76 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-obs-3.3_2.12 + + + + + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.apache.poi + shadeio.poi + + + com.norbitltd.spoiwo + shadeio.spoiwo + + + com.github.pjfanning + shadeio.pjfanning + + + org.apache.commons.compress + shadeio.commons.compress + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-obs/src/main/java/tech/mlsql/plugins/objectstore/obs/ByzerObs.scala b/byzer-objectstore-obs/src/main/java/tech/mlsql/plugins/objectstore/obs/ByzerObs.scala new file mode 100644 index 00000000..7641c3f9 --- /dev/null +++ b/byzer-objectstore-obs/src/main/java/tech/mlsql/plugins/objectstore/obs/ByzerObs.scala @@ -0,0 +1,29 @@ +package tech.mlsql.plugins.objectstore.obs + +import tech.mlsql.common.utils.log.Logging +import tech.mlsql.version.VersionCompatibility + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerObs extends tech.mlsql.app.App with VersionCompatibility with Logging { + override def run(args: Seq[String]): Unit = { + try { + Class.forName("org.apache.hadoop.fs.obs.OBSFileSystem") + logInfo("Success to load jar for OBS FileSystem ") + } catch { + case e: Exception => + logError("Fail to load jar for OBS FileSystem") + } + + } + + + override def supportedVersions: Seq[String] = { + ByzerObs.versions + } +} + +object ByzerObs { + val versions = Seq(">=2.0.1") +} \ No newline at end of file diff --git a/byzer-objectstore-oss/.repo/desc.template.plugin b/byzer-objectstore-oss/.repo/desc.template.plugin new file mode 100644 index 00000000..91b47f3c --- /dev/null +++ b/byzer-objectstore-oss/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-objectstore-oss-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.objectstore.oss.ByzerAliyunOSS +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-oss +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-objectstore-oss/.repo/pom.template.xml b/byzer-objectstore-oss/.repo/pom.template.xml new file mode 100644 index 00000000..1b2acbf5 --- /dev/null +++ b/byzer-objectstore-oss/.repo/pom.template.xml @@ -0,0 +1,81 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-oss-{{spark_binary_version}}_{{scala_binary_version}} + + + org.apache.hadoop + hadoop-aliyun + 3.2.4 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.oss.jetty + + + com.fasterxml.jackson.core + shadeio.oss.com.fasterxml.jackson.core + + + + + + + + com.fasterxml.jackson.databind + shadeio.oss.com.fasterxml.jackson.databind + + + org.apache.httpcomponents + shadeio.oss.org.apache.httpcomponents + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-oss/README.md b/byzer-objectstore-oss/README.md new file mode 100644 index 00000000..46ec0fcf --- /dev/null +++ b/byzer-objectstore-oss/README.md @@ -0,0 +1,26 @@ +# Byzer-objectstore-oss +Aliyun OSS +## Install + +``` +!plugin app install - "byzer-objectstore-oss-3.3" +``` + +## Uninstall + +``` +!plugin app remove "byzer-objectstore-oss-3.3" +``` + +## Usage + +``` +include lib.`github.com/allwefantasy/byzer-objectstore` +where alias="objectstore"; + +include local.`objectstore.oss.mod` +where ossID = "xxxx" and ossKey="xxxxx"; + +load FS.`oss://xxxxx-1255000012/test/a.parquet` +where implClass="parquet" as output; +``` \ No newline at end of file diff --git a/byzer-objectstore-oss/desc.plugin b/byzer-objectstore-oss/desc.plugin new file mode 100644 index 00000000..939e5905 --- /dev/null +++ b/byzer-objectstore-oss/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-objectstore-oss-3.3 +mainClass=tech.mlsql.plugins.objectstore.oss.ByzerAliyunOSS +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-oss +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-objectstore-oss/pom.xml b/byzer-objectstore-oss/pom.xml new file mode 100644 index 00000000..5416b594 --- /dev/null +++ b/byzer-objectstore-oss/pom.xml @@ -0,0 +1,81 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-oss-3.3_2.12 + + + org.apache.hadoop + hadoop-aliyun + 3.2.4 + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + org.eclipse.jetty + shadeio.oss.jetty + + + com.fasterxml.jackson.core + shadeio.oss.com.fasterxml.jackson.core + + + + + + + + com.fasterxml.jackson.databind + shadeio.oss.com.fasterxml.jackson.databind + + + org.apache.httpcomponents + shadeio.oss.org.apache.httpcomponents + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-oss/src/main/java/tech/mlsql/plugins/objectstore/oss/ByzerAliyunOSS.scala b/byzer-objectstore-oss/src/main/java/tech/mlsql/plugins/objectstore/oss/ByzerAliyunOSS.scala new file mode 100644 index 00000000..3b5c7c2d --- /dev/null +++ b/byzer-objectstore-oss/src/main/java/tech/mlsql/plugins/objectstore/oss/ByzerAliyunOSS.scala @@ -0,0 +1,30 @@ +package tech.mlsql.plugins.objectstore.oss + +import tech.mlsql.common.utils.log.Logging +import tech.mlsql.version.VersionCompatibility + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerAliyunOSS extends tech.mlsql.app.App with VersionCompatibility with Logging { + override def run(args: Seq[String]): Unit = { + try { + Class.forName("org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem") + logInfo("V3:Success to load jar for Aliyun OSS FileSystem ") + + } catch { + case e: Exception => + logError("Fail to load jar for Aliyun OSS FileSystem") + } + + } + + + override def supportedVersions: Seq[String] = { + ByzerAliyunOSS.versions + } +} + +object ByzerAliyunOSS { + val versions = Seq(">=2.0.1") +} \ No newline at end of file diff --git a/byzer-objectstore-s3/.repo/desc.template.plugin b/byzer-objectstore-s3/.repo/desc.template.plugin new file mode 100644 index 00000000..a7376e12 --- /dev/null +++ b/byzer-objectstore-s3/.repo/desc.template.plugin @@ -0,0 +1,16 @@ +moduleName=byzer-objectstore-s3-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.objectstore.s3.ByzerAwsS3 +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-s3 +mlsqlPluginType=app +desc=mlsql-shell + + + + + + diff --git a/byzer-objectstore-s3/.repo/pom.template.xml b/byzer-objectstore-s3/.repo/pom.template.xml new file mode 100644 index 00000000..31d24ebf --- /dev/null +++ b/byzer-objectstore-s3/.repo/pom.template.xml @@ -0,0 +1,82 @@ + + + + mlsql-plugins-{{spark_binary_version}}_{{scala_binary_version}} + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-s3-{{spark_binary_version}}_{{scala_binary_version}} + + + org.apache.hadoop + hadoop-aws + 3.3.2 + + + org.apache.hadoop + hadoop-common + 3.3.2 + + + + + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + com.fasterxml.jackson.core + shadeio.s3.com.fasterxml.jackson.core + + + com.fasterxml.jackson.databind + shadeio.s3.com.fasterxml.jackson.databind + + + com.thoughtworks.paranamer + shadeio.s3.com.thoughtworks.paranamer + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-s3/desc.plugin b/byzer-objectstore-s3/desc.plugin new file mode 100644 index 00000000..f9aa3aa1 --- /dev/null +++ b/byzer-objectstore-s3/desc.plugin @@ -0,0 +1,15 @@ +moduleName=byzer-objectstore-s3-3.3 +mainClass=tech.mlsql.plugins.objectstore.s3.ByzerAwsS3 +scala_version=2.12 +spark_version=3.3 +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/mlsql-objectstore-s3 +mlsqlPluginType=app +desc=mlsql-shell + + + + + diff --git a/byzer-objectstore-s3/pom.xml b/byzer-objectstore-s3/pom.xml new file mode 100644 index 00000000..b289c298 --- /dev/null +++ b/byzer-objectstore-s3/pom.xml @@ -0,0 +1,82 @@ + + + + mlsql-plugins-3.3_2.12 + tech.mlsql + 0.1.0-SNAPSHOT + + 4.0.0 + + byzer-objectstore-s3-3.3_2.12 + + + org.apache.hadoop + hadoop-aws + 3.3.2 + + + org.apache.hadoop + hadoop-common + 3.3.2 + + + + + + + + + + + shade + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + false + + + com.fasterxml.jackson.core + shadeio.s3.com.fasterxml.jackson.core + + + com.fasterxml.jackson.databind + shadeio.s3.com.fasterxml.jackson.databind + + + com.thoughtworks.paranamer + shadeio.s3.com.thoughtworks.paranamer + + + + + + + package + + shade + + + + + + + + + + \ No newline at end of file diff --git a/byzer-objectstore-s3/src/main/java/tech/mlsql/plugins/objectstore/s3/ByzerAwsS3.scala b/byzer-objectstore-s3/src/main/java/tech/mlsql/plugins/objectstore/s3/ByzerAwsS3.scala new file mode 100644 index 00000000..f6b05852 --- /dev/null +++ b/byzer-objectstore-s3/src/main/java/tech/mlsql/plugins/objectstore/s3/ByzerAwsS3.scala @@ -0,0 +1,29 @@ +package tech.mlsql.plugins.objectstore.s3 + +import tech.mlsql.common.utils.log.Logging +import tech.mlsql.version.VersionCompatibility + +/** + * 23/9/2022 WilliamZhu(allwefantasy@gmail.com) + */ +class ByzerAwsS3 extends tech.mlsql.app.App with VersionCompatibility with Logging { + override def run(args: Seq[String]): Unit = { + try { + Class.forName("org.apache.hadoop.fs.s3a.S3AFileSystem") + logInfo("Success to load jar for Aliyun OSS FileSystem ") + } catch { + case e: Exception => + logError("Fail to load jar for Aliyun OSS FileSystem") + } + + } + + + override def supportedVersions: Seq[String] = { + ByzerAwsS3.versions + } +} + +object ByzerAwsS3 { + val versions = Seq(">=2.0.1") +} \ No newline at end of file diff --git a/byzer-openmldb/desc.plugin b/byzer-openmldb/desc.plugin index 5bf8a0c0..fa3a6f9e 100644 --- a/byzer-openmldb/desc.plugin +++ b/byzer-openmldb/desc.plugin @@ -1,7 +1,7 @@ -moduleName=byzer-openmldb-3.0 +moduleName=byzer-openmldb-3.3 mainClass=tech.mlsql.plugins.openmldb.ByzerApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/byzer-openmldb/pom.xml b/byzer-openmldb/pom.xml index f4253990..cd6ed66c 100644 --- a/byzer-openmldb/pom.xml +++ b/byzer-openmldb/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - byzer-openmldb-3.0_2.12 + byzer-openmldb-3.3_2.12 1.2.32 diff --git a/byzer-yaml-visualization/desc.plugin b/byzer-yaml-visualization/desc.plugin index 76d91394..3b1a31f1 100644 --- a/byzer-yaml-visualization/desc.plugin +++ b/byzer-yaml-visualization/desc.plugin @@ -1,7 +1,7 @@ -moduleName=byzer-yaml-visualization-3.0 +moduleName=byzer-yaml-visualization-3.3 mainClass=tech.mlsql.plugins.visualization.ByzerVisualizationApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/byzer-yaml-visualization/pom.xml b/byzer-yaml-visualization/pom.xml index 36bb476b..6053f0b7 100644 --- a/byzer-yaml-visualization/pom.xml +++ b/byzer-yaml-visualization/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - byzer-yaml-visualization-3.0_2.12 + byzer-yaml-visualization-3.3_2.12 1.2.32 diff --git a/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/FigTest.scala b/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/FigTest.scala index 7b2b29bb..14337d10 100644 --- a/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/FigTest.scala +++ b/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/FigTest.scala @@ -1,12 +1,11 @@ package tech.mlsql.plugins.visiualization.test - -import org.scalatest.FlatSpec +import org.scalatest.funsuite.AnyFunSuite import tech.mlsql.plugins.visualization.{PlotlyRuntime, PythonHint} /** * 2/7/2022 WilliamZhu(allwefantasy@gmail.com) */ -class FigTest extends FlatSpec { +class FigTest extends AnyFunSuite { def printCode(v: String) = { val pr = new PlotlyRuntime() @@ -36,11 +35,11 @@ class FigTest extends FlatSpec { | uv: "uv"""".stripMargin - "yaml" should "to python code" in { + test("yaml should to python code") { printCode(template) } - "yaml" should "parse Boolean" in { + test("yaml should parse Boolean") { printCode( """ @@ -59,7 +58,7 @@ class FigTest extends FlatSpec { | size_max: 60 """.stripMargin) } - "yaml" should "with vvtype" in { + test("yaml should with vvtype") { printCode( """ |confFrom: counties diff --git a/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/PyLangTest.scala b/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/PyLangTest.scala index 4f308181..a2bf1884 100644 --- a/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/PyLangTest.scala +++ b/byzer-yaml-visualization/src/test/java/tech/mlsql/plugins/visiualization/test/PyLangTest.scala @@ -1,29 +1,29 @@ package tech.mlsql.plugins.visiualization.test -import org.scalatest.FlatSpec +import org.scalatest.funsuite.AnyFunSuite import tech.mlsql.plugins.visualization.pylang.PyLang /** * 3/7/2022 WilliamZhu(allwefantasy@gmail.com) */ -class PyLangTest extends FlatSpec { - "assign" should "work" in { +class PyLangTest extends AnyFunSuite { + test("assign should work") { val pl = PyLang().let("v").bind.str("jack").end.end assert(pl.toScript == """v = "jack"""") } - "variable" should "invoke func" in { + test("variable should invoke func") { val pl = PyLang().let("ray_context").invokeFunc("to_pandas").end.end assert(pl.toScript == """ray_context.to_pandas()""") } - "variable" should "invoke func with new assignment" in { + test("variable should invoke func with new assignment") { val pl = PyLang().let("ray_context").invokeFunc("to_pandas").end. namedVariableName("df").end assert(pl.toScript == """df = ray_context.to_pandas()""") } - "variable" should "invoke object create with new assignment" in { + test("variable should invoke object create with new assignment") { val pl = PyLang().let("ray_context"). invokeObjectCreate.addImport("from pyjava.api.mlsql import RayContext,PythonContext"). create("PythonContext").extend("object"). @@ -36,7 +36,7 @@ class PyLangTest extends FlatSpec { |context = PythonContext()""".stripMargin) } - "func" should "right ident" in { + test("right ident") { val genCode = PyLang().let("rayContext").invokeFunc("to_pandas").end. namedVariableName("df").end. func("add"). @@ -48,7 +48,7 @@ class PyLangTest extends FlatSpec { | a = "s"""".stripMargin) } - "variable" should "with parameters" in { + test("with parameters") { val genCode = PyLang().let("rayContext").invokeFunc("to_pandas").params. add("day", None).addKV("y", "jack", Some("\"")). end. diff --git a/connect-persist/.repo/pom.template.xml b/connect-persist/.repo/pom.template.xml index 4d51fbe5..a83ad15f 100644 --- a/connect-persist/.repo/pom.template.xml +++ b/connect-persist/.repo/pom.template.xml @@ -9,6 +9,6 @@ 4.0.0 - connect-persist-${spark.binary.version}_${scala.binary.version} + connect-persist-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/connect-persist/desc.plugin b/connect-persist/desc.plugin index 2f9dc406..bb2334e0 100644 --- a/connect-persist/desc.plugin +++ b/connect-persist/desc.plugin @@ -1,7 +1,7 @@ -moduleName=connect-persist-app-3.0 +moduleName=connect-persist-app-3.3 mainClass=tech.mlsql.plugins.app.ConnectPersistApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/connect-persist/pom.xml b/connect-persist/pom.xml index 48861e75..a52a32a6 100644 --- a/connect-persist/pom.xml +++ b/connect-persist/pom.xml @@ -3,12 +3,12 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - connect-persist-${spark.binary.version}_${scala.binary.version} + connect-persist-3.3_2.12 \ No newline at end of file diff --git a/delta-enhancer/.repo/pom.template.xml b/delta-enhancer/.repo/pom.template.xml index 5e5055be..a9f2ca92 100644 --- a/delta-enhancer/.repo/pom.template.xml +++ b/delta-enhancer/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - delta-enhancer-${spark.binary.version}_${scala.binary.version} + delta-enhancer-{{spark_binary_version}}_{{scala_binary_version}} tech.mlsql diff --git a/delta-enhancer/pom.xml b/delta-enhancer/pom.xml index f11d9436..95e5d4cd 100644 --- a/delta-enhancer/pom.xml +++ b/delta-enhancer/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - delta-enhancer-${spark.binary.version}_${scala.binary.version} + delta-enhancer-3.3_2.12 tech.mlsql diff --git a/ds-hbase-2x/desc.plugin b/ds-hbase-2x/desc.plugin index e0f393fc..7cecd945 100644 --- a/ds-hbase-2x/desc.plugin +++ b/ds-hbase-2x/desc.plugin @@ -1,7 +1,7 @@ -moduleName=ds-hbase-2x-3.0 +moduleName=ds-hbase-2x-3.3 mainClass=tech.mlsql.plugins.ds.MLSQLHBase2x scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/ds-hbase-2x/pom.xml b/ds-hbase-2x/pom.xml index 4fc9083f..a2ddb86c 100644 --- a/ds-hbase-2x/pom.xml +++ b/ds-hbase-2x/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT diff --git a/echo-controller/.repo/pom.template.xml b/echo-controller/.repo/pom.template.xml index 09ac44f0..66086680 100644 --- a/echo-controller/.repo/pom.template.xml +++ b/echo-controller/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - echo-controller-${spark.binary.version}_${scala.binary.version} + echo-controller-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/echo-controller/pom.xml b/echo-controller/pom.xml index 7d26dce1..dab3a562 100644 --- a/echo-controller/pom.xml +++ b/echo-controller/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - echo-controller-${spark.binary.version}_${scala.binary.version} + echo-controller-3.3_2.12 \ No newline at end of file diff --git a/install-all.sh b/install-all.sh index d0e3a9e9..cbe3f588 100755 --- a/install-all.sh +++ b/install-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -ALL_MODUELS="mlsql-shell mlsql-assert mlsql-mllib mlsql-excel connect-persist last-command run-script save-then-load stream-persist table-repartition" +ALL_MODUELS="mlsql-shell mlsql-assert mlsql-mllib mlsql-excel byzer-yaml-visualization connect-persist mlsql-ds mlsql-ke mlsql-canal last-command table-repartition" MODUELS=${1:-""} @@ -8,7 +8,7 @@ if [[ "${MODUELS}" == "" ]];then MODUELS=${ALL_MODUELS} fi -for spark_version in spark243 spark311 +for spark_version in 2.4 3.0 3.3 do for module in ${MODUELS} do diff --git a/install.sh b/install.sh index 4b437917..e7ebb2d6 100755 --- a/install.sh +++ b/install.sh @@ -2,19 +2,24 @@ PROJECT=/Users/allwefantasy/projects/byzer-extension MOUDLE_NAME=$1 VERSION="0.1.0-SNAPSHOT" -V=${2:-3.0} +V=${2:-3.3} MIDDLE="2.4_2.11" -SPARK="spark311" +SPARK="spark330" if [[ "${V}" == "2.4" ]] then SPARK=spark243 +elif [ "${V}" == "3.0" ]; then + SPARK=spark311 +elif [ "${V}" == "3.3" ]; then + SPARK=spark330 fi -if [[ "${SPARK}" == "spark311" ]] + +if [[ "${SPARK}" == "spark330" || "${SPARK}" == "spark311" ]] then - MIDDLE="3.0_2.12" + MIDDLE="${V}_2.12" fi echo ${MOUDLE_NAME} diff --git a/last-command/.repo/pom.template.xml b/last-command/.repo/pom.template.xml index ee24a3c0..5f962c6b 100644 --- a/last-command/.repo/pom.template.xml +++ b/last-command/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - last-command-${spark.binary.version}_${scala.binary.version} + last-command-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/last-command/pom.xml b/last-command/pom.xml index c49e23b8..d074166f 100644 --- a/last-command/pom.xml +++ b/last-command/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - last-command-${spark.binary.version}_${scala.binary.version} + last-command-3.3_2.12 \ No newline at end of file diff --git a/mlsql-analysis-toolkit/.repo/pom.template.xml b/mlsql-analysis-toolkit/.repo/pom.template.xml index 21b49385..bbda0997 100644 --- a/mlsql-analysis-toolkit/.repo/pom.template.xml +++ b/mlsql-analysis-toolkit/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - mlsql-analysis-toolkit-${spark.binary.version}_${scala.binary.version} + mlsql-analysis-toolkit-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/mlsql-analysis-toolkit/pom.xml b/mlsql-analysis-toolkit/pom.xml index d867d25a..872f2344 100644 --- a/mlsql-analysis-toolkit/pom.xml +++ b/mlsql-analysis-toolkit/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-analysis-toolkit-${spark.binary.version}_${scala.binary.version} + mlsql-analysis-toolkit-3.3_2.12 \ No newline at end of file diff --git a/mlsql-assert/desc.plugin b/mlsql-assert/desc.plugin index df9e60d8..b1ea7ec1 100644 --- a/mlsql-assert/desc.plugin +++ b/mlsql-assert/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-assert-3.0 +moduleName=mlsql-assert-3.3 mainClass=tech.mlsql.plugins.assert.app.MLSQLAssert scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-assert/pom.xml b/mlsql-assert/pom.xml index 95695b19..d36eeb70 100644 --- a/mlsql-assert/pom.xml +++ b/mlsql-assert/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-assert-3.0_2.12 + mlsql-assert-3.3_2.12 diff --git a/mlsql-bigdl/desc.plugin b/mlsql-bigdl/desc.plugin index 965fed58..0a211db6 100644 --- a/mlsql-bigdl/desc.plugin +++ b/mlsql-bigdl/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-bigdl-3.0 +moduleName=mlsql-bigdl-3.3 mainClass=tech.mlsql.plugins.bigdl.BigDLApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/mlsql-bigdl/pom.xml b/mlsql-bigdl/pom.xml index a920492f..0e5834b9 100644 --- a/mlsql-bigdl/pom.xml +++ b/mlsql-bigdl/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT diff --git a/mlsql-canal/desc.plugin b/mlsql-canal/desc.plugin index 2d425ac7..d5bca7de 100644 --- a/mlsql-canal/desc.plugin +++ b/mlsql-canal/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-canal-3.0 +moduleName=mlsql-canal-3.3 mainClass=tech.mlsql.plugins.canal.CanalApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=zml1206 mlsqlVersions="" diff --git a/mlsql-canal/pom.xml b/mlsql-canal/pom.xml index 36c97b9c..2070b6fe 100644 --- a/mlsql-canal/pom.xml +++ b/mlsql-canal/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-canal-3.0_2.12 + mlsql-canal-3.3_2.12 diff --git a/mlsql-cli/desc.plugin b/mlsql-cli/desc.plugin index df5c6513..1d594b2e 100644 --- a/mlsql-cli/desc.plugin +++ b/mlsql-cli/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-cli-3.0 +moduleName=mlsql-cli-3.3 mainClass=tech.mlsql.plugins.cli.app.MLSQLCli scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-cli/pom.xml b/mlsql-cli/pom.xml index 353acf9d..bac61887 100644 --- a/mlsql-cli/pom.xml +++ b/mlsql-cli/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-cli-3.0_2.12 + mlsql-cli-3.3_2.12 diff --git a/mlsql-cube/desc.plugin b/mlsql-cube/desc.plugin index 6c908422..2d70accf 100644 --- a/mlsql-cube/desc.plugin +++ b/mlsql-cube/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-cube-3.0 +moduleName=mlsql-cube-3.3 mainClass=tech.mlsql.plugins.shell.app.MLSQLCube scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-cube/pom.xml b/mlsql-cube/pom.xml index a5345b18..14e119bd 100644 --- a/mlsql-cube/pom.xml +++ b/mlsql-cube/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-cube-3.0_2.12 + mlsql-cube-3.3_2.12 diff --git a/mlsql-ds/desc.plugin b/mlsql-ds/desc.plugin index 6c32a730..97d6cdaa 100644 --- a/mlsql-ds/desc.plugin +++ b/mlsql-ds/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-ds-3.0 +moduleName=mlsql-ds-3.3 mainClass=tech.mlsql.plugins.ds.app.MLSQLDs scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-ds/pom.xml b/mlsql-ds/pom.xml index ca0c71c4..ff3faeb8 100644 --- a/mlsql-ds/pom.xml +++ b/mlsql-ds/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-ds-3.0_2.12 + mlsql-ds-3.3_2.12 com.databricks diff --git a/mlsql-excel/desc.plugin b/mlsql-excel/desc.plugin index eb1c4d7c..052b14c3 100644 --- a/mlsql-excel/desc.plugin +++ b/mlsql-excel/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-excel-3.0 +moduleName=mlsql-excel-3.3 mainClass=tech.mlsql.plugins.ds.MLSQLExcel scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/mlsql-excel/pom.xml b/mlsql-excel/pom.xml index 6e628393..1ee32a59 100644 --- a/mlsql-excel/pom.xml +++ b/mlsql-excel/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-excel-3.0_2.12 + mlsql-excel-3.3_2.12 org.apache.poi diff --git a/mlsql-ext-ets/desc.plugin b/mlsql-ext-ets/desc.plugin index da9e1cd6..5bf878c1 100644 --- a/mlsql-ext-ets/desc.plugin +++ b/mlsql-ext-ets/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-ext-ets-3.0 +moduleName=mlsql-ext-ets-3.3 mainClass=tech.mlsql.plugins.ext.ets.app.MLSQLETApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-ext-ets/pom.xml b/mlsql-ext-ets/pom.xml index 9d767b7e..84fd2edb 100644 --- a/mlsql-ext-ets/pom.xml +++ b/mlsql-ext-ets/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-ext-ets-3.0_2.12 + mlsql-ext-ets-3.3_2.12 1.2.32 @@ -21,7 +21,7 @@ - + diff --git a/mlsql-ext-ets/src/test/java/tech/mlsql/test/ETTest.scala b/mlsql-ext-ets/src/test/java/tech/mlsql/test/ETTest.scala deleted file mode 100644 index 12e170e1..00000000 --- a/mlsql-ext-ets/src/test/java/tech/mlsql/test/ETTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -package tech.mlsql.test - -import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import streaming.core.strategy.platform.SparkRuntime -import tech.mlsql.plugins.ets.EmptyTable - -/** - * 23/6/2022 WilliamZhu(allwefantasy@gmail.com) - */ -//class ETTest extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { -// "et" should "test-example" in { -// withBatchContext(setupBatchContext(batchParams)) { runtime: SparkRuntime => -// //执行sql -// implicit val spark = runtime.sparkSession -// val et = new EmptyTable() -// val df = spark.emptyDataFrame -// val newDf = et.train(df, "", Map()) -// newDf.show() -// -// } -// } -//} diff --git a/mlsql-ke/desc.plugin b/mlsql-ke/desc.plugin index c8d61651..638cf705 100644 --- a/mlsql-ke/desc.plugin +++ b/mlsql-ke/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-ke-3.0 +moduleName=mlsql-ke-3.3 mainClass=tech.mlsql.plugins.ke.app.MLSQLKE scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-ke/pom.xml b/mlsql-ke/pom.xml index 38b50e07..29d00942 100644 --- a/mlsql-ke/pom.xml +++ b/mlsql-ke/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-ke-3.0_2.12 + mlsql-ke-3.3_2.12 1.2.32 diff --git a/mlsql-language-server/desc.plugin b/mlsql-language-server/desc.plugin index f48e6ac2..9de4e3b6 100644 --- a/mlsql-language-server/desc.plugin +++ b/mlsql-language-server/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-language-server-3.0 +moduleName=mlsql-language-server-3.3 mainClass=tech.mlsql.plugins.ke.app.MLSQLLanguageServer scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-language-server/pom.xml b/mlsql-language-server/pom.xml index 1e940844..f0b7df78 100644 --- a/mlsql-language-server/pom.xml +++ b/mlsql-language-server/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-language-server-3.0_2.12 + mlsql-language-server-3.3_2.12 diff --git a/mlsql-mllib/desc.plugin b/mlsql-mllib/desc.plugin index 80d879f0..b0ae2512 100644 --- a/mlsql-mllib/desc.plugin +++ b/mlsql-mllib/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-mllib-3.0 +moduleName=mlsql-mllib-3.3 mainClass=tech.mlsql.plugins.mllib.app.MLSQLMllib scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-mllib/pom.xml b/mlsql-mllib/pom.xml index d4a3a442..d4eeda32 100644 --- a/mlsql-mllib/pom.xml +++ b/mlsql-mllib/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-mllib-3.0_2.12 + mlsql-mllib-3.3_2.12 diff --git a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryTest.scala b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryTest.scala index dd4146ca..2784587b 100644 --- a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryTest.scala +++ b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryTest.scala @@ -4,7 +4,8 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.DoubleType import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import streaming.core.strategy.platform.SparkRuntime import tech.mlsql.test.BasicMLSQLConfig @@ -18,7 +19,7 @@ import java.util.UUID * @Date: 2022/6/27 19:07 * */ -class SQLDataSummaryTest extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { +class SQLDataSummaryTest extends AnyFunSuite with SparkOperationUtil with BasicMLSQLConfig with BeforeAndAfterAll with BeforeAndAfterEach { def startParams = Array( "-streaming.master", "local[2]", "-streaming.name", "unit-test", @@ -30,7 +31,7 @@ class SQLDataSummaryTest extends FlatSpec with SparkOperationUtil with Matchers "-streaming.unittest", "true" ) - "DataSummary" should "Summarize the Dataset" in { + test("DataSummary should summarize the Dataset") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark: SparkSession = runtime.sparkSession val et = new SQLDataSummary() diff --git a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryV2Test.scala b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryV2Test.scala index 2314586e..69b86e1a 100644 --- a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryV2Test.scala +++ b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDataSummaryV2Test.scala @@ -1,9 +1,10 @@ package tech.mlsql.plugins.mllib.ets.fe import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.functions.{col, explode, struct, desc} +import org.apache.spark.sql.functions.{col, desc, explode, struct} import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.{BeforeAndAfterAll} import streaming.core.strategy.platform.SparkRuntime import tech.mlsql.test.BasicMLSQLConfig @@ -17,7 +18,7 @@ import java.util.{Date, UUID} * @Date: 2022/6/27 19:07 * */ -class SQLDataSummaryV2Test extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { +class SQLDataSummaryV2Test extends AnyFunSuite with SparkOperationUtil with BasicMLSQLConfig with BeforeAndAfterAll { def startParams = Array( "-streaming.master", "local[*]", "-streaming.name", "unit-test", @@ -32,8 +33,7 @@ class SQLDataSummaryV2Test extends FlatSpec with SparkOperationUtil with Matcher "-spark.executor.memoryOverheadFactor", "0.2", "-spark.dirver.maxResultSize", "2g" ) - - "DataSummary" should "Summarize the Dataset" in { + test("DataSummary should summarize the Dataset") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark: SparkSession = runtime.sparkSession val et = new SQLDataSummaryV2() diff --git a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDescriptiveMetricsTest.scala b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDescriptiveMetricsTest.scala index dad66d31..8049c242 100644 --- a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDescriptiveMetricsTest.scala +++ b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLDescriptiveMetricsTest.scala @@ -3,7 +3,8 @@ package tech.mlsql.plugins.mllib.ets.fe import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.DoubleType import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import streaming.core.strategy.platform.SparkRuntime import tech.mlsql.test.BasicMLSQLConfig @@ -14,7 +15,7 @@ import java.util.UUID /** * 12/07/2022 hellozepp(lisheng.zhanglin@163.com) */ -class SQLDescriptiveMetricsTest extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { +class SQLDescriptiveMetricsTest extends AnyFunSuite with SparkOperationUtil with BasicMLSQLConfig with BeforeAndAfterAll with BeforeAndAfterEach { def startParams = Array( "-streaming.master", "local[2]", "-streaming.name", "unit-test", @@ -26,7 +27,7 @@ class SQLDescriptiveMetricsTest extends FlatSpec with SparkOperationUtil with Ma "-streaming.unittest", "true" ) - "DescriptiveMetrics" should "Returns the frequency of a field in a table" in { + test("DescriptiveMetrics should Returns the frequency of a field in a table") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark = runtime.sparkSession val et = new SQLDescriptiveMetrics() diff --git a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLPatternSummaryTest.scala b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLPatternSummaryTest.scala index 1ca70e11..b7390a5b 100644 --- a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLPatternSummaryTest.scala +++ b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLPatternSummaryTest.scala @@ -1,9 +1,9 @@ package tech.mlsql.plugins.mllib.ets.fe import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import tech.mlsql.test.BasicMLSQLConfig -import org.apache.spark.ml.fpm.FPGrowth +import org.scalatest.funsuite.AnyFunSuite import streaming.core.strategy.platform.SparkRuntime import java.util.UUID @@ -14,7 +14,7 @@ import java.util.UUID * @Date: 2022/7/13 12:48 * */ -class SQLPatternSummaryTest extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { +class SQLPatternSummaryTest extends AnyFunSuite with SparkOperationUtil with BasicMLSQLConfig with BeforeAndAfterAll with BeforeAndAfterEach { def startParams = Array( "-streaming.master", "local[2]", @@ -27,7 +27,7 @@ class SQLPatternSummaryTest extends FlatSpec with SparkOperationUtil with Matche "-streaming.unittest", "true" ) - "DataSummary" should "Summarize the Dataset" in { + test("PatternSummary should Summarize the Dataset") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark = runtime.sparkSession val et = new SQLPatternDistribution() diff --git a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLUniqueIdentifierTest.scala b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLUniqueIdentifierTest.scala index 5fa12881..3670dcfc 100644 --- a/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLUniqueIdentifierTest.scala +++ b/mlsql-mllib/src/test/java/tech/mlsql/plugins/mllib/ets/fe/SQLUniqueIdentifierTest.scala @@ -3,7 +3,8 @@ package tech.mlsql.plugins.mllib.ets.fe import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.DoubleType import org.apache.spark.streaming.SparkOperationUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import streaming.core.strategy.platform.SparkRuntime import tech.mlsql.test.BasicMLSQLConfig @@ -14,7 +15,7 @@ import java.util.UUID /** * 12/07/2022 hellozepp(lisheng.zhanglin@163.com) */ -class SQLUniqueIdentifierTest extends FlatSpec with SparkOperationUtil with Matchers with BasicMLSQLConfig with BeforeAndAfterAll { +class SQLUniqueIdentifierTest extends AnyFunSuite with SparkOperationUtil with BasicMLSQLConfig with BeforeAndAfterAll with BeforeAndAfterEach { def startParams = Array( "-streaming.master", "local[2]", "-streaming.name", "unit-test", @@ -26,7 +27,7 @@ class SQLUniqueIdentifierTest extends FlatSpec with SparkOperationUtil with Matc "-streaming.unittest", "true" ) - "SQLUniqueIdentifier" should "Returns the unique id of a field in a table" in { + test("SQLUniqueIdentifier should Returns the unique id of a field in a table") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark = runtime.sparkSession val et = new SQLUniqueIdentifier() @@ -63,7 +64,7 @@ class SQLUniqueIdentifierTest extends FlatSpec with SparkOperationUtil with Matc } } - "param of startIndex" should "Returns the column starting at the specified index" in { + test("param of startIndex should Returns the column starting at the specified index") { withBatchContext(setupBatchContext(startParams)) { runtime: SparkRuntime => implicit val spark = runtime.sparkSession val et = new SQLUniqueIdentifier() diff --git a/mlsql-shell/desc.plugin b/mlsql-shell/desc.plugin index 858db631..aef262f4 100644 --- a/mlsql-shell/desc.plugin +++ b/mlsql-shell/desc.plugin @@ -1,7 +1,7 @@ -moduleName=mlsql-shell-3.0 +moduleName=mlsql-shell-3.3 mainClass=tech.mlsql.plugins.shell.app.MLSQLShell scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="" diff --git a/mlsql-shell/pom.xml b/mlsql-shell/pom.xml index 49004c68..143fd94f 100644 --- a/mlsql-shell/pom.xml +++ b/mlsql-shell/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - mlsql-shell-3.0_2.12 + mlsql-shell-3.3_2.12 diff --git a/pom.xml b/pom.xml index 2404e992..25d0a6c8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 tech.mlsql - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 pom 0.1.0-SNAPSHOT @@ -33,6 +33,11 @@ mlsql-cli byzer-yaml-visualization byzer-openmldb + byzer-objectstore-cos + byzer-objectstore-obs + byzer-objectstore-oss + byzer-objectstore-blob + byzer-objectstore-s3 byzer-xgboost MLSQL Plugins @@ -70,23 +75,21 @@ UTF-8 - 2.12.10 + 2.12.15 2.12 - 3.1.1 - 3.0 - 3.0 - + 3.3.0 + 3.3 + 3.3 + 3.2.13 + 2.4.0-SNAPSHOT + 0.4.0 16.0 4.5.3 - - 0.3.9.4 - 2.0.5 + 0.3.9.5 + 2.0.9 provided 2.6.5 - 2.3.0-SNAPSHOT - 0.2.0 0.8.0 - 3.0.3 @@ -128,7 +131,7 @@ tech.mlsql - streamingpro-spark-3.0.0-adaptor_${scala.binary.version} + streamingpro-spark-${spark.version}-adaptor_${scala.binary.version} ${mlsql.version} ${scope} @@ -208,24 +211,6 @@ - - spark-2.4 - - 2.4.3 - 2.4 - 2.4 - - - - - spark-3.0 - - 3.0.0-preview2 - 3.0 - 3.0 - - - disable-java8-doclint diff --git a/run-script/.repo/desc.template.plugin b/run-script/.repo/desc.template.plugin new file mode 100644 index 00000000..ab461d27 --- /dev/null +++ b/run-script/.repo/desc.template.plugin @@ -0,0 +1,15 @@ +moduleName=run-script-{{spark_binary_version}} +mainClass=tech.mlsql.plugins.et.RunScript +version=0.1.0-SNAPSHOT +author=allwefantasy +mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" +githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/run-script +scala_version={{scala_binary_version}} +spark_version={{spark_binary_version}} +mlsqlPluginType=et +desc=wow + + + + + diff --git a/run-script/.repo/pom.template.xml b/run-script/.repo/pom.template.xml index b10ef09c..144fc9d2 100644 --- a/run-script/.repo/pom.template.xml +++ b/run-script/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - run-script-${spark.binary.version}_${scala.binary.version} + run-script-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/run-script/desc.plugin b/run-script/desc.plugin index aa36683d..d378a9d9 100644 --- a/run-script/desc.plugin +++ b/run-script/desc.plugin @@ -1,15 +1,14 @@ -moduleName=run-script-2.4 +moduleName=run-script-3.3 mainClass=tech.mlsql.plugins.et.RunScript version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" githubUrl=https://github.com/allwefantasy/mlsql-plugins/tree/master/run-script -scala_version=2.11 -spark_version=2.4 +scala_version=2.12 +spark_version=3.3 mlsqlPluginType=et desc=wow - diff --git a/run-script/pom.xml b/run-script/pom.xml index 1e0bc6a2..312564a0 100644 --- a/run-script/pom.xml +++ b/run-script/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - run-script-${spark.binary.version}_${scala.binary.version} + run-script-3.3_2.12 \ No newline at end of file diff --git a/save-then-load/.repo/pom.template.xml b/save-then-load/.repo/pom.template.xml index 3b2f2926..e7d4e259 100644 --- a/save-then-load/.repo/pom.template.xml +++ b/save-then-load/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - save-then-load-${spark.binary.version}_${scala.binary.version} + save-then-load-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/save-then-load/pom.xml b/save-then-load/pom.xml index a92c6a03..7334be85 100644 --- a/save-then-load/pom.xml +++ b/save-then-load/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - save-then-load-${spark.binary.version}_${scala.binary.version} + save-then-load-3.3_2.12 \ No newline at end of file diff --git a/stream-persist/.repo/pom.template.xml b/stream-persist/.repo/pom.template.xml index 7b9e1c1e..241fe33c 100644 --- a/stream-persist/.repo/pom.template.xml +++ b/stream-persist/.repo/pom.template.xml @@ -9,6 +9,6 @@ 4.0.0 - stream-persist-${spark.binary.version}_${scala.binary.version} + stream-persist-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/stream-persist/desc.plugin b/stream-persist/desc.plugin index 6c800c58..bca59d70 100644 --- a/stream-persist/desc.plugin +++ b/stream-persist/desc.plugin @@ -1,7 +1,7 @@ -moduleName=stream-persist-app-3.0 +moduleName=stream-persist-app-3.3 mainClass=tech.mlsq.streambootstrapatstartup.StreamApp scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/stream-persist/pom.xml b/stream-persist/pom.xml index d8c9fcfd..a9c9ef74 100644 --- a/stream-persist/pom.xml +++ b/stream-persist/pom.xml @@ -3,12 +3,12 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - stream-persist-${spark.binary.version}_${scala.binary.version} + stream-persist-3.3_2.12 \ No newline at end of file diff --git a/table-repartition/.repo/pom.template.xml b/table-repartition/.repo/pom.template.xml index f28b87f9..d22a8d80 100644 --- a/table-repartition/.repo/pom.template.xml +++ b/table-repartition/.repo/pom.template.xml @@ -9,7 +9,7 @@ 4.0.0 - table-repartition-${spark.binary.version}_${scala.binary.version} + table-repartition-{{spark_binary_version}}_{{scala_binary_version}} \ No newline at end of file diff --git a/table-repartition/desc.plugin b/table-repartition/desc.plugin index 191d0e70..b09e706a 100644 --- a/table-repartition/desc.plugin +++ b/table-repartition/desc.plugin @@ -1,7 +1,7 @@ -moduleName=table-repartition-3.0 +moduleName=table-repartition-3.3 mainClass=tech.mlsql.plugins.et.TableRepartition scala_version=2.12 -spark_version=3.0 +spark_version=3.3 version=0.1.0-SNAPSHOT author=allwefantasy mlsqlVersions="1.5.0,1.5.0-SNAPSHOT,1.6.0,1.6.0-SNAPSHOT" diff --git a/table-repartition/pom.xml b/table-repartition/pom.xml index 8c3d70db..2885642a 100644 --- a/table-repartition/pom.xml +++ b/table-repartition/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - mlsql-plugins-3.0_2.12 + mlsql-plugins-3.3_2.12 tech.mlsql 0.1.0-SNAPSHOT 4.0.0 - table-repartition-${spark.binary.version}_${scala.binary.version} + table-repartition-3.3_2.12 \ No newline at end of file