diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml index 7d4acbd6..707e3116 100644 --- a/.github/workflows/build-extension.yml +++ b/.github/workflows/build-extension.yml @@ -44,42 +44,28 @@ jobs: - name: Build spark connector 2.4 2.11 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-2 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.11 -pl spark-doris-connector-spark-2 -am - name: Build spark connector 2.4 2.12 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Dscala.version=2.12.18 \ - -Dscala.major.version=2.12 \ - -Pspark-2 -pl spark-doris-connector-dist -am - - - name: Build spark connector 3.0 - run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.0 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.12 -pl spark-doris-connector-spark-2 -am - name: Build spark connector 3.1 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.1 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.1 -pl spark-doris-connector-spark-3.1 -am - name: Build spark connector 3.2 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.2 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.2 -pl spark-doris-connector-spark-3.2 -am - name: Build spark connector 3.3 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.3 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.3 -pl spark-doris-connector-spark-3.3 -am - name: Build spark connector 3.4 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.4 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.4 -pl spark-doris-connector-spark-3.4 -am - name: Build spark connector 3.5 run: | - cd spark-doris-connector && mvn clean install ${MVN_OPT} \ - -Pspark-3,spark-3.5 -pl spark-doris-connector-dist -am + cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.5 -pl spark-doris-connector-spark-3.5 -am diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml index 559f17dd..fddcda73 100644 --- a/.github/workflows/run-itcase-12.yml +++ b/.github/workflows/run-itcase-12.yml @@ -40,8 +40,8 @@ jobs: - name: Run ITCases for spark 2 run: | - cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" + cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" - name: Run ITCases for spark 3 run: | - cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" + cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" diff --git a/.github/workflows/run-itcase-20.yml b/.github/workflows/run-itcase-20.yml index 7af80ee3..d16d810e 100644 --- a/.github/workflows/run-itcase-20.yml +++ b/.github/workflows/run-itcase-20.yml @@ -40,9 +40,9 @@ jobs: - name: Run ITCases for spark 2 run: | - cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" + cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" - name: Run ITCases for spark 3 run: | - cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" + cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" \ No newline at end of file diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh index 88ea3050..2e953ced 100755 --- a/spark-doris-connector/build.sh +++ b/spark-doris-connector/build.sh @@ -142,32 +142,29 @@ selectScala() { selectSpark() { echo 'Spark-Doris-Connector supports multiple versions of spark. Which version do you need ?' - select spark in "2.4" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "other" + select spark in "2.4" "3.1" "3.2" "3.3" "3.4" "3.5" "other" do case $spark in "2.4") return 1 ;; - "3.0") - return 2 - ;; "3.1") - return 3 + return 2 ;; "3.2") - return 4 + return 3 ;; "3.3") - return 5 + return 4 ;; "3.4") - return 6 + return 5 ;; "3.5") - return 7 + return 6 ;; "other") - return 8 + return 7 ;; esac done @@ -189,18 +186,16 @@ SparkVer=$? if [ ${SparkVer} -eq 1 ]; then SPARK_VERSION="2.4.8" elif [ ${SparkVer} -eq 2 ]; then - SPARK_VERSION="3.0.3" -elif [ ${SparkVer} -eq 3 ]; then SPARK_VERSION="3.1.3" -elif [ ${SparkVer} -eq 4 ]; then +elif [ ${SparkVer} -eq 3 ]; then SPARK_VERSION="3.2.4" -elif [ ${SparkVer} -eq 5 ]; then +elif [ ${SparkVer} -eq 4 ]; then SPARK_VERSION="3.3.4" -elif [ ${SparkVer} -eq 6 ]; then +elif [ ${SparkVer} -eq 5 ]; then SPARK_VERSION="3.4.3" -elif [ ${SparkVer} -eq 7 ]; then +elif [ ${SparkVer} -eq 6 ]; then SPARK_VERSION="3.5.3" -elif [ ${SparkVer} -eq 8 ]; then +elif [ ${SparkVer} -eq 7 ]; then # shellcheck disable=SC2162 read -p 'Which spark version do you need? please input :' ver @@ -223,23 +218,23 @@ echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}" echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}" echo_g " build starting..." -SPARK_PRIMARY_VERSION=0 -[ ${SPARK_MAJOR_VERSION} != 0 ] && SPARK_PRIMARY_VERSION=${SPARK_MAJOR_VERSION%.*} +if [[ $SPARK_VERSION =~ ^3.* ]]; then + profile_name="spark-${SPARK_MAJOR_VERSION}" + module_suffix=${SPARK_MAJOR_VERSION} +else + profile_name="spark-${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}" + module_suffix="2" +fi -${MVN_BIN} clean install \ - -Dspark.version=${SPARK_VERSION} \ - -Dscala.version=${SCALA_VERSION} \ - -Dspark.major.version=${SPARK_MAJOR_VERSION} \ - -Dscala.major.version=${SCALA_MAJOR_VERSION} \ - -Pspark-${SPARK_PRIMARY_VERSION} -pl spark-doris-connector-dist -am "$@" +${MVN_BIN} clean install -P"${profile_name}" -am "$@" EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then DIST_DIR=${DORIS_HOME}/dist [ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR" - dist_jar=$(ls "${ROOT}"/spark-doris-connector-dist/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-") - rm -rf "${DIST_DIR}"/spark-doris-connector-dist/"${dist_jar}" - cp "${ROOT}"/spark-doris-connector-dist/target/"${dist_jar}" "$DIST_DIR" + dist_jar=$(ls "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-") + rm -rf "${DIST_DIR}"/spark-doris-connector-spark-"${module_suffix}"/"${dist_jar}" + cp "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target/"${dist_jar}" "$DIST_DIR" echo_g "*****************************************************************" echo_g "Successfully build Spark-Doris-Connector" diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml index 8191ed6f..db961877 100644 --- a/spark-doris-connector/pom.xml +++ b/spark-doris-connector/pom.xml @@ -41,11 +41,14 @@ spark-doris-connector-base - spark-doris-connector-shims - spark-doris-connector-dist spark-doris-connector-spark-2 - spark-doris-connector-spark-3 spark-doris-connector-it + spark-doris-connector-spark-3-base + spark-doris-connector-spark-3.1 + spark-doris-connector-spark-3.2 + spark-doris-connector-spark-3.3 + spark-doris-connector-spark-3.4 + spark-doris-connector-spark-3.5 @@ -92,6 +95,7 @@ 1.49 1 -Xmx512m + true @@ -108,20 +112,34 @@ org.apache.doris - spark-doris-connector-spark-3 + spark-doris-connector-spark-3-base ${project.version} org.apache.doris - spark-doris-connector-shims_spark-3-base + spark-doris-connector-spark-3.1 ${project.version} org.apache.doris - spark-doris-connector-shims_spark-${spark.major.version} + spark-doris-connector-spark-3.2 + ${project.version} + + + org.apache.doris + spark-doris-connector-spark-3.3 + ${project.version} + + + org.apache.doris + spark-doris-connector-spark-3.4 + ${project.version} + + + org.apache.doris + spark-doris-connector-spark-3.5 ${project.version} - org.apache.doris thrift-service @@ -764,7 +782,7 @@ not reuse forked jvm, so that each unit test will run in separate jvm. to avoid singleton confict<--> false - -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine} + -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar --add-opens=java.base/java.nio=ALL-UNNAMED @{argLine} @@ -773,22 +791,19 @@ - spark-2 - - true - + spark-2.4_2.11 2.4.8 2.4 - 2.11.10 + 2.11.12 2.11 - spark-3.0 + spark-2.4_2.12 - 3.0.0 - 3.0 + 2.4.8 + 2.4 2.12.18 2.12 @@ -813,6 +828,9 @@ spark-3.3 + + true + 3.3.0 3.3 diff --git a/spark-doris-connector/spark-doris-connector-base/pom.xml b/spark-doris-connector/spark-doris-connector-base/pom.xml index 4f2f0ef3..0e26fc2a 100644 --- a/spark-doris-connector/spark-doris-connector-base/pom.xml +++ b/spark-doris-connector/spark-doris-connector-base/pom.xml @@ -35,6 +35,7 @@ 8 8 UTF-8 + true @@ -296,4 +297,4 @@ - \ No newline at end of file + diff --git a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala index 3ae5ec6b..03a914af 100644 --- a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala +++ b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala @@ -38,7 +38,7 @@ import java.io.{ByteArrayOutputStream, IOException} import java.nio.charset.StandardCharsets import java.util.zip.GZIPOutputStream import java.util.{Base64, Properties, UUID} -import scala.collection.JavaConverters._ +import scala.collection.JavaConverters.{asJavaIteratorConverter, mapAsJavaMapConverter, propertiesAsScalaMapConverter} import scala.util.{Failure, Success, Try} case class CopyIntoResponse(code: Int, msg: String, content: String) @@ -234,7 +234,7 @@ class CopyIntoLoader(settings: SparkSettings, isStreaming: Boolean) extends Load } props.remove("columns") val properties = new Properties() - properties.putAll(props.mapValues(_.toString).asJava) + props.foreach(p => properties.setProperty(p._1, p._2)) properties } diff --git a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala index 275ad87b..f85eb287 100644 --- a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala +++ b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala @@ -67,7 +67,7 @@ object SchemaConvertors { def convertToSchema(tscanColumnDescs: Seq[TScanColumnDesc]): Schema = { val schema = new Schema(tscanColumnDescs.length) tscanColumnDescs.foreach(desc => { - println(desc.getName + " " + desc.getType.name()) + // println(desc.getName + " " + desc.getType.name()) schema.put(new Field(desc.getName, desc.getType.name, "", 0, 0, "")) }) schema diff --git a/spark-doris-connector/spark-doris-connector-dist/pom.xml b/spark-doris-connector/spark-doris-connector-dist/pom.xml deleted file mode 100644 index 5bda3b5e..00000000 --- a/spark-doris-connector/spark-doris-connector-dist/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - org.apache.doris - spark-doris-connector - ${revision} - ../pom.xml - - - spark-doris-connector-dist - jar - - - 8 - 8 - UTF-8 - - - - - spark-2 - - - org.apache.doris - spark-doris-connector-spark-2 - ${project.version} - - - - - spark-3 - - - org.apache.doris - spark-doris-connector-spark-3 - ${project.version} - - - - - - spark-doris-connector-${spark.major.version}_${scala.major.version}-${revision} - - - org.codehaus.mojo - flatten-maven-plugin - 1.2.5 - - true - resolveCiFriendliesOnly - - - - flatten - process-resources - - flatten - - - - flatten.clean - clean - - clean - - - - - - - - \ No newline at end of file diff --git a/spark-doris-connector/spark-doris-connector-it/pom.xml b/spark-doris-connector/spark-doris-connector-it/pom.xml index 761d5a19..9493c031 100644 --- a/spark-doris-connector/spark-doris-connector-it/pom.xml +++ b/spark-doris-connector/spark-doris-connector-it/pom.xml @@ -56,10 +56,12 @@ org.apache.spark spark-core_${scala.major.version} + test org.apache.spark spark-sql_${scala.major.version} + test org.scala-lang @@ -77,7 +79,7 @@ - spark-2 + spark-2-it true @@ -91,11 +93,11 @@ - spark-3 + spark-3-it org.apache.doris - spark-doris-connector-spark-3 + spark-doris-connector-spark-3.1 ${project.version} test diff --git a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala index e2d0e08d..a147a7d8 100644 --- a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala +++ b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala @@ -37,6 +37,7 @@ class DorisReaderITCase extends DorisTestBase { val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("rddSource") val sc = new SparkContext(sparkConf) + // sc.setLogLevel("DEBUG") val dorisSparkRDD = sc.dorisRDD( tableIdentifier = Some(DATABASE + "." + TABLE_READ), cfg = Some(Map( @@ -121,7 +122,7 @@ class DorisReaderITCase extends DorisTestBase { private def compareCollectResult(a1: Array[AnyRef], a2: Array[AnyRef]): Boolean = { if (a1.length == a2.length) { - for (idx <- a1.indices) { + for (idx <- 0 until a1.length) { if (!a1(idx).isInstanceOf[Array[AnyRef]] || !a2(idx).isInstanceOf[Array[AnyRef]]) { return false } @@ -130,7 +131,7 @@ class DorisReaderITCase extends DorisTestBase { if (arr1.length != arr2.length) { return false } - for (idx2 <- arr1.indices) { + for (idx2 <- 0 until arr2.length) { if (arr1(idx2) != arr2(idx2)) { return false } diff --git a/spark-doris-connector/spark-doris-connector-shims/pom.xml b/spark-doris-connector/spark-doris-connector-shims/pom.xml deleted file mode 100644 index e0f5c0cd..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.doris - spark-doris-connector - ${revision} - ../pom.xml - - pom - - spark-doris-connector-shims - - - spark-doris-connector-shims_spark-3.0 - spark-doris-connector-shims_spark-3.1 - spark-doris-connector-shims_spark-3.2 - spark-doris-connector-shims_spark-3.3 - spark-doris-connector-shims_spark-3.4 - spark-doris-connector-shims_spark-3.5 - spark-doris-connector-shims_spark-3-base - - - - - org.apache.spark - spark-sql_${scala.major.version} - provided - - - - \ No newline at end of file diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 0effe52d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - @throws[NoSuchNamespaceException] - override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index d137bd51..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -import org.apache.spark.sql.sources.DataSourceRegister - -abstract class DorisDataSource extends DataSourceRegister with DorisSourceProvider with Serializable { - override def shortName(): String = "doris" -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml deleted file mode 100644 index 02675f98..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - org.apache.doris - spark-doris-connector-shims - ${revision} - ../pom.xml - - jar - - spark-doris-connector-shims_spark-3.1 - - - 3.1.3 - 3.1 - 2.12.18 - 2.12 - - - - - org.apache.doris - spark-doris-connector-shims_spark-3-base - ${project.version} - - - - \ No newline at end of file diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 0effe52d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - @throws[NoSuchNamespaceException] - override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index 782a63d1..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 0effe52d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - @throws[NoSuchNamespaceException] - override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index 782a63d1..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 64e1f79d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index 782a63d1..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 64e1f79d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index 1235f5b0..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -abstract class DorisDataSource extends DorisSourceRegisterTrait {} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml deleted file mode 100644 index 42d6d7b6..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - 4.0.0 - - org.apache.doris - spark-doris-connector-shims - ${revision} - ../pom.xml - - jar - - spark-doris-connector-shims_spark-3.5 - - - 3.4.3 - 3.4 - 2.12.18 - 2.12 - - - - - org.apache.doris - spark-doris-connector-shims_spark-3-base - - - - \ No newline at end of file diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala deleted file mode 100644 index 64e1f79d..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.catalog - -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -abstract class DorisTableCatalogBase extends SupportsNamespaces { - - def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException() - -} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala deleted file mode 100644 index 5450bbf2..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.read - -import org.apache.doris.spark.config.DorisConfig -import org.apache.doris.spark.read.DorisScanBuilderTrait -import org.apache.spark.sql.types.StructType - -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala deleted file mode 100644 index 7bec3652..00000000 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.spark.sql.sources - -abstract class DorisDataSource extends DorisSourceRegisterTrait {} \ No newline at end of file diff --git a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml similarity index 79% rename from spark-doris-connector/spark-doris-connector-spark-3/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml index 4660d907..3d7499d0 100644 --- a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml @@ -26,22 +26,26 @@ spark-doris-connector ${revision} - jar - spark-doris-connector-spark-3 + spark-doris-connector-spark-3-base + + + 8 + 8 + UTF-8 + true + org.apache.doris - spark-doris-connector-shims_spark-${spark.major.version} - ${project.version} + spark-doris-connector-base org.apache.spark spark-sql_${scala.major.version} - ${spark.version} provided - \ No newline at end of file + diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala similarity index 85% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala index d9aca8a9..4cd5841d 100644 --- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala @@ -19,10 +19,8 @@ package org.apache.doris.spark.catalog import org.apache.doris.spark.client.DorisFrontendClient import org.apache.doris.spark.config.{DorisConfig, DorisOptions} -import org.apache.doris.spark.read.DorisScanBuilder import org.apache.doris.spark.rest.models.Schema import org.apache.doris.spark.util.SchemaConvertors -import org.apache.doris.spark.write.DorisWriteBuilder import org.apache.spark.sql.connector.catalog.TableCapability._ import org.apache.spark.sql.connector.catalog.{Identifier, SupportsRead, SupportsWrite, Table, TableCapability} import org.apache.spark.sql.connector.read.ScanBuilder @@ -35,7 +33,7 @@ import scala.collection.JavaConverters._ import scala.language.implicitConversions -class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) extends Table with SupportsRead with SupportsWrite { +abstract class DorisTableBase(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) extends Table with SupportsRead with SupportsWrite { private lazy val frontend:DorisFrontendClient = new DorisFrontendClient(config) @@ -54,12 +52,12 @@ class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[Str override def newScanBuilder(caseInsensitiveStringMap: CaseInsensitiveStringMap): ScanBuilder = { config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name()) - new DorisScanBuilder(config, schema()) + createScanBuilder(config, schema()) } override def newWriteBuilder(logicalWriteInfo: LogicalWriteInfo): WriteBuilder = { config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name()) - new DorisWriteBuilder(config, logicalWriteInfo.schema()) + createWriteBuilder(config, logicalWriteInfo.schema()) } private implicit def dorisSchemaToStructType(dorisSchema: Schema): StructType = { @@ -68,4 +66,8 @@ class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[Str })) } + protected def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder + + protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder + } diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala similarity index 89% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala index 9ca7f026..2c9ad0f9 100644 --- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala @@ -20,7 +20,7 @@ package org.apache.doris.spark.catalog import org.apache.doris.spark.client.DorisFrontendClient import org.apache.doris.spark.config.{DorisConfig, DorisOptions} import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException -import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, Table, TableCatalog, TableChange} +import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange} import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -28,13 +28,13 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap import java.util import scala.collection.JavaConverters._ -class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog { +trait DorisTableCatalogBase extends TableCatalog with SupportsNamespaces { - private var catalogName: Option[String] = None + protected var catalogName: Option[String] = None - private var dorisConfig: DorisConfig = _ + protected var dorisConfig: DorisConfig = _ - private var frontend: DorisFrontendClient = _ + protected var frontend: DorisFrontendClient = _ override def name(): String = { require(catalogName.nonEmpty, "The Doris table catalog is not initialed") @@ -54,7 +54,7 @@ class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog { override def loadTable(identifier: Identifier): Table = { checkIdentifier(identifier) - new DorisTable(identifier, DorisConfig.fromMap((dorisConfig.toMap.asScala + + newTableInstance(identifier, DorisConfig.fromMap((dorisConfig.toMap.asScala + (DorisOptions.DORIS_TABLE_IDENTIFIER.getName -> getFullTableName(identifier))).asJava, false), None) } @@ -103,4 +103,6 @@ class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog { (identifier.namespace() :+ identifier.name()).map(item => s"""`$item`""").mkString(".") } + def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table + } diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala similarity index 80% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala index 4fad7edd..a03435d5 100644 --- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.spark.sql.sources +package org.apache.doris.spark.catalog -import org.apache.doris.spark.catalog.DorisTable import org.apache.doris.spark.config.{DorisConfig, DorisOptions} import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableProvider} import org.apache.spark.sql.connector.expressions.Transform @@ -26,9 +25,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap import java.util -class DorisDataSourceV2 extends DorisDataSource with TableProvider { +abstract class DorisTableProviderBase extends TableProvider { - private var t: Table = _ + protected var t: Table = _ override def inferSchema(options: CaseInsensitiveStringMap): StructType = { if (t == null) t = getTable(options) @@ -41,7 +40,7 @@ class DorisDataSourceV2 extends DorisDataSource with TableProvider { val dorisConfig = DorisConfig.fromMap(properties, false) val tableIdentifier = dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER) val tableIdentifierArr = tableIdentifier.split("\\.") - new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, Some(schema)) + newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, Some(schema)) } } @@ -51,8 +50,10 @@ class DorisDataSourceV2 extends DorisDataSource with TableProvider { val dorisConfig = DorisConfig.fromMap(options, false) val tableIdentifier = dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER) val tableIdentifierArr = tableIdentifier.split("\\.") - new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, None) + newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, None) } } + def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table; + } diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala similarity index 94% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala index 2a01a6df..a6a97dc9 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownF import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType -protected[spark] abstract class DorisScanBuilderTrait(config: DorisConfig, schema: StructType) extends ScanBuilder +protected[spark] abstract class DorisScanBuilderBase(config: DorisConfig, schema: StructType) extends ScanBuilder with SupportsPushDownFilters with SupportsPushDownRequiredColumns { diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java similarity index 100% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala similarity index 100% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml similarity index 77% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml index f0105404..2dc1f38a 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml @@ -23,17 +23,18 @@ 4.0.0 org.apache.doris - spark-doris-connector-shims + spark-doris-connector ${revision} - ../pom.xml - jar - spark-doris-connector-shims_spark-3.0 + spark-doris-connector-spark-3.1 - 3.0.3 - 3.0 + 8 + 8 + UTF-8 + 3.1.0 + 3.1 2.12.18 2.12 @@ -41,13 +42,11 @@ org.apache.doris - spark-doris-connector-shims_spark-3-base - ${project.version} + spark-doris-connector-spark-3-base org.apache.spark spark-sql_${scala.major.version} - provided diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 93% rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index 4c44d93f..986b2530 100644 --- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -org.apache.doris.spark.sql.sources.DorisDataSourceV2 +org.apache.doris.spark.sql.sources.DorisDataSource diff --git a/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala new file mode 100644 index 00000000..49a346a9 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.doris.spark.read.DorisScanBuilder +import org.apache.doris.spark.write.DorisWriteBuilder +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.WriteBuilder +import org.apache.spark.sql.types.StructType + +class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) + extends DorisTableBase(identifier, config, schema) { + + override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema) + override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala similarity index 67% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala index 5450bbf2..ca5b8c7f 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala @@ -15,10 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.spark.read +package org.apache.doris.spark.catalog import org.apache.doris.spark.config.DorisConfig -import org.apache.doris.spark.read.DorisScanBuilderTrait +import org.apache.spark.sql.connector.catalog.{Identifier, Table} import org.apache.spark.sql.types.StructType -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} +class DorisTableCatalog extends DorisTableCatalogBase { + override def dropNamespace(strings: Array[String]): Boolean = throw new UnsupportedOperationException() + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala similarity index 95% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala index c6abaac1..9e199af5 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala @@ -20,4 +20,4 @@ package org.apache.doris.spark.read import org.apache.doris.spark.config.DorisConfig import org.apache.spark.sql.types.StructType -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} +class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala new file mode 100644 index 00000000..102ac922 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.sql.sources + +import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase} +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with DorisSourceProvider with Serializable { + + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml similarity index 72% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml index 68ae0932..a4d8670d 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml @@ -23,16 +23,17 @@ 4.0.0 org.apache.doris - spark-doris-connector-shims + spark-doris-connector ${revision} - ../pom.xml - jar - spark-doris-connector-shims_spark-3.2 + spark-doris-connector-spark-3.2 - 3.2.4 + 8 + 8 + UTF-8 + 3.2.0 3.2 2.12.18 2.12 @@ -41,7 +42,11 @@ org.apache.doris - spark-doris-connector-shims_spark-3-base + spark-doris-connector-spark-3-base + + + org.apache.spark + spark-sql_${scala.major.version} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..986b2530 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.spark.sql.sources.DorisDataSource diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala new file mode 100644 index 00000000..49a346a9 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.doris.spark.read.DorisScanBuilder +import org.apache.doris.spark.write.DorisWriteBuilder +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.WriteBuilder +import org.apache.spark.sql.types.StructType + +class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) + extends DorisTableBase(identifier, config, schema) { + + override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema) + override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala new file mode 100644 index 00000000..ca5b8c7f --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisTableCatalog extends DorisTableCatalogBase { + override def dropNamespace(strings: Array[String]): Boolean = throw new UnsupportedOperationException() + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala similarity index 95% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala index c6abaac1..9e199af5 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala @@ -20,4 +20,4 @@ package org.apache.doris.spark.read import org.apache.doris.spark.config.DorisConfig import org.apache.spark.sql.types.StructType -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} +class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala new file mode 100644 index 00000000..102ac922 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.sql.sources + +import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase} +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with DorisSourceProvider with Serializable { + + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml similarity index 72% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml index a33eb775..ecc71ed8 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml @@ -23,16 +23,17 @@ 4.0.0 org.apache.doris - spark-doris-connector-shims + spark-doris-connector ${revision} - ../pom.xml - jar - spark-doris-connector-shims_spark-3.3 + spark-doris-connector-spark-3.3 - 3.3.4 + 8 + 8 + UTF-8 + 3.3.0 3.3 2.12.18 2.12 @@ -41,7 +42,11 @@ org.apache.doris - spark-doris-connector-shims_spark-3-base + spark-doris-connector-spark-3-base + + + org.apache.spark + spark-sql_${scala.major.version} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..986b2530 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.spark.sql.sources.DorisDataSource diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala new file mode 100644 index 00000000..be461afc --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.doris.spark.read.DorisScanBuilder +import org.apache.doris.spark.write.DorisWriteBuilder +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.WriteBuilder +import org.apache.spark.sql.types.StructType + +class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) + extends DorisTableBase(identifier, config, schema) { + + override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema) + override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema) + +} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala new file mode 100644 index 00000000..e3ffcc92 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisTableCatalog extends DorisTableCatalogBase { + override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException() + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala similarity index 95% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala index c6abaac1..9e199af5 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala @@ -20,4 +20,4 @@ package org.apache.doris.spark.read import org.apache.doris.spark.config.DorisConfig import org.apache.spark.sql.types.StructType -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} +class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala new file mode 100644 index 00000000..c00cd240 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.sql.sources + +import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase} +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable { + + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) + +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml similarity index 72% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml index b06e8bd5..eeee285e 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml @@ -23,16 +23,17 @@ 4.0.0 org.apache.doris - spark-doris-connector-shims + spark-doris-connector ${revision} - ../pom.xml - jar - spark-doris-connector-shims_spark-3.4 + spark-doris-connector-spark-3.4 - 3.4.3 + 8 + 8 + UTF-8 + 3.4.0 3.4 2.12.18 2.12 @@ -41,7 +42,11 @@ org.apache.doris - spark-doris-connector-shims_spark-3-base + spark-doris-connector-spark-3-base + + + org.apache.spark + spark-sql_${scala.major.version} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..986b2530 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.spark.sql.sources.DorisDataSource diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala new file mode 100644 index 00000000..be461afc --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.doris.spark.read.DorisScanBuilder +import org.apache.doris.spark.write.DorisWriteBuilder +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.WriteBuilder +import org.apache.spark.sql.types.StructType + +class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) + extends DorisTableBase(identifier, config, schema) { + + override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema) + override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema) + +} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala new file mode 100644 index 00000000..e3ffcc92 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisTableCatalog extends DorisTableCatalogBase { + override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException() + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala similarity index 90% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala rename to spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala index 5450bbf2..9e199af5 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala @@ -18,7 +18,6 @@ package org.apache.doris.spark.read import org.apache.doris.spark.config.DorisConfig -import org.apache.doris.spark.read.DorisScanBuilderTrait import org.apache.spark.sql.types.StructType -class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {} +class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala new file mode 100644 index 00000000..c00cd240 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.sql.sources + +import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase} +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable { + + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) + +} diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml similarity index 70% rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml rename to spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml index 59293ed5..2f498b4d 100644 --- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml @@ -23,17 +23,18 @@ 4.0.0 org.apache.doris - spark-doris-connector-shims + spark-doris-connector ${revision} - ../pom.xml - jar - spark-doris-connector-shims_spark-3-base + spark-doris-connector-spark-3.5 - 3.0.3 - 3.0 + 8 + 8 + UTF-8 + 3.5.0 + 3.5 2.12.18 2.12 @@ -41,7 +42,11 @@ org.apache.doris - spark-doris-connector-base + spark-doris-connector-spark-3-base + + + org.apache.spark + spark-sql_${scala.major.version} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..986b2530 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.spark.sql.sources.DorisDataSource diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala new file mode 100644 index 00000000..be461afc --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.doris.spark.read.DorisScanBuilder +import org.apache.doris.spark.write.DorisWriteBuilder +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.WriteBuilder +import org.apache.spark.sql.types.StructType + +class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) + extends DorisTableBase(identifier, config, schema) { + + override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema) + override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema) + +} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala new file mode 100644 index 00000000..e3ffcc92 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.catalog + +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisTableCatalog extends DorisTableCatalogBase { + override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException() + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) +} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala new file mode 100644 index 00000000..9e199af5 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.read + +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.types.StructType + +class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {} diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala new file mode 100644 index 00000000..c00cd240 --- /dev/null +++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.spark.sql.sources + +import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase} +import org.apache.doris.spark.config.DorisConfig +import org.apache.spark.sql.connector.catalog.{Identifier, Table} +import org.apache.spark.sql.types.StructType + +class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable { + + override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table = + new DorisTable(identifier, config, schema) + +}