diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml
index 7d4acbd6..707e3116 100644
--- a/.github/workflows/build-extension.yml
+++ b/.github/workflows/build-extension.yml
@@ -44,42 +44,28 @@ jobs:
- name: Build spark connector 2.4 2.11
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-2 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.11 -pl spark-doris-connector-spark-2 -am
- name: Build spark connector 2.4 2.12
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Dscala.version=2.12.18 \
- -Dscala.major.version=2.12 \
- -Pspark-2 -pl spark-doris-connector-dist -am
-
- - name: Build spark connector 3.0
- run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.0 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.12 -pl spark-doris-connector-spark-2 -am
- name: Build spark connector 3.1
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.1 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.1 -pl spark-doris-connector-spark-3.1 -am
- name: Build spark connector 3.2
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.2 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.2 -pl spark-doris-connector-spark-3.2 -am
- name: Build spark connector 3.3
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.3 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.3 -pl spark-doris-connector-spark-3.3 -am
- name: Build spark connector 3.4
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.4 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.4 -pl spark-doris-connector-spark-3.4 -am
- name: Build spark connector 3.5
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.5 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.5 -pl spark-doris-connector-spark-3.5 -am
diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml
index 559f17dd..fddcda73 100644
--- a/.github/workflows/run-itcase-12.yml
+++ b/.github/workflows/run-itcase-12.yml
@@ -40,8 +40,8 @@ jobs:
- name: Run ITCases for spark 2
run: |
- cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+ cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
- name: Run ITCases for spark 3
run: |
- cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+ cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
diff --git a/.github/workflows/run-itcase-20.yml b/.github/workflows/run-itcase-20.yml
index 7af80ee3..d16d810e 100644
--- a/.github/workflows/run-itcase-20.yml
+++ b/.github/workflows/run-itcase-20.yml
@@ -40,9 +40,9 @@ jobs:
- name: Run ITCases for spark 2
run: |
- cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+ cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
- name: Run ITCases for spark 3
run: |
- cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+ cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
\ No newline at end of file
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
index 88ea3050..2e953ced 100755
--- a/spark-doris-connector/build.sh
+++ b/spark-doris-connector/build.sh
@@ -142,32 +142,29 @@ selectScala() {
selectSpark() {
echo 'Spark-Doris-Connector supports multiple versions of spark. Which version do you need ?'
- select spark in "2.4" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "other"
+ select spark in "2.4" "3.1" "3.2" "3.3" "3.4" "3.5" "other"
do
case $spark in
"2.4")
return 1
;;
- "3.0")
- return 2
- ;;
"3.1")
- return 3
+ return 2
;;
"3.2")
- return 4
+ return 3
;;
"3.3")
- return 5
+ return 4
;;
"3.4")
- return 6
+ return 5
;;
"3.5")
- return 7
+ return 6
;;
"other")
- return 8
+ return 7
;;
esac
done
@@ -189,18 +186,16 @@ SparkVer=$?
if [ ${SparkVer} -eq 1 ]; then
SPARK_VERSION="2.4.8"
elif [ ${SparkVer} -eq 2 ]; then
- SPARK_VERSION="3.0.3"
-elif [ ${SparkVer} -eq 3 ]; then
SPARK_VERSION="3.1.3"
-elif [ ${SparkVer} -eq 4 ]; then
+elif [ ${SparkVer} -eq 3 ]; then
SPARK_VERSION="3.2.4"
-elif [ ${SparkVer} -eq 5 ]; then
+elif [ ${SparkVer} -eq 4 ]; then
SPARK_VERSION="3.3.4"
-elif [ ${SparkVer} -eq 6 ]; then
+elif [ ${SparkVer} -eq 5 ]; then
SPARK_VERSION="3.4.3"
-elif [ ${SparkVer} -eq 7 ]; then
+elif [ ${SparkVer} -eq 6 ]; then
SPARK_VERSION="3.5.3"
-elif [ ${SparkVer} -eq 8 ]; then
+elif [ ${SparkVer} -eq 7 ]; then
# shellcheck disable=SC2162
read -p 'Which spark version do you need? please input
:' ver
@@ -223,23 +218,23 @@ echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}"
echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}"
echo_g " build starting..."
-SPARK_PRIMARY_VERSION=0
-[ ${SPARK_MAJOR_VERSION} != 0 ] && SPARK_PRIMARY_VERSION=${SPARK_MAJOR_VERSION%.*}
+if [[ $SPARK_VERSION =~ ^3.* ]]; then
+ profile_name="spark-${SPARK_MAJOR_VERSION}"
+ module_suffix=${SPARK_MAJOR_VERSION}
+else
+ profile_name="spark-${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}"
+ module_suffix="2"
+fi
-${MVN_BIN} clean install \
- -Dspark.version=${SPARK_VERSION} \
- -Dscala.version=${SCALA_VERSION} \
- -Dspark.major.version=${SPARK_MAJOR_VERSION} \
- -Dscala.major.version=${SCALA_MAJOR_VERSION} \
- -Pspark-${SPARK_PRIMARY_VERSION} -pl spark-doris-connector-dist -am "$@"
+${MVN_BIN} clean install -P"${profile_name}" -am "$@"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
DIST_DIR=${DORIS_HOME}/dist
[ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR"
- dist_jar=$(ls "${ROOT}"/spark-doris-connector-dist/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
- rm -rf "${DIST_DIR}"/spark-doris-connector-dist/"${dist_jar}"
- cp "${ROOT}"/spark-doris-connector-dist/target/"${dist_jar}" "$DIST_DIR"
+ dist_jar=$(ls "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
+ rm -rf "${DIST_DIR}"/spark-doris-connector-spark-"${module_suffix}"/"${dist_jar}"
+ cp "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target/"${dist_jar}" "$DIST_DIR"
echo_g "*****************************************************************"
echo_g "Successfully build Spark-Doris-Connector"
diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
index 8191ed6f..db961877 100644
--- a/spark-doris-connector/pom.xml
+++ b/spark-doris-connector/pom.xml
@@ -41,11 +41,14 @@
spark-doris-connector-base
- spark-doris-connector-shims
- spark-doris-connector-dist
spark-doris-connector-spark-2
- spark-doris-connector-spark-3
spark-doris-connector-it
+ spark-doris-connector-spark-3-base
+ spark-doris-connector-spark-3.1
+ spark-doris-connector-spark-3.2
+ spark-doris-connector-spark-3.3
+ spark-doris-connector-spark-3.4
+ spark-doris-connector-spark-3.5
@@ -92,6 +95,7 @@
1.49
1
-Xmx512m
+ true
@@ -108,20 +112,34 @@
org.apache.doris
- spark-doris-connector-spark-3
+ spark-doris-connector-spark-3-base
${project.version}
org.apache.doris
- spark-doris-connector-shims_spark-3-base
+ spark-doris-connector-spark-3.1
${project.version}
org.apache.doris
- spark-doris-connector-shims_spark-${spark.major.version}
+ spark-doris-connector-spark-3.2
+ ${project.version}
+
+
+ org.apache.doris
+ spark-doris-connector-spark-3.3
+ ${project.version}
+
+
+ org.apache.doris
+ spark-doris-connector-spark-3.4
+ ${project.version}
+
+
+ org.apache.doris
+ spark-doris-connector-spark-3.5
${project.version}
-
org.apache.doris
thrift-service
@@ -764,7 +782,7 @@
not reuse forked jvm, so that each unit test will run in separate jvm. to avoid singleton confict<-->
false
- -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine}
+ -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar --add-opens=java.base/java.nio=ALL-UNNAMED @{argLine}
@@ -773,22 +791,19 @@
- spark-2
-
- true
-
+ spark-2.4_2.11
2.4.8
2.4
- 2.11.10
+ 2.11.12
2.11
- spark-3.0
+ spark-2.4_2.12
- 3.0.0
- 3.0
+ 2.4.8
+ 2.4
2.12.18
2.12
@@ -813,6 +828,9 @@
spark-3.3
+
+ true
+
3.3.0
3.3
diff --git a/spark-doris-connector/spark-doris-connector-base/pom.xml b/spark-doris-connector/spark-doris-connector-base/pom.xml
index 4f2f0ef3..0e26fc2a 100644
--- a/spark-doris-connector/spark-doris-connector-base/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-base/pom.xml
@@ -35,6 +35,7 @@
8
8
UTF-8
+ true
@@ -296,4 +297,4 @@
-
\ No newline at end of file
+
diff --git a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
index 3ae5ec6b..03a914af 100644
--- a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
+++ b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
@@ -38,7 +38,7 @@ import java.io.{ByteArrayOutputStream, IOException}
import java.nio.charset.StandardCharsets
import java.util.zip.GZIPOutputStream
import java.util.{Base64, Properties, UUID}
-import scala.collection.JavaConverters._
+import scala.collection.JavaConverters.{asJavaIteratorConverter, mapAsJavaMapConverter, propertiesAsScalaMapConverter}
import scala.util.{Failure, Success, Try}
case class CopyIntoResponse(code: Int, msg: String, content: String)
@@ -234,7 +234,7 @@ class CopyIntoLoader(settings: SparkSettings, isStreaming: Boolean) extends Load
}
props.remove("columns")
val properties = new Properties()
- properties.putAll(props.mapValues(_.toString).asJava)
+ props.foreach(p => properties.setProperty(p._1, p._2))
properties
}
diff --git a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
index 275ad87b..f85eb287 100644
--- a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
+++ b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
@@ -67,7 +67,7 @@ object SchemaConvertors {
def convertToSchema(tscanColumnDescs: Seq[TScanColumnDesc]): Schema = {
val schema = new Schema(tscanColumnDescs.length)
tscanColumnDescs.foreach(desc => {
- println(desc.getName + " " + desc.getType.name())
+ // println(desc.getName + " " + desc.getType.name())
schema.put(new Field(desc.getName, desc.getType.name, "", 0, 0, ""))
})
schema
diff --git a/spark-doris-connector/spark-doris-connector-dist/pom.xml b/spark-doris-connector/spark-doris-connector-dist/pom.xml
deleted file mode 100644
index 5bda3b5e..00000000
--- a/spark-doris-connector/spark-doris-connector-dist/pom.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-
-
-
- 4.0.0
-
- org.apache.doris
- spark-doris-connector
- ${revision}
- ../pom.xml
-
-
- spark-doris-connector-dist
- jar
-
-
- 8
- 8
- UTF-8
-
-
-
-
- spark-2
-
-
- org.apache.doris
- spark-doris-connector-spark-2
- ${project.version}
-
-
-
-
- spark-3
-
-
- org.apache.doris
- spark-doris-connector-spark-3
- ${project.version}
-
-
-
-
-
- spark-doris-connector-${spark.major.version}_${scala.major.version}-${revision}
-
-
- org.codehaus.mojo
- flatten-maven-plugin
- 1.2.5
-
- true
- resolveCiFriendliesOnly
-
-
-
- flatten
- process-resources
-
- flatten
-
-
-
- flatten.clean
- clean
-
- clean
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-it/pom.xml b/spark-doris-connector/spark-doris-connector-it/pom.xml
index 761d5a19..9493c031 100644
--- a/spark-doris-connector/spark-doris-connector-it/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-it/pom.xml
@@ -56,10 +56,12 @@
org.apache.spark
spark-core_${scala.major.version}
+ test
org.apache.spark
spark-sql_${scala.major.version}
+ test
org.scala-lang
@@ -77,7 +79,7 @@
- spark-2
+ spark-2-it
true
@@ -91,11 +93,11 @@
- spark-3
+ spark-3-it
org.apache.doris
- spark-doris-connector-spark-3
+ spark-doris-connector-spark-3.1
${project.version}
test
diff --git a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
index e2d0e08d..a147a7d8 100644
--- a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
+++ b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
@@ -37,6 +37,7 @@ class DorisReaderITCase extends DorisTestBase {
val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("rddSource")
val sc = new SparkContext(sparkConf)
+ // sc.setLogLevel("DEBUG")
val dorisSparkRDD = sc.dorisRDD(
tableIdentifier = Some(DATABASE + "." + TABLE_READ),
cfg = Some(Map(
@@ -121,7 +122,7 @@ class DorisReaderITCase extends DorisTestBase {
private def compareCollectResult(a1: Array[AnyRef], a2: Array[AnyRef]): Boolean = {
if (a1.length == a2.length) {
- for (idx <- a1.indices) {
+ for (idx <- 0 until a1.length) {
if (!a1(idx).isInstanceOf[Array[AnyRef]] || !a2(idx).isInstanceOf[Array[AnyRef]]) {
return false
}
@@ -130,7 +131,7 @@ class DorisReaderITCase extends DorisTestBase {
if (arr1.length != arr2.length) {
return false
}
- for (idx2 <- arr1.indices) {
+ for (idx2 <- 0 until arr2.length) {
if (arr1(idx2) != arr2(idx2)) {
return false
}
diff --git a/spark-doris-connector/spark-doris-connector-shims/pom.xml b/spark-doris-connector/spark-doris-connector-shims/pom.xml
deleted file mode 100644
index e0f5c0cd..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/pom.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-
-
-
- 4.0.0
-
- org.apache.doris
- spark-doris-connector
- ${revision}
- ../pom.xml
-
- pom
-
- spark-doris-connector-shims
-
-
- spark-doris-connector-shims_spark-3.0
- spark-doris-connector-shims_spark-3.1
- spark-doris-connector-shims_spark-3.2
- spark-doris-connector-shims_spark-3.3
- spark-doris-connector-shims_spark-3.4
- spark-doris-connector-shims_spark-3.5
- spark-doris-connector-shims_spark-3-base
-
-
-
-
- org.apache.spark
- spark-sql_${scala.major.version}
- provided
-
-
-
-
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index d137bd51..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-import org.apache.spark.sql.sources.DataSourceRegister
-
-abstract class DorisDataSource extends DataSourceRegister with DorisSourceProvider with Serializable {
- override def shortName(): String = "doris"
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml
deleted file mode 100644
index 02675f98..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-
-
-
- 4.0.0
-
- org.apache.doris
- spark-doris-connector-shims
- ${revision}
- ../pom.xml
-
- jar
-
- spark-doris-connector-shims_spark-3.1
-
-
- 3.1.3
- 3.1
- 2.12.18
- 2.12
-
-
-
-
- org.apache.doris
- spark-doris-connector-shims_spark-3-base
- ${project.version}
-
-
-
-
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d1..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d1..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d1..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with DorisSourceProvider with Serializable {}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 1235f5b0..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait {}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml
deleted file mode 100644
index 42d6d7b6..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-
-
-
- 4.0.0
-
- org.apache.doris
- spark-doris-connector-shims
- ${revision}
- ../pom.xml
-
- jar
-
- spark-doris-connector-shims_spark-3.5
-
-
- 3.4.3
- 3.4
- 2.12.18
- 2.12
-
-
-
-
- org.apache.doris
- spark-doris-connector-shims_spark-3-base
-
-
-
-
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79d..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = throw new UnsupportedOperationException()
-
-}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
deleted file mode 100644
index 5450bbf2..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.read
-
-import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
-import org.apache.spark.sql.types.StructType
-
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 7bec3652..00000000
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait {}
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
similarity index 79%
rename from spark-doris-connector/spark-doris-connector-spark-3/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
index 4660d907..3d7499d0 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
@@ -26,22 +26,26 @@
spark-doris-connector
${revision}
- jar
- spark-doris-connector-spark-3
+ spark-doris-connector-spark-3-base
+
+
+ 8
+ 8
+ UTF-8
+ true
+
org.apache.doris
- spark-doris-connector-shims_spark-${spark.major.version}
- ${project.version}
+ spark-doris-connector-base
org.apache.spark
spark-sql_${scala.major.version}
- ${spark.version}
provided
-
\ No newline at end of file
+
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
similarity index 85%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
index d9aca8a9..4cd5841d 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
@@ -19,10 +19,8 @@ package org.apache.doris.spark.catalog
import org.apache.doris.spark.client.DorisFrontendClient
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
-import org.apache.doris.spark.read.DorisScanBuilder
import org.apache.doris.spark.rest.models.Schema
import org.apache.doris.spark.util.SchemaConvertors
-import org.apache.doris.spark.write.DorisWriteBuilder
import org.apache.spark.sql.connector.catalog.TableCapability._
import org.apache.spark.sql.connector.catalog.{Identifier, SupportsRead, SupportsWrite, Table, TableCapability}
import org.apache.spark.sql.connector.read.ScanBuilder
@@ -35,7 +33,7 @@ import scala.collection.JavaConverters._
import scala.language.implicitConversions
-class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) extends Table with SupportsRead with SupportsWrite {
+abstract class DorisTableBase(identifier: Identifier, config: DorisConfig, schema: Option[StructType]) extends Table with SupportsRead with SupportsWrite {
private lazy val frontend:DorisFrontendClient = new DorisFrontendClient(config)
@@ -54,12 +52,12 @@ class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[Str
override def newScanBuilder(caseInsensitiveStringMap: CaseInsensitiveStringMap): ScanBuilder = {
config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name())
- new DorisScanBuilder(config, schema())
+ createScanBuilder(config, schema())
}
override def newWriteBuilder(logicalWriteInfo: LogicalWriteInfo): WriteBuilder = {
config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name())
- new DorisWriteBuilder(config, logicalWriteInfo.schema())
+ createWriteBuilder(config, logicalWriteInfo.schema())
}
private implicit def dorisSchemaToStructType(dorisSchema: Schema): StructType = {
@@ -68,4 +66,8 @@ class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[Str
}))
}
+ protected def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder
+
+ protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder
+
}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
similarity index 89%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
index 9ca7f026..2c9ad0f9 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
@@ -20,7 +20,7 @@ package org.apache.doris.spark.catalog
import org.apache.doris.spark.client.DorisFrontendClient
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, Table, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange}
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -28,13 +28,13 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
import java.util
import scala.collection.JavaConverters._
-class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog {
+trait DorisTableCatalogBase extends TableCatalog with SupportsNamespaces {
- private var catalogName: Option[String] = None
+ protected var catalogName: Option[String] = None
- private var dorisConfig: DorisConfig = _
+ protected var dorisConfig: DorisConfig = _
- private var frontend: DorisFrontendClient = _
+ protected var frontend: DorisFrontendClient = _
override def name(): String = {
require(catalogName.nonEmpty, "The Doris table catalog is not initialed")
@@ -54,7 +54,7 @@ class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog {
override def loadTable(identifier: Identifier): Table = {
checkIdentifier(identifier)
- new DorisTable(identifier, DorisConfig.fromMap((dorisConfig.toMap.asScala +
+ newTableInstance(identifier, DorisConfig.fromMap((dorisConfig.toMap.asScala +
(DorisOptions.DORIS_TABLE_IDENTIFIER.getName -> getFullTableName(identifier))).asJava, false), None)
}
@@ -103,4 +103,6 @@ class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog {
(identifier.namespace() :+ identifier.name()).map(item => s"""`$item`""").mkString(".")
}
+ def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table
+
}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
similarity index 80%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
index 4fad7edd..a03435d5 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
@@ -15,9 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.sql.sources
+package org.apache.doris.spark.catalog
-import org.apache.doris.spark.catalog.DorisTable
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableProvider}
import org.apache.spark.sql.connector.expressions.Transform
@@ -26,9 +25,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
import java.util
-class DorisDataSourceV2 extends DorisDataSource with TableProvider {
+abstract class DorisTableProviderBase extends TableProvider {
- private var t: Table = _
+ protected var t: Table = _
override def inferSchema(options: CaseInsensitiveStringMap): StructType = {
if (t == null) t = getTable(options)
@@ -41,7 +40,7 @@ class DorisDataSourceV2 extends DorisDataSource with TableProvider {
val dorisConfig = DorisConfig.fromMap(properties, false)
val tableIdentifier = dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER)
val tableIdentifierArr = tableIdentifier.split("\\.")
- new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, Some(schema))
+ newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, Some(schema))
}
}
@@ -51,8 +50,10 @@ class DorisDataSourceV2 extends DorisDataSource with TableProvider {
val dorisConfig = DorisConfig.fromMap(options, false)
val tableIdentifier = dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER)
val tableIdentifierArr = tableIdentifier.split("\\.")
- new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, None)
+ newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)), tableIdentifierArr(1)), dorisConfig, None)
}
}
+ def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table;
+
}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
similarity index 94%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
index 2a01a6df..a6a97dc9 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownF
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
-protected[spark] abstract class DorisScanBuilderTrait(config: DorisConfig, schema: StructType) extends ScanBuilder
+protected[spark] abstract class DorisScanBuilderBase(config: DorisConfig, schema: StructType) extends ScanBuilder
with SupportsPushDownFilters
with SupportsPushDownRequiredColumns {
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
similarity index 100%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
similarity index 77%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
index f0105404..2dc1f38a 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
@@ -23,17 +23,18 @@
4.0.0
org.apache.doris
- spark-doris-connector-shims
+ spark-doris-connector
${revision}
- ../pom.xml
- jar
- spark-doris-connector-shims_spark-3.0
+ spark-doris-connector-spark-3.1
- 3.0.3
- 3.0
+ 8
+ 8
+ UTF-8
+ 3.1.0
+ 3.1
2.12.18
2.12
@@ -41,13 +42,11 @@
org.apache.doris
- spark-doris-connector-shims_spark-3-base
- ${project.version}
+ spark-doris-connector-spark-3-base
org.apache.spark
spark-sql_${scala.major.version}
- provided
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
rename from spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93f..986b2530 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
new file mode 100644
index 00000000..49a346a9
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.types.StructType
+
+class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+
+ override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf2..ca5b8c7f 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String]): Boolean = throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac1..9e199af5 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
new file mode 100644
index 00000000..102ac922
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.sql.sources
+
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with DorisSourceProvider with Serializable {
+
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
similarity index 72%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
index 68ae0932..a4d8670d 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
@@ -23,16 +23,17 @@
4.0.0
org.apache.doris
- spark-doris-connector-shims
+ spark-doris-connector
${revision}
- ../pom.xml
- jar
- spark-doris-connector-shims_spark-3.2
+ spark-doris-connector-spark-3.2
- 3.2.4
+ 8
+ 8
+ UTF-8
+ 3.2.0
3.2
2.12.18
2.12
@@ -41,7 +42,11 @@
org.apache.doris
- spark-doris-connector-shims_spark-3-base
+ spark-doris-connector-spark-3-base
+
+
+ org.apache.spark
+ spark-sql_${scala.major.version}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
new file mode 100644
index 00000000..986b2530
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
new file mode 100644
index 00000000..49a346a9
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.types.StructType
+
+class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+
+ override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
new file mode 100644
index 00000000..ca5b8c7f
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String]): Boolean = throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac1..9e199af5 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
new file mode 100644
index 00000000..102ac922
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.sql.sources
+
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with DorisSourceProvider with Serializable {
+
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
similarity index 72%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
index a33eb775..ecc71ed8 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
@@ -23,16 +23,17 @@
4.0.0
org.apache.doris
- spark-doris-connector-shims
+ spark-doris-connector
${revision}
- ../pom.xml
- jar
- spark-doris-connector-shims_spark-3.3
+ spark-doris-connector-spark-3.3
- 3.3.4
+ 8
+ 8
+ UTF-8
+ 3.3.0
3.3
2.12.18
2.12
@@ -41,7 +42,11 @@
org.apache.doris
- spark-doris-connector-shims_spark-3-base
+ spark-doris-connector-spark-3-base
+
+
+ org.apache.spark
+ spark-sql_${scala.major.version}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
new file mode 100644
index 00000000..986b2530
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
new file mode 100644
index 00000000..be461afc
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.types.StructType
+
+class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+
+ override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
+
+}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
new file mode 100644
index 00000000..e3ffcc92
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac1..9e199af5 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
new file mode 100644
index 00000000..c00cd240
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.sql.sources
+
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable {
+
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
similarity index 72%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
index b06e8bd5..eeee285e 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
@@ -23,16 +23,17 @@
4.0.0
org.apache.doris
- spark-doris-connector-shims
+ spark-doris-connector
${revision}
- ../pom.xml
- jar
- spark-doris-connector-shims_spark-3.4
+ spark-doris-connector-spark-3.4
- 3.4.3
+ 8
+ 8
+ UTF-8
+ 3.4.0
3.4
2.12.18
2.12
@@ -41,7 +42,11 @@
org.apache.doris
- spark-doris-connector-shims_spark-3-base
+ spark-doris-connector-spark-3-base
+
+
+ org.apache.spark
+ spark-sql_${scala.major.version}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
new file mode 100644
index 00000000..986b2530
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
new file mode 100644
index 00000000..be461afc
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.types.StructType
+
+class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+
+ override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
+
+}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
new file mode 100644
index 00000000..e3ffcc92
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 90%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index 5450bbf2..9e199af5 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -18,7 +18,6 @@
package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
new file mode 100644
index 00000000..c00cd240
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.sql.sources
+
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable {
+
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+
+}
diff --git a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
similarity index 70%
rename from spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
index 59293ed5..2f498b4d 100644
--- a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
@@ -23,17 +23,18 @@
4.0.0
org.apache.doris
- spark-doris-connector-shims
+ spark-doris-connector
${revision}
- ../pom.xml
- jar
- spark-doris-connector-shims_spark-3-base
+ spark-doris-connector-spark-3.5
- 3.0.3
- 3.0
+ 8
+ 8
+ UTF-8
+ 3.5.0
+ 3.5
2.12.18
2.12
@@ -41,7 +42,11 @@
org.apache.doris
- spark-doris-connector-base
+ spark-doris-connector-spark-3-base
+
+
+ org.apache.spark
+ spark-sql_${scala.major.version}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
new file mode 100644
index 00000000..986b2530
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
new file mode 100644
index 00000000..be461afc
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.types.StructType
+
+class DorisTable(identifier: Identifier, config: DorisConfig, schema: Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+
+ override def createScanBuilder(config: DorisConfig, schema: StructType): ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema: StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
+
+}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
new file mode 100644
index 00000000..e3ffcc92
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.catalog
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean = throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
new file mode 100644
index 00000000..9e199af5
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.read
+
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.types.StructType
+
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends DorisScanBuilderBase(config, schema) {}
diff --git a/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
new file mode 100644
index 00000000..c00cd240
--- /dev/null
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.sql.sources
+
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
+import org.apache.doris.spark.config.DorisConfig
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
+import org.apache.spark.sql.types.StructType
+
+class DorisDataSource extends DorisTableProviderBase with DorisSourceRegisterTrait with Serializable {
+
+ override def newTableInstance(identifier: Identifier, config: DorisConfig, schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+
+}