[refractor](connector) Refactor module structure for release (#247)

apache · Dec 31, 2024 · 8f59bb6 · 8f59bb6
1 parent dd61d06
commit 8f59bb6
Show file tree

Hide file tree

Showing 71 changed files with 702 additions and 697 deletions.
diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml
@@ -44,42 +44,28 @@ jobs:
 
     - name: Build spark connector 2.4 2.11
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-2 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.11 -pl spark-doris-connector-spark-2 -am
 
     - name: Build spark connector 2.4 2.12
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Dscala.version=2.12.18 \
-          -Dscala.major.version=2.12 \
-          -Pspark-2 -pl spark-doris-connector-dist -am      
-
-    - name: Build spark connector 3.0
-      run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.0 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-2.4_2.12 -pl spark-doris-connector-spark-2 -am   
 
     - name: Build spark connector 3.1
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.1 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.1 -pl spark-doris-connector-spark-3.1 -am
 
     - name: Build spark connector 3.2
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.2 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.2 -pl spark-doris-connector-spark-3.2 -am
 
     - name: Build spark connector 3.3
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.3 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.3 -pl spark-doris-connector-spark-3.3 -am
 
     - name: Build spark connector 3.4
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.4 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.4 -pl spark-doris-connector-spark-3.4 -am
 
     - name: Build spark connector 3.5
       run: |
-        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
-          -Pspark-3,spark-3.5 -pl spark-doris-connector-dist -am
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.5 -pl spark-doris-connector-spark-3.5 -am
diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml
@@ -40,8 +40,8 @@ jobs:
 
     - name: Run ITCases for spark 2
       run: |
-        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+        cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
 
     - name: Run ITCases for spark 3
       run: |
-        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+        cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
diff --git a/.github/workflows/run-itcase-20.yml b/.github/workflows/run-itcase-20.yml
@@ -40,9 +40,9 @@ jobs:
 
     - name: Run ITCases for spark 2
       run: |
-        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+        cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
 
     - name: Run ITCases for spark 3
       run: |
-        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+        cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
     
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
@@ -142,32 +142,29 @@ selectScala() {
 
 selectSpark() {
   echo 'Spark-Doris-Connector supports multiple versions of spark. Which version do you need ?'
-  select spark in "2.4" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5"  "other"
+  select spark in "2.4" "3.1" "3.2" "3.3" "3.4" "3.5"  "other"
   do
     case $spark in
       "2.4")
         return 1
         ;;
-      "3.0")
-        return 2
-        ;;
       "3.1")
-        return 3
+        return 2
         ;;
       "3.2")
-        return 4
+        return 3
         ;;
       "3.3")
-        return 5
+        return 4
         ;;
       "3.4")
-        return 6
+        return 5
         ;;
       "3.5")
-        return 7
+        return 6
         ;;
       "other")
-        return 8
+        return 7
         ;;
     esac
   done
@@ -189,18 +186,16 @@ SparkVer=$?
 if [ ${SparkVer} -eq 1 ]; then
     SPARK_VERSION="2.4.8"
 elif [ ${SparkVer} -eq 2 ]; then
-    SPARK_VERSION="3.0.3"
-elif [ ${SparkVer} -eq 3 ]; then
     SPARK_VERSION="3.1.3"
-elif [ ${SparkVer} -eq 4 ]; then
+elif [ ${SparkVer} -eq 3 ]; then
     SPARK_VERSION="3.2.4"
-elif [ ${SparkVer} -eq 5 ]; then
+elif [ ${SparkVer} -eq 4 ]; then
     SPARK_VERSION="3.3.4"
-elif [ ${SparkVer} -eq 6 ]; then
+elif [ ${SparkVer} -eq 5 ]; then
     SPARK_VERSION="3.4.3"
-elif [ ${SparkVer} -eq 7 ]; then
+elif [ ${SparkVer} -eq 6 ]; then
     SPARK_VERSION="3.5.3"
-elif [ ${SparkVer} -eq 8 ]; then
+elif [ ${SparkVer} -eq 7 ]; then
     # shellcheck disable=SC2162
     read -p 'Which spark version do you need? please input
     :' ver
@@ -223,23 +218,23 @@ echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}"
 echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}"
 echo_g " build starting..."
 
-SPARK_PRIMARY_VERSION=0
-[ ${SPARK_MAJOR_VERSION} != 0 ] && SPARK_PRIMARY_VERSION=${SPARK_MAJOR_VERSION%.*}
+if [[ $SPARK_VERSION =~ ^3.* ]]; then
+  profile_name="spark-${SPARK_MAJOR_VERSION}"
+  module_suffix=${SPARK_MAJOR_VERSION}
+else
+  profile_name="spark-${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}"
+  module_suffix="2"
+fi
 
-${MVN_BIN} clean install \
-  -Dspark.version=${SPARK_VERSION} \
-  -Dscala.version=${SCALA_VERSION} \
-  -Dspark.major.version=${SPARK_MAJOR_VERSION} \
-  -Dscala.major.version=${SCALA_MAJOR_VERSION} \
-  -Pspark-${SPARK_PRIMARY_VERSION} -pl spark-doris-connector-dist -am "$@"
+${MVN_BIN} clean install -P"${profile_name}" -am "$@"
 
 EXIT_CODE=$?
 if [ $EXIT_CODE -eq 0 ]; then
   DIST_DIR=${DORIS_HOME}/dist
   [ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR"
-  dist_jar=$(ls "${ROOT}"/spark-doris-connector-dist/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
-  rm -rf "${DIST_DIR}"/spark-doris-connector-dist/"${dist_jar}"
-  cp "${ROOT}"/spark-doris-connector-dist/target/"${dist_jar}" "$DIST_DIR"
+  dist_jar=$(ls "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
+  rm -rf "${DIST_DIR}"/spark-doris-connector-spark-"${module_suffix}"/"${dist_jar}"
+  cp "${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target/"${dist_jar}" "$DIST_DIR"
 
   echo_g "*****************************************************************"
   echo_g "Successfully build Spark-Doris-Connector"

diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
@@ -41,11 +41,14 @@
     </licenses>
     <modules>
         <module>spark-doris-connector-base</module>
-        <module>spark-doris-connector-shims</module>
-        <module>spark-doris-connector-dist</module>
         <module>spark-doris-connector-spark-2</module>
-        <module>spark-doris-connector-spark-3</module>
         <module>spark-doris-connector-it</module>
+        <module>spark-doris-connector-spark-3-base</module>
+        <module>spark-doris-connector-spark-3.1</module>
+        <module>spark-doris-connector-spark-3.2</module>
+        <module>spark-doris-connector-spark-3.3</module>
+        <module>spark-doris-connector-spark-3.4</module>
+        <module>spark-doris-connector-spark-3.5</module>
     </modules>
 
     <scm>
@@ -92,6 +95,7 @@
         <jmockit.version>1.49</jmockit.version>
         <fe_ut_parallel>1</fe_ut_parallel>
         <argLine>-Xmx512m</argLine>
+        <maven.deploy.skip>true</maven.deploy.skip>
     </properties>
 
     <dependencyManagement>
@@ -108,20 +112,34 @@
             </dependency>
             <dependency>
                 <groupId>org.apache.doris</groupId>
-                <artifactId>spark-doris-connector-spark-3</artifactId>
+                <artifactId>spark-doris-connector-spark-3-base</artifactId>
                 <version>${project.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.doris</groupId>
-                <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+                <artifactId>spark-doris-connector-spark-3.1</artifactId>
                 <version>${project.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.doris</groupId>
-                <artifactId>spark-doris-connector-shims_spark-${spark.major.version}</artifactId>
+                <artifactId>spark-doris-connector-spark-3.2</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.doris</groupId>
+                <artifactId>spark-doris-connector-spark-3.3</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.doris</groupId>
+                <artifactId>spark-doris-connector-spark-3.4</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.doris</groupId>
+                <artifactId>spark-doris-connector-spark-3.5</artifactId>
                 <version>${project.version}</version>
             </dependency>
-
             <dependency>
                 <groupId>org.apache.doris</groupId>
                 <artifactId>thrift-service</artifactId>
@@ -764,7 +782,7 @@
                             <!-->not reuse forked jvm, so that each unit test will run in separate jvm. to avoid singleton confict<-->
                             <reuseForks>false</reuseForks>
                             <argLine>
-                                -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine}
+                                -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar --add-opens=java.base/java.nio=ALL-UNNAMED @{argLine}
                             </argLine>
                         </configuration>
                     </plugin>
@@ -773,22 +791,19 @@
         </profile>
 
         <profile>
-            <id>spark-2</id>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
+            <id>spark-2.4_2.11</id>
             <properties>
                 <spark.version>2.4.8</spark.version>
                 <spark.major.version>2.4</spark.major.version>
-                <scala.version>2.11.10</scala.version>
+                <scala.version>2.11.12</scala.version>
                 <scala.major.version>2.11</scala.major.version>
             </properties>
         </profile>
         <profile>
-            <id>spark-3.0</id>
+            <id>spark-2.4_2.12</id>
             <properties>
-                <spark.version>3.0.0</spark.version>
-                <spark.major.version>3.0</spark.major.version>
+                <spark.version>2.4.8</spark.version>
+                <spark.major.version>2.4</spark.major.version>
                 <scala.version>2.12.18</scala.version>
                 <scala.major.version>2.12</scala.major.version>
             </properties>
@@ -813,6 +828,9 @@
         </profile>
         <profile>
             <id>spark-3.3</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
             <properties>
                 <spark.version>3.3.0</spark.version>
                 <spark.major.version>3.3</spark.major.version>

diff --git a/spark-doris-connector/spark-doris-connector-base/pom.xml b/spark-doris-connector/spark-doris-connector-base/pom.xml
@@ -35,6 +35,7 @@
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.deploy.skip>true</maven.deploy.skip>
     </properties>
 
     <dependencies>
@@ -296,4 +297,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/...park-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala b/...park-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
@@ -38,7 +38,7 @@ import java.io.{ByteArrayOutputStream, IOException}
 import java.nio.charset.StandardCharsets
 import java.util.zip.GZIPOutputStream
 import java.util.{Base64, Properties, UUID}
-import scala.collection.JavaConverters._
+import scala.collection.JavaConverters.{asJavaIteratorConverter, mapAsJavaMapConverter, propertiesAsScalaMapConverter}
 import scala.util.{Failure, Success, Try}
 
 case class CopyIntoResponse(code: Int, msg: String, content: String)
@@ -234,7 +234,7 @@ class CopyIntoLoader(settings: SparkSettings, isStreaming: Boolean) extends Load
     }
     props.remove("columns")
     val properties = new Properties()
-    properties.putAll(props.mapValues(_.toString).asJava)
+    props.foreach(p => properties.setProperty(p._1, p._2))
     properties
   }
 

diff --git a/...rk-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala b/...rk-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
@@ -67,7 +67,7 @@ object SchemaConvertors {
   def convertToSchema(tscanColumnDescs: Seq[TScanColumnDesc]): Schema = {
     val schema = new Schema(tscanColumnDescs.length)
     tscanColumnDescs.foreach(desc => {
-      println(desc.getName + " " + desc.getType.name())
+      // println(desc.getName + " " + desc.getType.name())
       schema.put(new Field(desc.getName, desc.getType.name, "", 0, 0, ""))
     })
     schema