diff --git a/bin/start_be.sh b/bin/start_be.sh
index 5f17fefca46e86..6eead1c1e467b8 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -78,7 +78,11 @@ if [[ "${MAX_FILE_COUNT}" -lt 65536 ]]; then
fi
# add java libs
-for f in "${DORIS_HOME}/lib/java_extensions/java-udf"/*.jar; do
+preload_jars=("preload-extensions")
+preload_jars+=("java-udf")
+
+for preload_jar in "${preload_jars[@]}"; do
+ f="${DORIS_HOME}/lib/java_extensions/${preload_jar}/${preload_jar}-jar-with-dependencies.jar"
if [[ -z "${DORIS_CLASSPATH}" ]]; then
export DORIS_CLASSPATH="${f}"
else
diff --git a/build.sh b/build.sh
index 92f98f31a9553d..776909bec4f983 100755
--- a/build.sh
+++ b/build.sh
@@ -457,6 +457,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then
modules+=("be-java-extensions/paimon-scanner")
modules+=("be-java-extensions/max-compute-scanner")
modules+=("be-java-extensions/avro-scanner")
+ modules+=("be-java-extensions/preload-extensions")
fi
FE_MODULES="$(
IFS=','
@@ -673,6 +674,7 @@ EOF
extensions_modules+=("paimon-scanner")
extensions_modules+=("max-compute-scanner")
extensions_modules+=("avro-scanner")
+ extensions_modules+=("preload-extensions")
BE_JAVA_EXTENSIONS_DIR="${DORIS_OUTPUT}/be/lib/java_extensions/"
rm -rf "${BE_JAVA_EXTENSIONS_DIR}"
diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/be-java-extensions/avro-scanner/pom.xml
index bb2ff671877907..f95fa947051336 100644
--- a/fe/be-java-extensions/avro-scanner/pom.xml
+++ b/fe/be-java-extensions/avro-scanner/pom.xml
@@ -41,16 +41,6 @@ under the License.
java-common
${project.version}
-
- org.apache.avro
- avro
-
-
- org.apache.avro
- avro-tools
-
-
-
org.apache.hadoop
hadoop-client
@@ -73,6 +63,7 @@ under the License.
org.apache.doris
hive-catalog-shade
+ provided
diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml
index bb115ae245a145..32d3ccc4b4f5da 100644
--- a/fe/be-java-extensions/hudi-scanner/pom.xml
+++ b/fe/be-java-extensions/hudi-scanner/pom.xml
@@ -42,27 +42,47 @@ under the License.
org.scala-lang
scala-library
${scala.version}
- compile
+ provided
+
+
+ org.apache.hadoop
+ hadoop-common
+ provided
org.apache.hudi
- hudi-spark-common_${scala.binary.version}
+ hudi-spark-client
${hudi.version}
+ provided
org.apache.hudi
- hudi-spark-client
+ hudi-spark-common_${scala.binary.version}
${hudi.version}
+ provided
+
+
+ org.apache.avro
+ avro
+ provided
+
+
+ org.apache.avro
+ avro-tools
+
+
org.apache.hudi
hudi-spark3-common
${hudi.version}
+ provided
org.apache.hudi
hudi-spark3.2.x_${scala.binary.version}
${hudi.version}
+ provided
json4s-ast_2.11
@@ -86,7 +106,7 @@ under the License.
org.apache.parquet
parquet-avro
1.10.1
- compile
+ provided
org.apache.spark
@@ -110,13 +130,25 @@ under the License.
${spark.version}
- compile
+ provided
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark.version}
+ provided
+
+
+ org.apache.spark
+ spark-launcher_${scala.binary.version}
+ ${spark.version}
+ provided
org.apache.spark
spark-catalyst_${scala.binary.version}
${spark.version}
- compile
+ provided
org.codehaus.janino
@@ -128,23 +160,12 @@ under the License.
-
- org.apache.spark
- spark-sql_${scala.binary.version}
- ${spark.version}
- compile
-
-
- org.apache.spark
- spark-launcher_${scala.binary.version}
- ${spark.version}
- compile
-
org.codehaus.janino
janino
${janino.version}
+ provided
org.codehaus.janino
@@ -156,12 +177,14 @@ under the License.
org.codehaus.janino
commons-compiler
${janino.version}
+ provided
com.fasterxml.jackson.module
jackson-module-scala_${scala.binary.version}
${jackson.version}
+ provided
com.google.guava
@@ -169,15 +192,16 @@ under the License.
-
org.apache.doris
java-common
${project.version}
-
-
- org.apache.hadoop
- hadoop-common
+
+
+ org.apache.thrift
+ libthrift
+
+
diff --git a/fe/be-java-extensions/jdbc-scanner/pom.xml b/fe/be-java-extensions/jdbc-scanner/pom.xml
index 2b62dbe4101157..0f3eace7dda8d4 100644
--- a/fe/be-java-extensions/jdbc-scanner/pom.xml
+++ b/fe/be-java-extensions/jdbc-scanner/pom.xml
@@ -43,16 +43,24 @@ under the License.
com.oracle.database.jdbc
ojdbc8
+ provided
-
com.alibaba
druid
+ provided
com.clickhouse
clickhouse-jdbc
all
+ provided
+
+
+ com.oracle.ojdbc
+ orai18n
+ 19.3.0.0
+ provided
diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml
index f1ffe1527f9248..af8584d17760dc 100644
--- a/fe/be-java-extensions/pom.xml
+++ b/fe/be-java-extensions/pom.xml
@@ -28,6 +28,7 @@ under the License.
paimon-scanner
max-compute-scanner
avro-scanner
+ preload-extensions
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml b/fe/be-java-extensions/preload-extensions/pom.xml
new file mode 100644
index 00000000000000..db8663212aa4e4
--- /dev/null
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -0,0 +1,247 @@
+
+
+
+
+ be-java-extensions
+ org.apache.doris
+ ${revision}
+
+ 4.0.0
+
+ preload-extensions
+
+
+ 8
+ 8
+ 2.12
+ 3.2.0
+ 3.0.16
+
+
+
+
+ org.apache.arrow
+ arrow-memory-unsafe
+ ${arrow.version}
+ compile
+
+
+ org.apache.parquet
+ parquet-avro
+ 1.10.1
+ compile
+
+
+ org.scala-lang
+ scala-library
+ ${scala.version}
+ compile
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hudi
+ hudi-spark-client
+ ${hudi.version}
+
+
+ org.apache.hudi
+ hudi-spark-common_${scala.binary.version}
+ ${hudi.version}
+
+
+ org.apache.avro
+ avro
+
+
+ org.apache.avro
+ avro-tools
+
+
+
+
+ org.apache.hudi
+ hudi-spark3-common
+ ${hudi.version}
+
+
+ org.apache.hudi
+ hudi-spark3.2.x_${scala.binary.version}
+ ${hudi.version}
+
+
+ json4s-ast_2.11
+ org.json4s
+
+
+ json4s-core_2.11
+ org.json4s
+
+
+ json4s-jackson_2.11
+ org.json4s
+
+
+ json4s-scalap_2.11
+ org.json4s
+
+
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+
+
+ javax.servlet
+ *
+
+
+ jackson-module-scala_2.12
+ com.fasterxml.jackson.module
+
+
+ hadoop-client-api
+ org.apache.hadoop
+
+
+ hadoop-client-runtime
+ org.apache.hadoop
+
+
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-launcher_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-catalyst_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.codehaus.janino
+ janino
+
+
+ org.codehaus.janino
+ commons-compiler
+
+
+
+
+
+ org.codehaus.janino
+ janino
+ ${janino.version}
+
+
+ org.codehaus.janino
+ commons-compiler
+
+
+
+
+ org.codehaus.janino
+ commons-compiler
+ ${janino.version}
+
+
+
+ com.fasterxml.jackson.module
+ jackson-module-scala_${scala.binary.version}
+ ${jackson.version}
+
+
+ com.google.guava
+ guava
+
+
+
+
+
+ com.oracle.database.jdbc
+ ojdbc8
+
+
+ com.alibaba
+ druid
+
+
+ com.clickhouse
+ clickhouse-jdbc
+ all
+ compile
+
+
+ com.oracle.ojdbc
+ orai18n
+ 19.3.0.0
+
+
+ org.apache.doris
+ hive-catalog-shade
+
+
+
+
+ preload-extensions
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+
+ src/main/resources/package.xml
+
+
+
+
+
+
+
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md
new file mode 100644
index 00000000000000..6857726c5bb62c
--- /dev/null
+++ b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md
@@ -0,0 +1,217 @@
+- [Preload Dependencies For BE Extensions](#Preload-Dependencies-For-BE-Extensions)
+ - [Avro Scanner](#Avro-Scanner)
+ - [Hudi Scanner](#Hudi-Scanner)
+ - [MaxCompute Scanner](#MaxCompute-Scanner)
+ - [Paimon Scanner](#Paimon-Scanner)
+ - [JDBC Scanner](#JDBC-Scanner)
+
+# Preload Dependencies For BE Extensions
+
+## Avro Scanner
+
+Avro Scanner Compile Dependencies:
+
+```
+
+ org.apache.avro
+ avro
+
+
+ org.apache.avro
+ avro-tools
+
+
+
+```
+
+## Hudi Scanner
+
+Hudi Scanner Compile Dependencies:
+
+```
+
+ org.apache.parquet
+ parquet-avro
+ 1.10.1
+ compile
+
+
+ org.scala-lang
+ scala-library
+ ${scala.version}
+ compile
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hudi
+ hudi-spark-client
+ ${hudi.version}
+
+
+ org.apache.hudi
+ hudi-spark-common_${scala.binary.version}
+ ${hudi.version}
+
+
+ org.apache.hudi
+ hudi-spark3-common
+ ${hudi.version}
+
+
+ org.apache.hudi
+ hudi-spark3.2.x_${scala.binary.version}
+ ${hudi.version}
+
+
+ json4s-ast_2.11
+ org.json4s
+
+
+ json4s-core_2.11
+ org.json4s
+
+
+ json4s-jackson_2.11
+ org.json4s
+
+
+ json4s-scalap_2.11
+ org.json4s
+
+
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+
+
+ javax.servlet
+ *
+
+
+ jackson-module-scala_2.12
+ com.fasterxml.jackson.module
+
+
+ hadoop-client-api
+ org.apache.hadoop
+
+
+ hadoop-client-runtime
+ org.apache.hadoop
+
+
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-launcher_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.apache.spark
+ spark-catalyst_${scala.binary.version}
+ ${spark.version}
+ compile
+
+
+ org.codehaus.janino
+ janino
+
+
+ org.codehaus.janino
+ commons-compiler
+
+
+
+
+
+ org.codehaus.janino
+ janino
+ ${janino.version}
+
+
+ org.codehaus.janino
+ commons-compiler
+
+
+
+
+ org.codehaus.janino
+ commons-compiler
+ ${janino.version}
+
+
+
+ com.fasterxml.jackson.module
+ jackson-module-scala_${scala.binary.version}
+ ${jackson.version}
+
+
+ com.google.guava
+ guava
+
+
+
+```
+
+
+## MaxCompute Scanner
+
+MaxCompute Scanner Compile Dependencies:
+
+```
+
+ org.apache.arrow
+ arrow-memory-unsafe
+ ${arrow.version}
+ compile
+
+```
+
+## Paimon Scanner
+
+```
+
+```
+
+## JDBC Scanner
+
+JDBC Scanner Compile Dependencies:
+
+```
+
+ com.oracle.database.jdbc
+ ojdbc8
+
+
+ com.alibaba
+ druid
+
+
+ com.clickhouse
+ clickhouse-jdbc
+ all
+ compile
+
+
+ com.oracle.ojdbc
+ orai18n
+ 19.3.0.0
+
+
+ org.apache.doris
+ hive-catalog-shade
+
+```
\ No newline at end of file
diff --git a/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml b/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml
new file mode 100644
index 00000000000000..4bbb2610603363
--- /dev/null
+++ b/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml
@@ -0,0 +1,41 @@
+
+
+
+ jar-with-dependencies
+
+ jar
+
+ false
+
+
+ /
+ true
+ true
+ runtime
+
+
+ **/Log4j2Plugins.dat
+
+
+
+
+