diff --git a/bin/start_be.sh b/bin/start_be.sh index 5f17fefca46e86..6eead1c1e467b8 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -78,7 +78,11 @@ if [[ "${MAX_FILE_COUNT}" -lt 65536 ]]; then fi # add java libs -for f in "${DORIS_HOME}/lib/java_extensions/java-udf"/*.jar; do +preload_jars=("preload-extensions") +preload_jars+=("java-udf") + +for preload_jar in "${preload_jars[@]}"; do + f="${DORIS_HOME}/lib/java_extensions/${preload_jar}/${preload_jar}-jar-with-dependencies.jar" if [[ -z "${DORIS_CLASSPATH}" ]]; then export DORIS_CLASSPATH="${f}" else diff --git a/build.sh b/build.sh index 92f98f31a9553d..776909bec4f983 100755 --- a/build.sh +++ b/build.sh @@ -457,6 +457,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/paimon-scanner") modules+=("be-java-extensions/max-compute-scanner") modules+=("be-java-extensions/avro-scanner") + modules+=("be-java-extensions/preload-extensions") fi FE_MODULES="$( IFS=',' @@ -673,6 +674,7 @@ EOF extensions_modules+=("paimon-scanner") extensions_modules+=("max-compute-scanner") extensions_modules+=("avro-scanner") + extensions_modules+=("preload-extensions") BE_JAVA_EXTENSIONS_DIR="${DORIS_OUTPUT}/be/lib/java_extensions/" rm -rf "${BE_JAVA_EXTENSIONS_DIR}" diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/be-java-extensions/avro-scanner/pom.xml index bb2ff671877907..f95fa947051336 100644 --- a/fe/be-java-extensions/avro-scanner/pom.xml +++ b/fe/be-java-extensions/avro-scanner/pom.xml @@ -41,16 +41,6 @@ under the License. java-common ${project.version} - - org.apache.avro - avro - - - org.apache.avro - avro-tools - - - org.apache.hadoop hadoop-client @@ -73,6 +63,7 @@ under the License. org.apache.doris hive-catalog-shade + provided diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml index bb115ae245a145..32d3ccc4b4f5da 100644 --- a/fe/be-java-extensions/hudi-scanner/pom.xml +++ b/fe/be-java-extensions/hudi-scanner/pom.xml @@ -42,27 +42,47 @@ under the License. org.scala-lang scala-library ${scala.version} - compile + provided + + + org.apache.hadoop + hadoop-common + provided org.apache.hudi - hudi-spark-common_${scala.binary.version} + hudi-spark-client ${hudi.version} + provided org.apache.hudi - hudi-spark-client + hudi-spark-common_${scala.binary.version} ${hudi.version} + provided + + + org.apache.avro + avro + provided + + + org.apache.avro + avro-tools + + org.apache.hudi hudi-spark3-common ${hudi.version} + provided org.apache.hudi hudi-spark3.2.x_${scala.binary.version} ${hudi.version} + provided json4s-ast_2.11 @@ -86,7 +106,7 @@ under the License. org.apache.parquet parquet-avro 1.10.1 - compile + provided org.apache.spark @@ -110,13 +130,25 @@ under the License. ${spark.version} - compile + provided + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + provided + + + org.apache.spark + spark-launcher_${scala.binary.version} + ${spark.version} + provided org.apache.spark spark-catalyst_${scala.binary.version} ${spark.version} - compile + provided org.codehaus.janino @@ -128,23 +160,12 @@ under the License. - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - compile - - - org.apache.spark - spark-launcher_${scala.binary.version} - ${spark.version} - compile - org.codehaus.janino janino ${janino.version} + provided org.codehaus.janino @@ -156,12 +177,14 @@ under the License. org.codehaus.janino commons-compiler ${janino.version} + provided com.fasterxml.jackson.module jackson-module-scala_${scala.binary.version} ${jackson.version} + provided com.google.guava @@ -169,15 +192,16 @@ under the License. - org.apache.doris java-common ${project.version} - - - org.apache.hadoop - hadoop-common + + + org.apache.thrift + libthrift + + diff --git a/fe/be-java-extensions/jdbc-scanner/pom.xml b/fe/be-java-extensions/jdbc-scanner/pom.xml index 2b62dbe4101157..0f3eace7dda8d4 100644 --- a/fe/be-java-extensions/jdbc-scanner/pom.xml +++ b/fe/be-java-extensions/jdbc-scanner/pom.xml @@ -43,16 +43,24 @@ under the License. com.oracle.database.jdbc ojdbc8 + provided - com.alibaba druid + provided com.clickhouse clickhouse-jdbc all + provided + + + com.oracle.ojdbc + orai18n + 19.3.0.0 + provided diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index f1ffe1527f9248..af8584d17760dc 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -28,6 +28,7 @@ under the License. paimon-scanner max-compute-scanner avro-scanner + preload-extensions diff --git a/fe/be-java-extensions/preload-extensions/pom.xml b/fe/be-java-extensions/preload-extensions/pom.xml new file mode 100644 index 00000000000000..db8663212aa4e4 --- /dev/null +++ b/fe/be-java-extensions/preload-extensions/pom.xml @@ -0,0 +1,247 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + preload-extensions + + + 8 + 8 + 2.12 + 3.2.0 + 3.0.16 + + + + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + compile + + + org.apache.parquet + parquet-avro + 1.10.1 + compile + + + org.scala-lang + scala-library + ${scala.version} + compile + + + + org.apache.hadoop + hadoop-common + + + org.apache.hudi + hudi-spark-client + ${hudi.version} + + + org.apache.hudi + hudi-spark-common_${scala.binary.version} + ${hudi.version} + + + org.apache.avro + avro + + + org.apache.avro + avro-tools + + + + + org.apache.hudi + hudi-spark3-common + ${hudi.version} + + + org.apache.hudi + hudi-spark3.2.x_${scala.binary.version} + ${hudi.version} + + + json4s-ast_2.11 + org.json4s + + + json4s-core_2.11 + org.json4s + + + json4s-jackson_2.11 + org.json4s + + + json4s-scalap_2.11 + org.json4s + + + + + org.apache.spark + spark-core_${scala.binary.version} + + + javax.servlet + * + + + jackson-module-scala_2.12 + com.fasterxml.jackson.module + + + hadoop-client-api + org.apache.hadoop + + + hadoop-client-runtime + org.apache.hadoop + + + ${spark.version} + compile + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-launcher_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + compile + + + org.codehaus.janino + janino + + + org.codehaus.janino + commons-compiler + + + + + + org.codehaus.janino + janino + ${janino.version} + + + org.codehaus.janino + commons-compiler + + + + + org.codehaus.janino + commons-compiler + ${janino.version} + + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.binary.version} + ${jackson.version} + + + com.google.guava + guava + + + + + + com.oracle.database.jdbc + ojdbc8 + + + com.alibaba + druid + + + com.clickhouse + clickhouse-jdbc + all + compile + + + com.oracle.ojdbc + orai18n + 19.3.0.0 + + + org.apache.doris + hive-catalog-shade + + + + + preload-extensions + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md new file mode 100644 index 00000000000000..6857726c5bb62c --- /dev/null +++ b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/README.md @@ -0,0 +1,217 @@ +- [Preload Dependencies For BE Extensions](#Preload-Dependencies-For-BE-Extensions) + - [Avro Scanner](#Avro-Scanner) + - [Hudi Scanner](#Hudi-Scanner) + - [MaxCompute Scanner](#MaxCompute-Scanner) + - [Paimon Scanner](#Paimon-Scanner) + - [JDBC Scanner](#JDBC-Scanner) + +# Preload Dependencies For BE Extensions + +## Avro Scanner + +Avro Scanner Compile Dependencies: + +``` + + org.apache.avro + avro + + + org.apache.avro + avro-tools + + + +``` + +## Hudi Scanner + +Hudi Scanner Compile Dependencies: + +``` + + org.apache.parquet + parquet-avro + 1.10.1 + compile + + + org.scala-lang + scala-library + ${scala.version} + compile + + + org.apache.hadoop + hadoop-common + + + org.apache.hudi + hudi-spark-client + ${hudi.version} + + + org.apache.hudi + hudi-spark-common_${scala.binary.version} + ${hudi.version} + + + org.apache.hudi + hudi-spark3-common + ${hudi.version} + + + org.apache.hudi + hudi-spark3.2.x_${scala.binary.version} + ${hudi.version} + + + json4s-ast_2.11 + org.json4s + + + json4s-core_2.11 + org.json4s + + + json4s-jackson_2.11 + org.json4s + + + json4s-scalap_2.11 + org.json4s + + + + + org.apache.spark + spark-core_${scala.binary.version} + + + javax.servlet + * + + + jackson-module-scala_2.12 + com.fasterxml.jackson.module + + + hadoop-client-api + org.apache.hadoop + + + hadoop-client-runtime + org.apache.hadoop + + + ${spark.version} + compile + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-launcher_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + compile + + + org.codehaus.janino + janino + + + org.codehaus.janino + commons-compiler + + + + + + org.codehaus.janino + janino + ${janino.version} + + + org.codehaus.janino + commons-compiler + + + + + org.codehaus.janino + commons-compiler + ${janino.version} + + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.binary.version} + ${jackson.version} + + + com.google.guava + guava + + + +``` + + +## MaxCompute Scanner + +MaxCompute Scanner Compile Dependencies: + +``` + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + compile + +``` + +## Paimon Scanner + +``` + +``` + +## JDBC Scanner + +JDBC Scanner Compile Dependencies: + +``` + + com.oracle.database.jdbc + ojdbc8 + + + com.alibaba + druid + + + com.clickhouse + clickhouse-jdbc + all + compile + + + com.oracle.ojdbc + orai18n + 19.3.0.0 + + + org.apache.doris + hive-catalog-shade + +``` \ No newline at end of file diff --git a/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml b/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml new file mode 100644 index 00000000000000..4bbb2610603363 --- /dev/null +++ b/fe/be-java-extensions/preload-extensions/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + +