Add source env list in common.properties (#13317)

dolphinscheduler-common/src/main/resources/common.properties

@@ -124,4 +124,7 @@ ml.mlflow.preset_repository=https://github.com/apache/dolphinscheduler-mlflow
 ml.mlflow.preset_repository_version="main"
 
 # way to collect applicationId: log(original regex match), aop
-appId.collect: log
+appId.collect=log
+
+# The default env list will be load by Shell task, e.g. /etc/profile,~/.bash_profile
+shell.env_source_list=

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java

@@ -18,7 +18,9 @@
 package org.apache.dolphinscheduler.plugin.task.api;
 
 import org.apache.dolphinscheduler.plugin.task.api.utils.FileUtils;
+import org.apache.dolphinscheduler.plugin.task.api.utils.ShellUtils;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.SystemUtils;
 
@@ -93,13 +95,23 @@ protected void createCommandFileIfNotExists(String execCommand, String commandFi
         if (SystemUtils.IS_OS_WINDOWS) {
             sb.append("@echo off").append(System.lineSeparator());
             sb.append("cd /d %~dp0").append(System.lineSeparator());
+            if (CollectionUtils.isNotEmpty(ShellUtils.ENV_SOURCE_LIST)) {
+                for (String envSourceFile : ShellUtils.ENV_SOURCE_LIST) {
+                    sb.append("call ").append(envSourceFile).append("\n");
+                }
+            }
             if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) {
                 sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator());
             }
         } else {
             sb.append("#!/bin/bash").append(System.lineSeparator());
             sb.append("BASEDIR=$(cd `dirname $0`; pwd)").append(System.lineSeparator());
             sb.append("cd $BASEDIR").append(System.lineSeparator());
+            if (CollectionUtils.isNotEmpty(ShellUtils.ENV_SOURCE_LIST)) {
+                for (String envSourceFile : ShellUtils.ENV_SOURCE_LIST) {
+                    sb.append("source ").append(envSourceFile).append("\n");
+                }
+            }
             if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) {
                 sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator());
             }

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtils.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.task.api.utils;
+
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class ShellUtils {
+
+    public List<String> ENV_SOURCE_LIST = Arrays.stream(
+            Optional.ofNullable(PropertyUtils.getString("shell.env_source_list"))
+                    .map(s -> s.split(",")).orElse(new String[0]))
+            .map(String::trim)
+            .collect(Collectors.toList());
+}

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtilsTest.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.task.api.utils;
+
+import java.util.ArrayList;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ShellUtilsTest {
+
+    @Test
+    public void testGetEnvSourceList() {
+        Assertions.assertEquals(new ArrayList<>(), ShellUtils.ENV_SOURCE_LIST);
+    }
+
+}

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties

@@ -0,0 +1,130 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# user data local directory path, please make sure the directory exists and have read write permissions
+data.basedir.path=/tmp/dolphinscheduler
+
+# resource view suffixs
+#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
+
+# resource storage type: HDFS, S3, OSS, NONE
+resource.storage.type=NONE
+# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
+resource.storage.upload.base.path=/dolphinscheduler
+
+# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.access.key.id=minioadmin
+# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.secret.access.key=minioadmin
+# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
+resource.aws.region=cn-north-1
+# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
+resource.aws.s3.bucket.name=dolphinscheduler
+# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
+resource.aws.s3.endpoint=http://localhost:9000
+
+# alibaba cloud access key id, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.access.key.id=<your-access-key-id>
+# alibaba cloud access key secret, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.access.key.secret=<your-access-key-secret>
+# alibaba cloud region, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.region=cn-hangzhou
+# oss bucket name, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
+# oss bucket endpoint, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
+
+# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
+resource.hdfs.root.user=hdfs
+# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
+resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
+
+# whether to startup kerberos
+hadoop.security.authentication.startup.state=false
+
+# java.security.krb5.conf path
+java.security.krb5.conf.path=/opt/krb5.conf
+
+# login user from keytab username
+login.user.keytab.username=[email protected]
+
+# login user from keytab path
+login.user.keytab.path=/opt/hdfs.headless.keytab
+
+# kerberos expire time, the unit is hour
+kerberos.expire.time=2
+
+
+# resourcemanager port, the default value is 8088 if not specified
+resource.manager.httpaddress.port=8088
+# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
+yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
+# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
+yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
+# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
+yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
+
+# datasource encryption enable
+datasource.encryption.enable=false
+
+# datasource encryption salt
+datasource.encryption.salt=!@#$%^&*
+
+# data quality option
+data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+
+#data-quality.error.output.path=/tmp/data-quality-error-data
+
+# Network IP gets priority, default inner outer
+
+# Whether hive SQL is executed in the same session
+support.hive.oneSession=false
+
+# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
+sudo.enable=true
+
+# network interface preferred like eth0, default: empty
+#dolphin.scheduler.network.interface.preferred=
+
+# network IP gets priority, default: inner outer
+#dolphin.scheduler.network.priority.strategy=default
+
+# system env path
+#dolphinscheduler.env.path=dolphinscheduler_env.sh
+
+# development state
+development.state=false
+
+# rpc port
+alert.rpc.port=50052
+
+# set path of conda.sh
+conda.path=/opt/anaconda3/etc/profile.d/conda.sh
+
+# Task resource limit state
+task.resource.limit.state=false
+
+# mlflow task plugin preset repository
+ml.mlflow.preset_repository=https://github.com/apache/dolphinscheduler-mlflow
+# mlflow task plugin preset repository version
+ml.mlflow.preset_repository_version="main"
+
+# way to collect applicationId: log(original regex match), aop
+appId.collect=log
+
+# The default env list will be load by Shell task, e.g. /etc/profile,~/.bash_profile
+# shell.env_source_list=/etc/profile,~/.bash_profile