From 6634ff98aff47887328447cb638536a5e2e47d60 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Wed, 4 Jan 2023 20:53:52 +0800 Subject: [PATCH] Add source env list in common.properties (#13317) --- .../src/main/resources/common.properties | 5 +- .../plugin/task/api/ShellCommandExecutor.java | 12 ++ .../plugin/task/api/utils/ShellUtils.java | 37 +++++ .../plugin/task/api/utils/ShellUtilsTest.java | 32 +++++ .../src/test/resources/common.properties | 130 ++++++++++++++++++ 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtilsTest.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 4a962043f05d..0ea630f41293 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -124,4 +124,7 @@ ml.mlflow.preset_repository=https://github.com/apache/dolphinscheduler-mlflow ml.mlflow.preset_repository_version="main" # way to collect applicationId: log(original regex match), aop -appId.collect: log \ No newline at end of file +appId.collect=log + +# The default env list will be load by Shell task, e.g. /etc/profile,~/.bash_profile +shell.env_source_list= \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index ec9612220525..18799227b177 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -18,7 +18,9 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.api.utils.FileUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ShellUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; @@ -93,6 +95,11 @@ protected void createCommandFileIfNotExists(String execCommand, String commandFi if (SystemUtils.IS_OS_WINDOWS) { sb.append("@echo off").append(System.lineSeparator()); sb.append("cd /d %~dp0").append(System.lineSeparator()); + if (CollectionUtils.isNotEmpty(ShellUtils.ENV_SOURCE_LIST)) { + for (String envSourceFile : ShellUtils.ENV_SOURCE_LIST) { + sb.append("call ").append(envSourceFile).append("\n"); + } + } if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator()); } @@ -100,6 +107,11 @@ protected void createCommandFileIfNotExists(String execCommand, String commandFi sb.append("#!/bin/bash").append(System.lineSeparator()); sb.append("BASEDIR=$(cd `dirname $0`; pwd)").append(System.lineSeparator()); sb.append("cd $BASEDIR").append(System.lineSeparator()); + if (CollectionUtils.isNotEmpty(ShellUtils.ENV_SOURCE_LIST)) { + for (String envSourceFile : ShellUtils.ENV_SOURCE_LIST) { + sb.append("source ").append(envSourceFile).append("\n"); + } + } if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtils.java new file mode 100644 index 000000000000..7988bd045966 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import org.apache.dolphinscheduler.common.utils.PropertyUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ShellUtils { + + public List ENV_SOURCE_LIST = Arrays.stream( + Optional.ofNullable(PropertyUtils.getString("shell.env_source_list")) + .map(s -> s.split(",")).orElse(new String[0])) + .map(String::trim) + .collect(Collectors.toList()); +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtilsTest.java new file mode 100644 index 000000000000..c437edd076e1 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ShellUtilsTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import java.util.ArrayList; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ShellUtilsTest { + + @Test + public void testGetEnvSourceList() { + Assertions.assertEquals(new ArrayList<>(), ShellUtils.ENV_SOURCE_LIST); + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties new file mode 100644 index 000000000000..1f4d6779cf32 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties @@ -0,0 +1,130 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# resource view suffixs +#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js + +# resource storage type: HDFS, S3, OSS, NONE +resource.storage.type=NONE +# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended +resource.storage.upload.base.path=/dolphinscheduler + +# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.access.key.id=minioadmin +# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.secret.access.key=minioadmin +# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required +resource.aws.region=cn-north-1 +# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. +resource.aws.s3.bucket.name=dolphinscheduler +# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn +resource.aws.s3.endpoint=http://localhost:9000 + +# alibaba cloud access key id, required if you set resource.storage.type=OSS +resource.alibaba.cloud.access.key.id= +# alibaba cloud access key secret, required if you set resource.storage.type=OSS +resource.alibaba.cloud.access.key.secret= +# alibaba cloud region, required if you set resource.storage.type=OSS +resource.alibaba.cloud.region=cn-hangzhou +# oss bucket name, required if you set resource.storage.type=OSS +resource.alibaba.cloud.oss.bucket.name=dolphinscheduler +# oss bucket endpoint, required if you set resource.storage.type=OSS +resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com + +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path +resource.hdfs.root.user=hdfs +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +resource.hdfs.fs.defaultFS=hdfs://mycluster:8020 + +# whether to startup kerberos +hadoop.security.authentication.startup.state=false + +# java.security.krb5.conf path +java.security.krb5.conf.path=/opt/krb5.conf + +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM + +# login user from keytab path +login.user.keytab.path=/opt/hdfs.headless.keytab + +# kerberos expire time, the unit is hour +kerberos.expire.time=2 + + +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) +yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s + +# datasource encryption enable +datasource.encryption.enable=false + +# datasource encryption salt +datasource.encryption.salt=!@#$%^&* + +# data quality option +data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar + +#data-quality.error.output.path=/tmp/data-quality-error-data + +# Network IP gets priority, default inner outer + +# Whether hive SQL is executed in the same session +support.hive.oneSession=false + +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=true + +# network interface preferred like eth0, default: empty +#dolphin.scheduler.network.interface.preferred= + +# network IP gets priority, default: inner outer +#dolphin.scheduler.network.priority.strategy=default + +# system env path +#dolphinscheduler.env.path=dolphinscheduler_env.sh + +# development state +development.state=false + +# rpc port +alert.rpc.port=50052 + +# set path of conda.sh +conda.path=/opt/anaconda3/etc/profile.d/conda.sh + +# Task resource limit state +task.resource.limit.state=false + +# mlflow task plugin preset repository +ml.mlflow.preset_repository=https://github.com/apache/dolphinscheduler-mlflow +# mlflow task plugin preset repository version +ml.mlflow.preset_repository_version="main" + +# way to collect applicationId: log(original regex match), aop +appId.collect=log + +# The default env list will be load by Shell task, e.g. /etc/profile,~/.bash_profile +# shell.env_source_list=/etc/profile,~/.bash_profile \ No newline at end of file