Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Improvement-13476][*] Improve DS load hdfs configuration automatically by using environment variable HADOOP_CONF_DIR #13478

Closed
wants to merge 8 commits into from
Closed

[Improvement-13476][*] Improve DS load hdfs configuration automatically by using environment variable HADOOP_CONF_DIR #13478

Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ffe23da
[fix-13476][Api] Improve DS load hdfs configuration automatically by…
Feb 1, 2023
9a655d1
[fix-13476][Api] fix HdfsStorageOperator NullPointerException bug
Feb 1, 2023
a055906
[fix-13476][Api] fix the code style & add unit test
Feb 2, 2023
73157dc
[fix-13476][Api] fix the code style
Feb 2, 2023
95f3d47
update docs for HADOOP_HOME and HADOOP_CONF_DIR
Feb 3, 2023
3e5fd3f
add compatibility for hadoop2.2
Feb 3, 2023
836f326
remove controversial env variable JAVA_HOME
Feb 4, 2023
a5ac4bb
remove 'public' modifier in HdfsStorageOperatorTest
Feb 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions ...fs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public class HdfsStorageOperator implements Closeable, StorageOperate {

@Override
public HdfsStorageOperator load(String key) throws Exception {
return new HdfsStorageOperator(hdfsProperties);
return new HdfsStorageOperator();
}
});

Expand Down Expand Up @@ -132,39 +132,47 @@ private void initHdfsPath() {
private void init() throws NullPointerException {
try {
configuration = new HdfsConfiguration();

String hdfsUser = hdfsProperties.getUser();
String hadoopConfDir = System.getenv("HADOOP_CONF_DIR");
if(StringUtils.isNotBlank(hadoopConfDir)){
configuration.addResource(new Path(hadoopConfDir,"hdfs-site.xml"));
configuration.addResource(new Path(hadoopConfDir,"core-site.xml"));
}
boolean isKerberosAuthen = false;
if (CommonUtils.loadKerberosConf(configuration)) {
hdfsUser = "";
isKerberosAuthen = true;
}

String defaultFS = getDefaultFS();
// first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
// the default is the local file system
if (StringUtils.isNotBlank(defaultFS)) {
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs.");
configuration.set(Constants.HDFS_DEFAULT_FS, defaultFS);
fsRelatedProps.forEach((key, value) -> configuration.set(key, value));
// todo : question - Is this convenient for users to configure HDFS? And Whether it is convenient enough to configure environment variables `HADOOP_CONF_DIR`
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("resource.hdfs.fs.");
fsRelatedProps.forEach((key, value) -> configuration.set(key.substring(14), value));
} else {
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULT_FS);
throw new NullPointerException(
String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULT_FS));
}

if (!defaultFS.startsWith("file")) {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULT_FS,
logger.info("get property:{} -> {}, will use HDFS storage", Constants.FS_DEFAULT_FS,
defaultFS);
}

if (StringUtils.isNotEmpty(hdfsUser)) {
String hdfsUser = hdfsProperties.getUser();
if (!isKerberosAuthen && StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> {
fs = FileSystem.get(configuration);
return true;
});
} else {
logger.warn("resource.hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
if(!isKerberosAuthen){
logger.warn("resource.hdfs.root.user is not set value!");
}
}

} catch (Exception e) {
Expand All @@ -184,7 +192,8 @@ public Configuration getConfiguration() {
*/
public String getDefaultFS() {
String defaultFS = getConfiguration().get(Constants.FS_DEFAULT_FS);
if (StringUtils.isBlank(defaultFS)) {
if (StringUtils.isBlank(defaultFS) || defaultFS.equals("file:///")) {
logger.info("the property [{}] value from core-site.xml is [{}] . It is not the expected value, will use the config value in common.properties.", Constants.HDFS_DEFAULT_FS, defaultFS);
defaultFS = hdfsProperties.getDefaultFS();
}
return defaultFS;
Expand Down