forked from apache/gravitino
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[apache#3337] improvement(hadoop-catalog): Support user impersonation…
… for Hadoop catalog. (apache#3352) ### What changes were proposed in this pull request? Add user impersonation for the Hadoop catalog. ### Why are the changes needed? We need authentication for the encrypted HDFS cluster. Fix: apache#3337 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? UT(TO add).
- Loading branch information
Showing
12 changed files
with
893 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
...talog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopProxyPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Copyright 2024 Datastrato Pvt Ltd. | ||
| * This software is licensed under the Apache License version 2. | ||
| */ | ||
|
|
||
| package com.datastrato.gravitino.catalog.hadoop; | ||
|
|
||
| import com.datastrato.gravitino.connector.CatalogOperations; | ||
| import com.datastrato.gravitino.connector.ProxyPlugin; | ||
| import com.datastrato.gravitino.utils.Executable; | ||
| import java.io.IOException; | ||
| import java.lang.reflect.InvocationTargetException; | ||
| import java.lang.reflect.UndeclaredThrowableException; | ||
| import java.security.Principal; | ||
| import java.security.PrivilegedActionException; | ||
| import java.security.PrivilegedExceptionAction; | ||
| import java.util.Map; | ||
| import org.apache.hadoop.security.UserGroupInformation; | ||
|
|
||
| public class HadoopProxyPlugin implements ProxyPlugin { | ||
| private HadoopCatalogOperations ops; | ||
| private UserGroupInformation realUser; | ||
|
|
||
| public HadoopProxyPlugin() { | ||
| try { | ||
| realUser = UserGroupInformation.getCurrentUser(); | ||
| } catch (IOException ioe) { | ||
| throw new IllegalStateException("Fail to init HadoopCatalogProxyPlugin"); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public Object doAs( | ||
| Principal principal, Executable<Object, Exception> action, Map<String, String> properties) | ||
| throws Throwable { | ||
| try { | ||
| UserGroupInformation proxyUser; | ||
|
|
||
| if (UserGroupInformation.isSecurityEnabled() && ops != null) { | ||
| // The Gravitino server may use multiple KDC servers. | ||
| // The http authentication use one KDC server, the Hadoop catalog may use another KDC | ||
| // server. | ||
| // The KerberosAuthenticator will remove realm of principal. | ||
| // And then we add the realm of Hadoop catalog to the user. | ||
| String proxyKerberosPrincipalName = principal.getName(); | ||
| if (!proxyKerberosPrincipalName.contains("@")) { | ||
| proxyKerberosPrincipalName = | ||
| String.format("%s@%s", proxyKerberosPrincipalName, ops.getKerberosRealm()); | ||
| } | ||
|
|
||
| proxyUser = UserGroupInformation.createProxyUser(proxyKerberosPrincipalName, realUser); | ||
| } else { | ||
| proxyUser = UserGroupInformation.createProxyUser(principal.getName(), realUser); | ||
| } | ||
|
|
||
| return proxyUser.doAs((PrivilegedExceptionAction<Object>) action::execute); | ||
| } catch (UndeclaredThrowableException e) { | ||
| Throwable innerException = e.getCause(); | ||
| if (innerException instanceof PrivilegedActionException) { | ||
| throw innerException.getCause(); | ||
| } else if (innerException instanceof InvocationTargetException) { | ||
| throw innerException.getCause(); | ||
| } else { | ||
| throw innerException; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void bindCatalogOperation(CatalogOperations ops) { | ||
| this.ops = ((HadoopCatalogOperations) ops); | ||
| this.ops.setProxyPlugin(this); | ||
| } | ||
| } |
69 changes: 69 additions & 0 deletions
69
.../src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/AuthenticationConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Copyright 2024 Datastrato Pvt Ltd. | ||
| * This software is licensed under the Apache License version 2. | ||
| */ | ||
|
|
||
| package com.datastrato.gravitino.catalog.hadoop.kerberos; | ||
|
|
||
| import com.datastrato.gravitino.Config; | ||
| import com.datastrato.gravitino.config.ConfigBuilder; | ||
| import com.datastrato.gravitino.config.ConfigConstants; | ||
| import com.datastrato.gravitino.config.ConfigEntry; | ||
| import com.datastrato.gravitino.connector.PropertyEntry; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import java.util.Map; | ||
|
|
||
| public class AuthenticationConfig extends Config { | ||
| public static final String ENABLE_AUTH_KEY = "authentication.enable"; | ||
| public static final String AUTH_TYPE_KEY = "authentication.type"; | ||
|
|
||
| public AuthenticationConfig(Map<String, String> properties) { | ||
| super(false); | ||
| loadFromMap(properties, k -> true); | ||
| } | ||
|
|
||
| public static final ConfigEntry<Boolean> ENABLE_AUTH_ENTRY = | ||
| new ConfigBuilder(ENABLE_AUTH_KEY) | ||
| .doc("Whether to enable authentication for Hadoop catalog") | ||
| .version(ConfigConstants.VERSION_0_5_1) | ||
| .booleanConf() | ||
| .createWithDefault(false); | ||
|
|
||
| public static final ConfigEntry<String> AUTH_TYPE_ENTRY = | ||
| new ConfigBuilder(AUTH_TYPE_KEY) | ||
| .doc("The type of authentication for Hadoop catalog, currently we only support kerberos") | ||
| .version(ConfigConstants.VERSION_0_5_1) | ||
| .stringConf() | ||
| .create(); | ||
|
|
||
| public boolean isEnableAuth() { | ||
| return get(ENABLE_AUTH_ENTRY); | ||
| } | ||
|
|
||
| public String getAuthType() { | ||
| return get(AUTH_TYPE_ENTRY); | ||
| } | ||
|
|
||
| public static final Map<String, PropertyEntry<?>> AUTHENTICATION_PROPERTY_ENTRIES = | ||
| new ImmutableMap.Builder<String, PropertyEntry<?>>() | ||
| .put( | ||
| ENABLE_AUTH_KEY, | ||
| PropertyEntry.booleanPropertyEntry( | ||
| ENABLE_AUTH_KEY, | ||
| "Whether to enable authentication for Hadoop catalog", | ||
| false, | ||
| true, | ||
| false, | ||
| false, | ||
| false)) | ||
| .put( | ||
| AUTH_TYPE_KEY, | ||
| PropertyEntry.stringImmutablePropertyEntry( | ||
| AUTH_TYPE_KEY, | ||
| "The type of authentication for Hadoop catalog, currently we only support kerberos", | ||
| false, | ||
| null, | ||
| false, | ||
| false)) | ||
| .build(); | ||
| } |
51 changes: 51 additions & 0 deletions
51
...hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/FetchFileUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /* | ||
| * Copyright 2024 Datastrato Pvt Ltd. | ||
| * This software is licensed under the Apache License version 2. | ||
| */ | ||
| package com.datastrato.gravitino.catalog.hadoop.kerberos; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.net.URI; | ||
| import java.net.URISyntaxException; | ||
| import java.nio.file.Files; | ||
| import java.util.Optional; | ||
| import org.apache.commons.io.FileUtils; | ||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.fs.FileSystem; | ||
| import org.apache.hadoop.fs.Path; | ||
|
|
||
| public class FetchFileUtils { | ||
|
|
||
| private FetchFileUtils() {} | ||
|
|
||
| public static void fetchFileFromUri( | ||
| String fileUri, File destFile, int timeout, Configuration conf) throws IOException { | ||
| try { | ||
| URI uri = new URI(fileUri); | ||
| String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); | ||
|
|
||
| switch (scheme) { | ||
| case "http": | ||
| case "https": | ||
| case "ftp": | ||
| FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); | ||
| break; | ||
|
|
||
| case "file": | ||
| Files.createSymbolicLink(destFile.toPath(), new File(uri.getPath()).toPath()); | ||
| break; | ||
|
|
||
| case "hdfs": | ||
| FileSystem.get(conf).copyToLocalFile(new Path(uri), new Path(destFile.toURI())); | ||
| break; | ||
|
|
||
| default: | ||
| throw new IllegalArgumentException( | ||
| String.format("Doesn't support the scheme %s", scheme)); | ||
| } | ||
| } catch (URISyntaxException ue) { | ||
| throw new IllegalArgumentException("The uri of file has the wrong format", ue); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.