diff --git a/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java new file mode 100644 index 00000000000..308d76a8dd2 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** OSS token credential. */ +public class OSSTokenCredential implements Credential { + + /** OSS token credential type. */ + public static final String OSS_TOKEN_CREDENTIAL_TYPE = "oss-token"; + /** OSS access key ID used to access OSS data. */ + public static final String GRAVITINO_OSS_SESSION_ACCESS_KEY_ID = "oss-access-key-id"; + /** OSS secret access key used to access OSS data. */ + public static final String GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY = "oss-secret-access-key"; + /** OSS security token. */ + public static final String GRAVITINO_OSS_TOKEN = "oss-security-token"; + + private final String accessKeyId; + private final String secretAccessKey; + private final String securityToken; + private final long expireTimeInMS; + + /** + * Constructs an instance of {@link OSSTokenCredential} with secret key and token. + * + * @param accessKeyId The oss access key ID. + * @param secretAccessKey The oss secret access key. + * @param securityToken The oss security token. + * @param expireTimeInMS The oss token expire time in ms. + */ + public OSSTokenCredential( + String accessKeyId, String secretAccessKey, String securityToken, long expireTimeInMS) { + Preconditions.checkArgument( + StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not be empty"); + Preconditions.checkArgument( + StringUtils.isNotBlank(secretAccessKey), "OSS access key secret should not be empty"); + Preconditions.checkArgument( + StringUtils.isNotBlank(securityToken), "OSS security token should not be empty"); + + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.securityToken = securityToken; + this.expireTimeInMS = expireTimeInMS; + } + + @Override + public String credentialType() { + return OSS_TOKEN_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return expireTimeInMS; + } + + @Override + public Map credentialInfo() { + return (new ImmutableMap.Builder()) + .put(GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, accessKeyId) + .put(GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, secretAccessKey) + .put(GRAVITINO_OSS_TOKEN, securityToken) + .build(); + } + + /** + * Get oss access key ID. + * + * @return The oss access key ID. + */ + public String accessKeyId() { + return accessKeyId; + } + + /** + * Get oss secret access key. + * + * @return The oss secret access key. + */ + public String secretAccessKey() { + return secretAccessKey; + } + + /** + * Get oss security token. + * + * @return The oss security token. + */ + public String securityToken() { + return securityToken; + } +} diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts index e0d8e9e15a9..e803c517b38 100644 --- a/bundles/aliyun-bundle/build.gradle.kts +++ b/bundles/aliyun-bundle/build.gradle.kts @@ -25,16 +25,26 @@ plugins { } dependencies { + compileOnly(project(":api")) + compileOnly(project(":core")) + compileOnly(project(":catalogs:catalog-common")) compileOnly(project(":catalogs:catalog-hadoop")) compileOnly(libs.hadoop3.common) + + implementation(libs.aliyun.credentials.sdk) implementation(libs.hadoop3.oss) + // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add + // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 + implementation(libs.sun.activation) + // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) implementation(libs.commons.lang3) + implementation(project(":catalogs:catalog-common")) { exclude("*") } diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java new file mode 100644 index 00000000000..04ef0022a10 --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.oss.credential; + +import com.aliyun.credentials.Client; +import com.aliyun.credentials.models.Config; +import com.aliyun.credentials.models.CredentialModel; +import com.aliyun.credentials.utils.AuthConstant; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.OSSTokenCredential; +import org.apache.gravitino.credential.PathBasedCredentialContext; +import org.apache.gravitino.credential.config.OSSCredentialConfig; +import org.apache.gravitino.oss.credential.policy.Condition; +import org.apache.gravitino.oss.credential.policy.Effect; +import org.apache.gravitino.oss.credential.policy.Policy; +import org.apache.gravitino.oss.credential.policy.Statement; +import org.apache.gravitino.oss.credential.policy.StringLike; + +/** Generates OSS token to access OSS data. */ +public class OSSTokenProvider implements CredentialProvider { + private final ObjectMapper objectMapper = new ObjectMapper(); + private String accessKeyId; + private String secretAccessKey; + private String roleArn; + private String externalID; + private int tokenExpireSecs; + private String region; + + /** + * Initializes the credential provider with catalog properties. + * + * @param properties catalog properties that can be used to configure the provider. The specific + * properties required vary by implementation. + */ + @Override + public void initialize(Map properties) { + OSSCredentialConfig credentialConfig = new OSSCredentialConfig(properties); + this.roleArn = credentialConfig.ossRoleArn(); + this.externalID = credentialConfig.externalID(); + this.tokenExpireSecs = credentialConfig.tokenExpireInSecs(); + this.accessKeyId = credentialConfig.accessKeyID(); + this.secretAccessKey = credentialConfig.secretAccessKey(); + this.region = credentialConfig.region(); + } + + /** + * Returns the type of credential, it should be identical in Gravitino. + * + * @return A string identifying the type of credentials. + */ + @Override + public String credentialType() { + return OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE; + } + + /** + * Obtains a credential based on the provided context information. + * + * @param context A context object providing necessary information for retrieving credentials. + * @return A Credential object containing the authentication information needed to access a system + * or resource. Null will be returned if no credential is available. + */ + @Nullable + @Override + public Credential getCredential(CredentialContext context) { + if (!(context instanceof PathBasedCredentialContext)) { + return null; + } + PathBasedCredentialContext pathBasedCredentialContext = (PathBasedCredentialContext) context; + CredentialModel credentialModel = + createOSSCredentialModel( + roleArn, + pathBasedCredentialContext.getReadPaths(), + pathBasedCredentialContext.getWritePaths(), + pathBasedCredentialContext.getUserName()); + return new OSSTokenCredential( + credentialModel.accessKeyId, + credentialModel.accessKeySecret, + credentialModel.securityToken, + credentialModel.expiration); + } + + private CredentialModel createOSSCredentialModel( + String roleArn, Set readLocations, Set writeLocations, String userName) { + Config config = new Config(); + config.setAccessKeyId(accessKeyId); + config.setAccessKeySecret(secretAccessKey); + config.setType(AuthConstant.RAM_ROLE_ARN); + config.setRoleArn(roleArn); + config.setRoleSessionName(getRoleName(userName)); + if (StringUtils.isNotBlank(externalID)) { + config.setExternalId(externalID); + } + config.setRoleSessionExpiration(tokenExpireSecs); + config.setPolicy(createPolicy(readLocations, writeLocations)); + // Local object and client is a simple proxy that does not require manual release + Client client = new Client(config); + return client.getCredential(); + } + + // reference: + // https://www.alibabacloud.com/help/en/oss/user-guide/tutorial-use-ram-policies-to-control-access-to-oss?spm=a2c63.p38356.help-menu-31815.d_2_4_5_1.5536471b56XPRQ + private String createPolicy(Set readLocations, Set writeLocations) { + Policy.Builder policyBuilder = Policy.builder().version("1"); + + // Allow read and write access to the specified locations + Statement.Builder allowGetObjectStatementBuilder = + Statement.builder() + .effect(Effect.ALLOW) + .addAction("oss:GetObject") + .addAction("oss:GetObjectVersion"); + // Add support for bucket-level policies + Map bucketListStatementBuilder = new HashMap<>(); + Map bucketGetLocationStatementBuilder = new HashMap<>(); + + String arnPrefix = getArnPrefix(); + Stream.concat(readLocations.stream(), writeLocations.stream()) + .distinct() + .forEach( + location -> { + URI uri = URI.create(location); + allowGetObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri)); + String bucketArn = arnPrefix + getBucketName(uri); + // ListBucket + bucketListStatementBuilder.computeIfAbsent( + bucketArn, + key -> + Statement.builder() + .effect(Effect.ALLOW) + .addAction("oss:ListBucket") + .addResource(key) + .condition(getCondition(uri))); + // GetBucketLocation + bucketGetLocationStatementBuilder.computeIfAbsent( + bucketArn, + key -> + Statement.builder() + .effect(Effect.ALLOW) + .addAction("oss:GetBucketLocation") + .addResource(key)); + }); + + if (!writeLocations.isEmpty()) { + Statement.Builder allowPutObjectStatementBuilder = + Statement.builder() + .effect(Effect.ALLOW) + .addAction("oss:PutObject") + .addAction("oss:DeleteObject"); + writeLocations.forEach( + location -> { + URI uri = URI.create(location); + allowPutObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri)); + }); + policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); + } + + if (!bucketListStatementBuilder.isEmpty()) { + bucketListStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + } else { + // add list privilege with 0 resources + policyBuilder.addStatement( + Statement.builder().effect(Effect.ALLOW).addAction("oss:ListBucket").build()); + } + bucketGetLocationStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + + policyBuilder.addStatement(allowGetObjectStatementBuilder.build()); + try { + return objectMapper.writeValueAsString(policyBuilder.build()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Condition getCondition(URI uri) { + return Condition.builder() + .stringLike( + StringLike.builder() + .addPrefix(concatPathWithSep(trimLeadingSlash(uri.getPath()), "*", "/")) + .build()) + .build(); + } + + private String getArnPrefix() { + if (StringUtils.isNotEmpty(region)) { + return "acs:oss:" + region + ":*:"; + } + return "acs:oss:*:*:"; + } + + private String getBucketName(URI uri) { + return uri.getHost(); + } + + private String getOssUriWithArn(String arnPrefix, URI uri) { + return arnPrefix + concatPathWithSep(removeSchemaFromOSSUri(uri), "*", "/"); + } + + private static String concatPathWithSep(String leftPath, String rightPath, String fileSep) { + if (leftPath.endsWith(fileSep) && rightPath.startsWith(fileSep)) { + return leftPath + rightPath.substring(1); + } else if (!leftPath.endsWith(fileSep) && !rightPath.startsWith(fileSep)) { + return leftPath + fileSep + rightPath; + } else { + return leftPath + rightPath; + } + } + + // Transform 'oss://bucket/path' to /bucket/path + private String removeSchemaFromOSSUri(URI uri) { + String bucket = uri.getHost(); + String path = trimLeadingSlash(uri.getPath()); + return String.join( + "/", Stream.of(bucket, path).filter(Objects::nonNull).toArray(String[]::new)); + } + + private String trimLeadingSlash(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + + private String getRoleName(String userName) { + return "gravitino_" + userName; + } + + @Override + public void close() throws IOException {} +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java new file mode 100644 index 00000000000..a2b20612cb8 --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential.policy; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Condition { + + @JsonProperty("StringLike") + private StringLike stringLike; + + private Condition(Builder builder) { + this.stringLike = builder.stringLike; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private StringLike stringLike; + + public Builder stringLike(StringLike stringLike) { + this.stringLike = stringLike; + return this; + } + + public Condition build() { + return new Condition(this); + } + } + + @SuppressWarnings("unused") + public StringLike getStringLike() { + return stringLike; + } +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java new file mode 100644 index 00000000000..5906cb13c8c --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential.policy; + +public class Effect { + public static final String ALLOW = "Allow"; +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java new file mode 100644 index 00000000000..4c8c0f5ba8d --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential.policy; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Policy { + + @JsonProperty("Version") + private String version; + + @JsonProperty("Statement") + private List statements; + + private Policy(Builder builder) { + this.version = builder.version; + this.statements = builder.statements; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String version; + private List statements = new ArrayList<>(); + + public Builder version(String version) { + this.version = version; + return this; + } + + public Builder addStatement(Statement statement) { + this.statements.add(statement); + return this; + } + + public Policy build() { + return new Policy(this); + } + } + + @SuppressWarnings("unused") + public String getVersion() { + return version; + } + + @SuppressWarnings("unused") + public List getStatements() { + return statements; + } +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java new file mode 100644 index 00000000000..2dca6e74f6d --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential.policy; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Statement { + + @JsonProperty("Effect") + private String effect; + + @JsonProperty("Action") + private List actions; + + @JsonProperty("Resource") + private List resources; + + @JsonProperty("Condition") + private Condition condition; + + private Statement(Builder builder) { + this.effect = builder.effect; + this.actions = builder.actions; + this.resources = builder.resources; + this.condition = builder.condition; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String effect; + private List actions = new ArrayList<>(); + private List resources = new ArrayList<>(); + private Condition condition; + + public Builder effect(String effect) { + this.effect = effect; + return this; + } + + public Builder addAction(String action) { + this.actions.add(action); + return this; + } + + public Builder addResource(String resource) { + this.resources.add(resource); + return this; + } + + public Builder condition(Condition condition) { + this.condition = condition; + return this; + } + + public Statement build() { + return new Statement(this); + } + } + + @SuppressWarnings("unused") + public String getEffect() { + return effect; + } + + @SuppressWarnings("unused") + public List getActions() { + return actions; + } + + @SuppressWarnings("unused") + public List getResources() { + return resources; + } + + @SuppressWarnings("unused") + public Condition getCondition() { + return condition; + } +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java new file mode 100644 index 00000000000..7438907bd3a --- /dev/null +++ b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential.policy; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class StringLike { + + @JsonProperty("oss:Prefix") + private List prefix; + + private StringLike(Builder builder) { + this.prefix = builder.prefix; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private List prefix = new ArrayList<>(); + + public Builder addPrefix(String prefix) { + this.prefix.add(prefix); + return this; + } + + public StringLike build() { + return new StringLike(this); + } + } + + @SuppressWarnings("unused") + public List getPrefix() { + return prefix; + } +} diff --git a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider new file mode 100644 index 00000000000..d2f4be51b7c --- /dev/null +++ b/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.oss.credential.OSSTokenProvider \ No newline at end of file diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 665a78ee7ce..739cd013909 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -26,5 +26,8 @@ public class CredentialConstants { public static final String GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "gcs-token"; + public static final String OSS_TOKEN_CREDENTIAL_PROVIDER = "oss-token"; + public static final String OSS_TOKEN_EXPIRE_IN_SECS = "oss-token-expire-in-secs"; + private CredentialConstants() {} } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java index 3885eb360ce..8471682aea9 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java @@ -18,15 +18,22 @@ */ package org.apache.gravitino.storage; -// Defines the unified OSS properties for different catalogs and connectors. +// Properties for OSS. public class OSSProperties { - // The endpoint of Aliyun OSS service. + // The region of Aliyun OSS. + public static final String GRAVITINO_OSS_REGION = "oss-region"; + // The endpoint of Aliyun OSS. public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint"; // The static access key ID used to access OSS data. public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id"; // The static access key secret used to access OSS data. public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET = "oss-secret-access-key"; + // OSS role arn + public static final String GRAVITINO_OSS_ROLE_ARN = "oss-role-arn"; + // OSS external id + public static final String GRAVITINO_OSS_EXTERNAL_ID = "oss-external-id"; + private OSSProperties() {} } diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index 811f04c53a7..8f56f802d7e 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -34,6 +34,10 @@ public class CredentialPropertyUtils { @VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token"; @VisibleForTesting static final String ICEBERG_GCS_TOKEN = "gcs.oauth2.token"; + @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID = "client.access-key-id"; + @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET = "client.access-key-secret"; + @VisibleForTesting static final String ICEBERG_OSS_SECURITY_TOKEN = "client.security-token"; + private static Map icebergCredentialPropertyMap = ImmutableMap.of( GCSTokenCredential.GCS_TOKEN_NAME, @@ -43,7 +47,13 @@ public class CredentialPropertyUtils { S3SecretKeyCredential.GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, ICEBERG_S3_SECRET_ACCESS_KEY, S3TokenCredential.GRAVITINO_S3_TOKEN, - ICEBERG_S3_TOKEN); + ICEBERG_S3_TOKEN, + OSSTokenCredential.GRAVITINO_OSS_TOKEN, + ICEBERG_OSS_SECURITY_TOKEN, + OSSTokenCredential.GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, + ICEBERG_OSS_ACCESS_KEY_ID, + OSSTokenCredential.GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, + ICEBERG_OSS_ACCESS_KEY_SECRET); /** * Transforms a specific credential into a map of Iceberg properties. @@ -62,6 +72,9 @@ public static Map toIcebergProperties(Credential credential) { if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) { return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); } + if (credential instanceof OSSTokenCredential) { + return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); + } return credential.toProperties(); } diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java index 4d3ad698b51..3b4571e01e9 100644 --- a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java @@ -51,4 +51,21 @@ void testToIcebergProperties() { "secret"); Assertions.assertEquals(expectedProperties, icebergProperties); } + + @Test + void testToIcebergPropertiesForOSS() { + OSSTokenCredential ossTokenCredential = + new OSSTokenCredential("key", "secret", "security-token", 0); + Map icebergProperties = + CredentialPropertyUtils.toIcebergProperties(ossTokenCredential); + Map expectedProperties = + ImmutableMap.of( + CredentialPropertyUtils.ICEBERG_OSS_ACCESS_KEY_ID, + "key", + CredentialPropertyUtils.ICEBERG_OSS_ACCESS_KEY_SECRET, + "secret", + CredentialPropertyUtils.ICEBERG_OSS_SECURITY_TOKEN, + "security-token"); + Assertions.assertEquals(expectedProperties, icebergProperties); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java new file mode 100644 index 00000000000..0422bb3b750 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.config; + +import java.util.Map; +import javax.validation.constraints.NotNull; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.config.ConfigBuilder; +import org.apache.gravitino.config.ConfigConstants; +import org.apache.gravitino.config.ConfigEntry; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.storage.OSSProperties; + +public class OSSCredentialConfig extends Config { + + public static final ConfigEntry OSS_REGION = + new ConfigBuilder(OSSProperties.GRAVITINO_OSS_REGION) + .doc("The region of the OSS service") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .create(); + + public static final ConfigEntry OSS_ACCESS_KEY_ID = + new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID) + .doc("The static access key ID used to access OSS data") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry OSS_SECRET_ACCESS_KEY = + new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET) + .doc("The static secret access key used to access OSS data") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry OSS_ROLE_ARN = + new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ROLE_ARN) + .doc("OSS role arn") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry OSS_EXTERNAL_ID = + new ConfigBuilder(OSSProperties.GRAVITINO_OSS_EXTERNAL_ID) + .doc("OSS external ID") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .create(); + + public static final ConfigEntry OSS_TOKEN_EXPIRE_IN_SECS = + new ConfigBuilder(CredentialConstants.OSS_TOKEN_EXPIRE_IN_SECS) + .doc("OSS token expire in seconds") + .version(ConfigConstants.VERSION_0_8_0) + .intConf() + .createWithDefault(3600); + + public OSSCredentialConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } + + public String region() { + return this.get(OSS_REGION); + } + + @NotNull + public String ossRoleArn() { + return this.get(OSS_ROLE_ARN); + } + + @NotNull + public String accessKeyID() { + return this.get(OSS_ACCESS_KEY_ID); + } + + @NotNull + public String secretAccessKey() { + return this.get(OSS_SECRET_ACCESS_KEY); + } + + public String externalID() { + return this.get(OSS_EXTERNAL_ID); + } + + public Integer tokenExpireInSecs() { + return this.get(OSS_TOKEN_EXPIRE_IN_SECS); + } +} diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 20fc4221d14..4f626d31429 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -17,7 +17,7 @@ The Apache Gravitino Iceberg REST Server follows the [Apache Iceberg REST API sp - multi table transaction - pagination - Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend. -- Supports credential vending for `S3` and `GCS`. +- Supports credential vending for `S3`、`GCS` and `OSS`. - Supports different storages like `S3`, `HDFS`, `OSS`, `GCS` and provides the capability to support other storages. - Supports event listener. - Supports Audit log. @@ -128,17 +128,23 @@ To configure the JDBC catalog backend, set the `gravitino.iceberg-rest.warehouse #### OSS configuration -Gravitino Iceberg REST service supports using static access-key-id and secret-access-key to access OSS data. +Gravitino Iceberg REST service supports using static access-key-id and secret-access-key or generating temporary token to access OSS data. -| Configuration item | Description | Default value | Required | Since Version | -|------------------------------------------------|-------------------------------------------------------------------------------------------------------|---------------|----------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID used to access OSS data. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access key used to access OSS data. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS service. | (none) | No | 0.7.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------|----------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID used to access OSS data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access key used to access OSS data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS service. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.oss-region` | The region of the OSS service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to access the OSS data, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to generate token, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token expire time in secs, only used when `credential-provider-type` is `oss-token`. | 3600 | No | 0.8.0-incubating | For other Iceberg OSS properties not managed by Gravitino like `client.security-token`, you could config it directly by `gravitino.iceberg-rest.client.security-token`. +If you set `credential-provider-type` explicitly, please downloading [Gravitino Aliyun bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle), and place it to the classpath of Iceberg REST server. + :::info Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the auxiliary server, `libs` for the standalone server. ::: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f1f29cf186d..88636c5a52e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -113,6 +113,7 @@ datanucleus-rdbms = "4.1.19" datanucleus-jdo = "3.2.0-m3" hudi = "0.15.0" google-auth = "1.28.0" +aliyun-credentials = "0.3.12" [libraries] aws-iam = { group = "software.amazon.awssdk", name = "iam", version.ref = "awssdk" } @@ -266,6 +267,8 @@ jettison = { group = "org.codehaus.jettison", name = "jettison", version.ref = " google-auth-http = { group = "com.google.auth", name = "google-auth-library-oauth2-http", version.ref = "google-auth" } google-auth-credentials = { group = "com.google.auth", name = "google-auth-library-credentials", version.ref = "google-auth" } +aliyun-credentials-sdk = { group='com.aliyun', name='credentials-java', version.ref='aliyun-credentials' } + [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api", "log4j-layout-template-json"] jetty = ["jetty-server", "jetty-servlet", "jetty-webapp", "jetty-servlets"] diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 5d84ef77826..e46193c9774 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -62,6 +62,7 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) + testImplementation(project(":bundles:aliyun-bundle")) testImplementation(project(":bundles:aws-bundle")) testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java new file mode 100644 index 00000000000..fb6bac65bf6 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.integration.test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.DownloaderUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.gravitino.storage.OSSProperties; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.platform.commons.util.StringUtils; + +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class IcebergRESTOSSIT extends IcebergRESTJdbcCatalogIT { + + private String warehouse; + private String accessKey; + private String secretKey; + private String endpoint; + private String roleArn; + private String externalId; + private String region; + + @Override + void initEnv() { + this.warehouse = + String.format( + "oss://%s/gravitino-test", + getFromEnvOrDefault("GRAVITINO_OSS_BUCKET", "{BUCKET_NAME}")); + this.accessKey = getFromEnvOrDefault("GRAVITINO_OSS_ACCESS_KEY", "{ACCESS_KEY}"); + this.secretKey = getFromEnvOrDefault("GRAVITINO_OSS_SECRET_KEY", "{SECRET_KEY}"); + this.endpoint = getFromEnvOrDefault("GRAVITINO_OSS_ENDPOINT", "{GRAVITINO_OSS_ENDPOINT}"); + this.region = getFromEnvOrDefault("GRAVITINO_OSS_REGION", "oss-cn-hangzhou"); + this.roleArn = getFromEnvOrDefault("GRAVITINO_OSS_ROLE_ARN", "{ROLE_ARN}"); + this.externalId = getFromEnvOrDefault("GRAVITINO_OSS_EXTERNAL_ID", ""); + + if (ITUtils.isEmbedded()) { + return; + } + try { + downloadIcebergForAliyunJar(); + } catch (IOException e) { + LOG.warn("Download Iceberg Aliyun bundle jar failed,", e); + throw new RuntimeException(e); + } + copyAliyunOSSJar(); + } + + @Override + public Map getCatalogConfig() { + HashMap m = new HashMap(); + m.putAll(getCatalogJdbcConfig()); + m.putAll(getOSSConfig()); + return m; + } + + public boolean supportsCredentialVending() { + return true; + } + + private Map getOSSConfig() { + Map configMap = new HashMap(); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + CredentialConstants.OSS_TOKEN_CREDENTIAL_PROVIDER); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_REGION, region); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ENDPOINT, endpoint); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, accessKey); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, + secretKey); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ROLE_ARN, roleArn); + if (StringUtils.isNotBlank(externalId)) { + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_EXTERNAL_ID, + externalId); + } + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL, + "org.apache.iceberg.aliyun.oss.OSSFileIO"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.WAREHOUSE, warehouse); + + return configMap; + } + + private void downloadIcebergForAliyunJar() throws IOException { + String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar"; + String icebergBundleJarUri = + "https://repo1.maven.org/maven2/org/apache/iceberg/" + + "iceberg-aliyun/1.5.2/" + + icebergBundleJarName; + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir); + } + + private void copyAliyunOSSJar() { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir); + } + + private String getFromEnvOrDefault(String envVar, String defaultValue) { + String envValue = System.getenv(envVar); + return Optional.ofNullable(envValue).orElse(defaultValue); + } +}