[Feature][Resource Center] Add support for Alibaba Cloud OSS as storage of resource center

docs/docs/en/guide/resource/configuration.md

@@ -45,9 +45,9 @@ data.basedir.path=/tmp/dolphinscheduler
 # resource view suffixs
 #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
 
-# resource storage type: HDFS, S3, NONE
+# resource storage type: HDFS, S3, OSS, NONE
 resource.storage.type=NONE
-# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
+# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.storage.upload.base.path=/tmp/dolphinscheduler
 
 # The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
@@ -61,6 +61,17 @@ resource.aws.s3.bucket.name=dolphinscheduler
 # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
 resource.aws.s3.endpoint=http://localhost:9000
 
+# alibaba cloud access key id, required if you set resource.storage.type=OSS 
+resource.alibaba.cloud.access.key.id=<your-access-key-id>
+# alibaba cloud access key secret, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.access.key.secret=<your-access-key-secret>
+# alibaba cloud region, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.region=cn-hangzhou
+# oss bucket name, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
+# oss bucket endpoint, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
+
 # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
 resource.hdfs.root.user=hdfs
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir

docs/docs/zh/guide/resource/configuration.md

@@ -46,10 +46,10 @@
 # user data local directory path, please make sure the directory exists and have read write permissions
 data.basedir.path=/tmp/dolphinscheduler
 
-# resource storage type: HDFS, S3, NONE
+# resource storage type: HDFS, S3, OSS, NONE
 resource.storage.type=HDFS
 
-# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration,
+# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
 # please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.storage.upload.base.path=/tmp/dolphinscheduler
 
@@ -64,6 +64,17 @@ resource.aws.s3.bucket.name=dolphinscheduler
 # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
 resource.aws.s3.endpoint=http://localhost:9000
 
+# alibaba cloud access key id, required if you set resource.storage.type=OSS 
+resource.alibaba.cloud.access.key.id=<your-access-key-id>
+# alibaba cloud access key secret, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.access.key.secret=<your-access-key-secret>
+# alibaba cloud region, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.region=cn-hangzhou
+# oss bucket name, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
+# oss bucket endpoint, required if you set resource.storage.type=OSS
+resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
+
 # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
 resource.hdfs.root.user=root
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@@ -410,6 +410,7 @@ public Result<Object> updateResource(User loginUser,
             return result;
         }
 
+        // TODO: deal with OSS
         if (resource.isDirectory() && storageOperate.returnStorageType().equals(ResUploadType.S3)
                 && !resource.getFileName().equals(name)) {
             logger.warn("Directory in S3 storage can not be renamed.");
@@ -874,7 +875,6 @@ public Result<Object> delete(User loginUser, int resourceId) throws IOException
         List<Integer> allChildren = listAllChildren(resource, true);
 
         Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]);
-
         if (needDeleteResourceIdArray.length >= 2) {
             logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", resource.getFileName());
             putMsg(result, Status.RESOURCE_HAS_FOLDER, resource.getFileName());

dolphinscheduler-bom/pom.xml

@@ -85,6 +85,7 @@
         <error_prone_annotations.version>2.5.1</error_prone_annotations.version>
         <hibernate-validator.version>6.2.2.Final</hibernate-validator.version>
         <aws-sdk.version>1.12.300</aws-sdk.version>
+        <aliyun-sdk-oss.version>3.15.1</aliyun-sdk-oss.version>
         <joda-time.version>2.10.13</joda-time.version>
         <okhttp.version>3.14.9</okhttp.version>
         <json-path.version>2.7.0</json-path.version>
@@ -626,6 +627,12 @@
                 <version>${aws-sdk.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.aliyun.oss</groupId>
+                <artifactId>aliyun-sdk-oss</artifactId>
+                <version>${aliyun-sdk-oss.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>joda-time</groupId>
                 <artifactId>joda-time</artifactId>

dolphinscheduler-common/pom.xml

@@ -15,18 +15,30 @@
   ~ See the License for the specific language governing permissions and
   ~ limitations under the License.
   -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
         <artifactId>dolphinscheduler</artifactId>
         <version>dev-SNAPSHOT</version>
     </parent>
     <artifactId>dolphinscheduler-common</artifactId>
-    <name>dolphinscheduler-common</name>
 
     <packaging>jar</packaging>
+    <name>dolphinscheduler-common</name>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.dolphinscheduler</groupId>
+                <artifactId>dolphinscheduler-bom</artifactId>
+                <version>${project.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
     <dependencies>
         <!-- dolphinscheduler -->
         <dependency>
@@ -94,8 +106,8 @@
                     <artifactId>jdk.tools</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>javax.servlet</groupId>
@@ -159,20 +171,20 @@
                     <artifactId>jsp-api</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-json</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-json</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-server</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-core</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jetty</artifactId>
                     <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
@@ -186,8 +198,8 @@
                     <artifactId>slf4j-log4j12</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>org.codehaus.jackson</groupId>
@@ -211,20 +223,20 @@
                     <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-client</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-client</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-core</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jaxb-api</artifactId>
                     <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>log4j</artifactId>
                     <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
@@ -259,16 +271,16 @@
                     <artifactId>leveldbjni-all</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-core</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>jersey-server</artifactId>
                     <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
                 </exclusion>
                 <exclusion>
-                    <artifactId>log4j</artifactId>
                     <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
@@ -279,6 +291,11 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.github.oshi</groupId>
             <artifactId>oshi-core</artifactId>
@@ -299,8 +316,8 @@
         </dependency>
 
         <dependency>
-            <artifactId>servlet-api</artifactId>
             <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
             <scope>test</scope>
         </dependency>
 
@@ -311,16 +328,4 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-
-    <dependencyManagement>
-        <dependencies>
-            <dependency>
-                <groupId>org.apache.dolphinscheduler</groupId>
-                <artifactId>dolphinscheduler-bom</artifactId>
-                <version>${project.version}</version>
-                <type>pom</type>
-                <scope>import</scope>
-            </dependency>
-        </dependencies>
-    </dependencyManagement>
 </project>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -57,6 +57,8 @@ private Constants() {
 
     public static final String STORAGE_S3 = "S3";
 
+    public static final String STORAGE_OSS = "OSS";
+
     public static final String STORAGE_HDFS = "HDFS";
 
     public static final String EMPTY_STRING = "";
@@ -150,6 +152,10 @@ private Constants() {
 
     public static final String AWS_S3_BUCKET_NAME = "resource.aws.s3.bucket.name";
     public static final String AWS_END_POINT = "resource.aws.s3.endpoint";
+
+    public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name";
+    public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint";
+
     /**
      * comma ,
      */

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/StoreConfiguration.java

@@ -17,19 +17,21 @@
 
 package org.apache.dolphinscheduler.common.config;
 
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_STORAGE_TYPE;
+import static org.apache.dolphinscheduler.common.Constants.STORAGE_HDFS;
+import static org.apache.dolphinscheduler.common.Constants.STORAGE_OSS;
+import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3;
+
 import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.common.utils.HadoopUtils;
+import org.apache.dolphinscheduler.common.utils.OssOperator;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.common.utils.S3Utils;
+
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;
 
-import static org.apache.dolphinscheduler.common.Constants.RESOURCE_STORAGE_TYPE;
-import static org.apache.dolphinscheduler.common.Constants.STORAGE_HDFS;
-import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3;
-
-
 /**
  * choose the impl of storage by RESOURCE_STORAGE_TYPE
  */
@@ -41,6 +43,12 @@ public class StoreConfiguration {
     @Bean
     public StorageOperate storageOperate() {
         switch (PropertyUtils.getString(RESOURCE_STORAGE_TYPE)) {
+            case STORAGE_OSS:
+                OssOperator ossOperator = new OssOperator();
+                // TODO: change to use ossOperator.init(ossConnection) after DS supports Configuration / Connection
+                // Center
+                ossOperator.init();
+                return ossOperator;
             case STORAGE_S3:
                 return S3Utils.getInstance();
             case STORAGE_HDFS:
@@ -50,5 +58,4 @@ public StorageOperate storageOperate() {
         }
     }
 
-
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java

@@ -21,10 +21,5 @@
  * data base types
  */
 public enum ResUploadType {
-  /**
-   * 0 hdfs
-   * 1 s3
-   * 2 none
-   */
-  HDFS,S3,NONE
+    HDFS, S3, OSS, NONE
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/factory/OssClientFactory.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.factory;
+
+import org.apache.dolphinscheduler.common.model.OssConnection;
+
+import lombok.experimental.UtilityClass;
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+
+@UtilityClass
+public class OssClientFactory {
+
+    public OSS buildOssClient(OssConnection ossConnection) {
+        return new OSSClientBuilder().build(ossConnection.getEndPoint(),
+                ossConnection.getAccessKeyId(), ossConnection.getAccessKeySecret());
+    }
+}