Merge pull request #1523 from dingxiaobo/opensoure_20220922

DataX 2209, MaxCompute、DataHub、SLS、MySQL、OceanBase、StarRocks、ElasticSearch
alibaba · Sep 30, 2022 · 5c554a7 · 5c554a7
2 parents ced5a45 + 874a256
commit 5c554a7
Show file tree

Hide file tree

Showing 133 changed files with 5,505 additions and 2,537 deletions.
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源
 
 # Quick Start
 
-##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/20220530/datax.tar.gz)
+##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202209/datax.tar.gz)
 
 
 ##### 请点击：[Quick Start](https://github.com/alibaba/DataX/blob/master/userGuid.md)
@@ -95,6 +95,9 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
 
 DataX 后续计划月度迭代更新，也欢迎感兴趣的同学提交 Pull requests，月度更新内容会介绍介绍如下。
 
+- [datax_v202209]（https://github.com/alibaba/DataX/releases/tag/datax_v202209)
+    - 涉及通道能力更新（MaxCompute、Datahub、SLS等）、安全漏洞更新、通用打包更新等
+
 - [datax_v202205]（https://github.com/alibaba/DataX/releases/tag/datax_v202205)
     - 涉及通道能力更新（MaxCompute、Hologres、OSS、Tdengine等）、安全漏洞更新、通用打包更新等
 

diff --git a/clickhousewriter/src/main/resources/plugin.json b/clickhousewriter/src/main/resources/plugin.json
@@ -2,5 +2,5 @@
     "name": "clickhousewriter",
     "class": "com.alibaba.datax.plugin.writer.clickhousewriter.ClickhouseWriter",
     "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql.",
-    "developer": "jiye.tjy"
+    "developer": "alibaba"
 }
diff --git a/common/src/main/java/com/alibaba/datax/common/util/Configuration.java b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
@@ -411,6 +411,15 @@ public List<Object> getList(final String path) {
 		return list;
 	}
 
+	public <T> List<T> getListWithJson(final String path, Class<T> t) {
+		Object object = this.get(path, List.class);
+		if (null == object) {
+			return null;
+		}
+
+		return JSON.parseArray(JSON.toJSONString(object),t);
+	}
+
 	/**
 	 * 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
 	 */

diff --git a/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java b/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java
@@ -3,6 +3,8 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.text.DecimalFormat;
 import java.util.HashMap;
 import java.util.Map;
@@ -82,4 +84,20 @@ public static String compressMiddle(String s, int headLength, int tailLength) {
         return s.substring(0, headLength) + "..." + s.substring(s.length() - tailLength);
     }
 
+    public static String getMd5(String plainText) {
+        try {
+            StringBuilder builder = new StringBuilder();
+            for (byte b : MessageDigest.getInstance("MD5").digest(plainText.getBytes())) {
+                int i = b & 0xff;
+                if (i < 0x10) {
+                    builder.append('0');
+                }
+                builder.append(Integer.toHexString(i));
+            }
+            return builder.toString();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }
diff --git a/datahubreader/pom.xml b/datahubreader/pom.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>datax-all</artifactId>
+        <groupId>com.alibaba.datax</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>datahubreader</artifactId>
+
+    <version>0.0.1-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>datax-common</artifactId>
+            <version>${datax-project-version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>slf4j-log4j12</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.datahub</groupId>
+            <artifactId>aliyun-sdk-datahub</artifactId>
+            <version>2.21.6-public</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- compiler plugin -->
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>${jdk-version}</source>
+                    <target>${jdk-version}</target>
+                    <encoding>${project-sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+            <!-- assembly plugin -->
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>src/main/assembly/package.xml</descriptor>
+                    </descriptors>
+                    <finalName>datax</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>dwzip</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/datahubreader/src/main/assembly/package.xml b/datahubreader/src/main/assembly/package.xml
@@ -0,0 +1,34 @@
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id></id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>plugin.json</include>
+            </includes>
+            <outputDirectory>plugin/reader/datahubreader</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>target/</directory>
+            <includes>
+                <include>datahubreader-0.0.1-SNAPSHOT.jar</include>
+            </includes>
+            <outputDirectory>plugin/reader/datahubreader</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>plugin/reader/datahubreader/libs</outputDirectory>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java
@@ -0,0 +1,8 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+public class Constant {
+
+    public static String DATETIME_FORMAT = "yyyyMMddHHmmss";
+    public static String DATE_FORMAT = "yyyyMMdd";
+
+}
diff --git a/...ader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java b/...ader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java
@@ -0,0 +1,42 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.aliyun.datahub.client.DatahubClient;
+import com.aliyun.datahub.client.DatahubClientBuilder;
+import com.aliyun.datahub.client.auth.Account;
+import com.aliyun.datahub.client.auth.AliyunAccount;
+import com.aliyun.datahub.client.common.DatahubConfig;
+import com.aliyun.datahub.client.http.HttpConfig;
+import org.apache.commons.lang3.StringUtils;
+
+public class DatahubClientHelper {
+    public static DatahubClient getDatahubClient(Configuration jobConfig) {
+        String accessId = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_ID,
+                DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+        String accessKey = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_KEY,
+                DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+        String endpoint = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ENDPOINT,
+                DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+        Account account = new AliyunAccount(accessId, accessKey);
+        // 是否开启二进制传输，服务端2.12版本开始支持
+        boolean enableBinary = jobConfig.getBool("enableBinary", false);
+        DatahubConfig datahubConfig = new DatahubConfig(endpoint, account, enableBinary);
+        // HttpConfig可不设置，不设置时采用默认值
+        // 读写数据推荐打开网络传输 LZ4压缩
+        HttpConfig httpConfig = null;
+        String httpConfigStr = jobConfig.getString("httpConfig");
+        if (StringUtils.isNotBlank(httpConfigStr)) {
+            httpConfig = JSON.parseObject(httpConfigStr, new TypeReference<HttpConfig>() {
+            });
+        }
+
+        DatahubClientBuilder builder = DatahubClientBuilder.newBuilder().setDatahubConfig(datahubConfig);
+        if (null != httpConfig) {
+            builder.setHttpConfig(httpConfig);
+        }
+        DatahubClient datahubClient = builder.build();
+        return datahubClient;
+    }
+}