getTypes() {
+ return TYPES;
+ }
+
+ /**
+ * Get the name of the Java class for the given value type.
+ *
+ * @param type the value type
+ * @return the class name
+ */
+ public static String getTypeClassName(int type) {
+ switch (type) {
+ case BOOLEAN:
+ // "java.lang.Boolean";
+ return Boolean.class.getName();
+ case BYTE:
+ // "java.lang.Byte";
+ return Byte.class.getName();
+ case SHORT:
+ // "java.lang.Short";
+ return Short.class.getName();
+ case INT:
+ // "java.lang.Integer";
+ return Integer.class.getName();
+ case LONG:
+ // "java.lang.Long";
+ return Long.class.getName();
+ case DECIMAL:
+ // "java.math.BigDecimal";
+ return BigDecimal.class.getName();
+ case TIME:
+ // "java.sql.Time";
+ return Time.class.getName();
+ case DATE:
+ // "java.sql.Date";
+ return Date.class.getName();
+ case TIMESTAMP:
+ // "java.sql.Timestamp";
+ return Timestamp.class.getName();
+ case STRING:
+ // case STRING_IGNORECASE:
+ // case STRING_FIXED:
+ case MULTI_VALUE:
+ // "java.lang.String";
+ return String.class.getName();
+ case DOUBLE:
+ // "java.lang.Double";
+ return Double.class.getName();
+ case FLOAT:
+ // "java.lang.Float";
+ return Float.class.getName();
+ // case NULL:
+ // return null;
+ default:
+ throw new IllegalArgumentException("type=" + type);
+ }
+ }
+
+ /**
+ * Get the data type object for the given value type.
+ *
+ * @param type the value type
+ * @return the data type object
+ */
+ public static ColumnDataType getDataType(int type) {
+ if (type < 0 || type >= TYPE_COUNT) {
+ throw new IllegalArgumentException("type=" + type);
+ }
+ ColumnDataType dt = TYPES_BY_VALUE_TYPE.get(type);
+ // if (dt == null) {
+ // dt = TYPES_BY_VALUE_TYPE.get(NULL);
+ // }
+ return dt;
+ }
+
+ /**
+ * Convert a value type to a SQL type.
+ *
+ * @param type the value type
+ * @return the SQL type
+ */
+ public static int convertTypeToSQLType(int type) {
+ return getDataType(type).sqlType;
+ }
+
+ /**
+ * Convert a SQL type to a value type.
+ *
+ * @param sqlType the SQL type
+ * @return the value type
+ */
+ public static int convertSQLTypeToValueType(int sqlType) {
+ switch (sqlType) {
+ // case Types.CHAR:
+ // case Types.NCHAR:
+ // return STRING_FIXED;
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.NVARCHAR:
+ case Types.LONGNVARCHAR:
+ return STRING;
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ return DECIMAL;
+ case Types.BIT:
+ case Types.BOOLEAN:
+ return BOOLEAN;
+ case Types.INTEGER:
+ return INT;
+ case Types.SMALLINT:
+ return SHORT;
+ case Types.TINYINT:
+ return BYTE;
+ case Types.BIGINT:
+ return LONG;
+ case Types.REAL:
+ return FLOAT;
+ case Types.DOUBLE:
+ case Types.FLOAT:
+ return DOUBLE;
+ case Types.DATE:
+ return DATE;
+ case Types.TIME:
+ return TIME;
+ case Types.TIMESTAMP:
+ return TIMESTAMP;
+ // case Types.NULL:
+ // return NULL;
+ default:
+ throw new IllegalArgumentException("JDBC Type: " + sqlType);
+ }
+ }
+
+ /**
+ * Get the value type for the given Java class.
+ *
+ * @param x the Java class
+ * @return the value type
+ */
+ public static int getTypeFromClass(Class x) {
+ // if (x == null || Void.TYPE == x) {
+ // return NULL;
+ // }
+ if (x.isPrimitive()) {
+ x = getNonPrimitiveClass(x);
+ }
+ if (String.class == x) {
+ return STRING;
+ } else if (Integer.class == x) {
+ return INT;
+ } else if (Long.class == x) {
+ return LONG;
+ } else if (Boolean.class == x) {
+ return BOOLEAN;
+ } else if (Double.class == x) {
+ return DOUBLE;
+ } else if (Byte.class == x) {
+ return BYTE;
+ } else if (Short.class == x) {
+ return SHORT;
+ } else if (Float.class == x) {
+ return FLOAT;
+ // } else if (Void.class == x) {
+ // return NULL;
+ } else if (BigDecimal.class.isAssignableFrom(x)) {
+ return DECIMAL;
+ } else if (Date.class.isAssignableFrom(x)) {
+ return DATE;
+ } else if (Time.class.isAssignableFrom(x)) {
+ return TIME;
+ } else if (Timestamp.class.isAssignableFrom(x)) {
+ return TIMESTAMP;
+ } else if (java.util.Date.class.isAssignableFrom(x)) {
+ return TIMESTAMP;
+ } else {
+ throw new IllegalArgumentException("class=" + x);
+ }
+ }
+
+ /**
+ * Convert primitive class names to java.lang.* class names.
+ *
+ * @param clazz the class (for example: int)
+ * @return the non-primitive class (for example: java.lang.Integer)
+ */
+ public static Class getNonPrimitiveClass(Class clazz) {
+ if (!clazz.isPrimitive()) {
+ return clazz;
+ } else if (clazz == boolean.class) {
+ return Boolean.class;
+ } else if (clazz == byte.class) {
+ return Byte.class;
+ } else if (clazz == char.class) {
+ return Character.class;
+ } else if (clazz == double.class) {
+ return Double.class;
+ } else if (clazz == float.class) {
+ return Float.class;
+ } else if (clazz == int.class) {
+ return Integer.class;
+ } else if (clazz == long.class) {
+ return Long.class;
+ } else if (clazz == short.class) {
+ return Short.class;
+ } else if (clazz == void.class) {
+ return Void.class;
+ }
+ return clazz;
+ }
+
+ /**
+ * Get a data type object from a type name.
+ *
+ * @param s the type name
+ * @return the data type object
+ */
+ public static ColumnDataType getTypeByName(String s) {
+ return TYPES_BY_NAME.get(s);
+ }
+
+ /**
+ * Check if the given value type is a String (VARCHAR,...).
+ *
+ * @param type the value type
+ * @return true if the value type is a String type
+ */
+ public static boolean isStringType(int type) {
+ if (type == STRING /* || type == STRING_FIXED || type == STRING_IGNORECASE */
+ || type == MULTI_VALUE) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @return
+ */
+ public boolean supportsAdd() {
+ return supportsAdd(type);
+ }
+
+ /**
+ * Check if the given value type supports the add operation.
+ *
+ * @param type the value type
+ * @return true if add is supported
+ */
+ public static boolean supportsAdd(int type) {
+ switch (type) {
+ case BYTE:
+ case DECIMAL:
+ case DOUBLE:
+ case FLOAT:
+ case INT:
+ case LONG:
+ case SHORT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Get the data type that will not overflow when calling 'add' 2 billion times.
+ *
+ * @param type the value type
+ * @return the data type that supports adding
+ */
+ public static int getAddProofType(int type) {
+ switch (type) {
+ case BYTE:
+ return LONG;
+ case FLOAT:
+ return DOUBLE;
+ case INT:
+ return LONG;
+ case LONG:
+ return DECIMAL;
+ case SHORT:
+ return LONG;
+ default:
+ return type;
+ }
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
new file mode 100644
index 0000000000..030ce35d10
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
@@ -0,0 +1,72 @@
+package com.alibaba.datax.plugin.writer.adswriter.ads;
+
+/**
+ * ADS column meta.
+ * select ordinal_position,column_name,data_type,type_name,column_comment
+ *
+ * @since 0.0.1
+ */
+public class ColumnInfo {
+
+ private int ordinal;
+ private String name;
+ private ColumnDataType dataType;
+ private boolean isDeleted;
+ private String comment;
+
+ public int getOrdinal() {
+ return ordinal;
+ }
+
+ public void setOrdinal(int ordinal) {
+ this.ordinal = ordinal;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public ColumnDataType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(ColumnDataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public boolean isDeleted() {
+ return isDeleted;
+ }
+
+ public void setDeleted(boolean isDeleted) {
+ this.isDeleted = isDeleted;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ColumnInfo [ordinal=").append(ordinal).append(", name=").append(name).append(", dataType=")
+ .append(dataType).append(", isDeleted=").append(isDeleted).append(", comment=").append(comment)
+ .append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
new file mode 100644
index 0000000000..eac324d1fd
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
@@ -0,0 +1,135 @@
+package com.alibaba.datax.plugin.writer.adswriter.ads;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ADS table meta.
+ * select table_schema, table_name,comments
+ *
+ * select ordinal_position,column_name,data_type,type_name,column_comment
+ *
+ * @since 0.0.1
+ */
+public class TableInfo {
+
+ private String tableSchema;
+ private String tableName;
+ private List columns;
+ private String comments;
+ private String tableType;
+
+ private String updateType;
+ private String partitionType;
+ private String partitionColumn;
+ private int partitionCount;
+ private List primaryKeyColumns;
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("TableInfo [tableSchema=").append(tableSchema).append(", tableName=").append(tableName)
+ .append(", columns=").append(columns).append(", comments=").append(comments).append(",updateType=").append(updateType)
+ .append(",partitionType=").append(partitionType).append(",partitionColumn=").append(partitionColumn).append(",partitionCount=").append(partitionCount)
+ .append(",primaryKeyColumns=").append(primaryKeyColumns).append("]");
+ return builder.toString();
+ }
+
+ public String getTableSchema() {
+ return tableSchema;
+ }
+
+ public void setTableSchema(String tableSchema) {
+ this.tableSchema = tableSchema;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List getColumns() {
+ return columns;
+ }
+
+ public List getColumnsNames() {
+ List columnNames = new ArrayList();
+ for (ColumnInfo column : this.getColumns()) {
+ columnNames.add(column.getName());
+ }
+ return columnNames;
+ }
+
+ public void setColumns(List columns) {
+ this.columns = columns;
+ }
+
+ public String getComments() {
+ return comments;
+ }
+
+ public void setComments(String comments) {
+ this.comments = comments;
+ }
+
+ public String getTableType() {
+ return tableType;
+ }
+
+ public void setTableType(String tableType) {
+ this.tableType = tableType;
+ }
+
+ public String getUpdateType() {
+ return updateType;
+ }
+
+ public void setUpdateType(String updateType) {
+ this.updateType = updateType;
+ }
+
+ public String getPartitionType() {
+ return partitionType;
+ }
+
+ public void setPartitionType(String partitionType) {
+ this.partitionType = partitionType;
+ }
+
+ public String getPartitionColumn() {
+ return partitionColumn;
+ }
+
+ public void setPartitionColumn(String partitionColumn) {
+ this.partitionColumn = partitionColumn;
+ }
+
+ public int getPartitionCount() {
+ return partitionCount;
+ }
+
+ public void setPartitionCount(int partitionCount) {
+ this.partitionCount = partitionCount;
+ }
+
+ public List getPrimaryKeyColumns() {
+ return primaryKeyColumns;
+ }
+
+ public void setPrimaryKeyColumns(List primaryKeyColumns) {
+ this.primaryKeyColumns = primaryKeyColumns;
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
new file mode 100644
index 0000000000..b396c49ffa
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ADS meta and service.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter.ads;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
new file mode 100644
index 0000000000..7211fb9755
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
@@ -0,0 +1,631 @@
+package com.alibaba.datax.plugin.writer.adswriter.insert;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.plugin.TaskPluginCollector;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.RetryUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
+import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
+import com.alibaba.datax.plugin.writer.adswriter.util.Key;
+import com.mysql.jdbc.JDBC4PreparedStatement;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+
+public class AdsInsertProxy {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsInsertProxy.class);
+ private static final boolean IS_DEBUG_ENABLE = LOG.isDebugEnabled();
+ private static final int MAX_EXCEPTION_CAUSE_ITER = 100;
+
+ private String table;
+ private List columns;
+ private TaskPluginCollector taskPluginCollector;
+ private Configuration configuration;
+ private Boolean emptyAsNull;
+
+ private String writeMode;
+
+ private String insertSqlPrefix;
+ private String deleteSqlPrefix;
+ private int opColumnIndex;
+ private String lastDmlMode;
+ // columnName:
+ private Map> adsTableColumnsMetaData;
+ private Map> userConfigColumnsMetaData;
+ // columnName: index @ ads column
+ private Map primaryKeyNameIndexMap;
+
+ private int retryTimeUpperLimit;
+ private Connection currentConnection;
+
+ private String partitionColumn;
+ private int partitionColumnIndex = -1;
+ private int partitionCount;
+
+ public AdsInsertProxy(String table, List columns, Configuration configuration, TaskPluginCollector taskPluginCollector, TableInfo tableInfo) {
+ this.table = table;
+ this.columns = columns;
+ this.configuration = configuration;
+ this.taskPluginCollector = taskPluginCollector;
+ this.emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
+ this.writeMode = configuration.getString(Key.WRITE_MODE);
+ this.insertSqlPrefix = String.format(Constant.INSERT_TEMPLATE, this.table, StringUtils.join(columns, ","));
+ this.deleteSqlPrefix = String.format(Constant.DELETE_TEMPLATE, this.table);
+ this.opColumnIndex = configuration.getInt(Key.OPIndex, 0);
+ this.retryTimeUpperLimit = configuration.getInt(
+ Key.RETRY_CONNECTION_TIME, Constant.DEFAULT_RETRY_TIMES);
+ this.partitionCount = tableInfo.getPartitionCount();
+ this.partitionColumn = tableInfo.getPartitionColumn();
+
+ //目前ads新建的表如果未插入数据不能通过select colums from table where 1=2，获取列信息，需要读取ads数据字典
+ //not this: this.resultSetMetaData = DBUtil.getColumnMetaData(connection, this.table, StringUtils.join(this.columns, ","));
+ //no retry here(fetch meta data) 注意实时表列换序的可能
+ this.adsTableColumnsMetaData = AdsInsertUtil.getColumnMetaData(tableInfo, this.columns);
+ this.userConfigColumnsMetaData = new HashMap>();
+
+ List primaryKeyColumnName = tableInfo.getPrimaryKeyColumns();
+ List adsColumnsNames = tableInfo.getColumnsNames();
+ this.primaryKeyNameIndexMap = new HashMap();
+ //warn: 要使用用户配置的column顺序, 不要使用从ads元数据获取的column顺序, 原来复用load列顺序其实有问题的
+ for (int i = 0; i < this.columns.size(); i++) {
+ String oriEachColumn = this.columns.get(i);
+ String eachColumn = oriEachColumn;
+ // 防御性保留字
+ if (eachColumn.startsWith(Constant.ADS_QUOTE_CHARACTER) && eachColumn.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
+ eachColumn = eachColumn.substring(1, eachColumn.length() - 1);
+ }
+ for (String eachPrimary : primaryKeyColumnName) {
+ if (eachColumn.equalsIgnoreCase(eachPrimary)) {
+ this.primaryKeyNameIndexMap.put(oriEachColumn, i);
+ }
+ }
+ for (String eachAdsColumn : adsColumnsNames) {
+ if (eachColumn.equalsIgnoreCase(eachAdsColumn)) {
+ this.userConfigColumnsMetaData.put(oriEachColumn, this.adsTableColumnsMetaData.get(eachAdsColumn));
+ }
+ }
+
+ // 根据第几个column分区列排序，ads实时表只有一级分区、最多256个分区
+ if (eachColumn.equalsIgnoreCase(this.partitionColumn)) {
+ this.partitionColumnIndex = i;
+ }
+ }
+ }
+
+ public void startWriteWithConnection(RecordReceiver recordReceiver,
+ Connection connection,
+ int columnNumber) {
+ this.currentConnection = connection;
+ int batchSize = this.configuration.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE);
+ // 默认情况下bufferSize需要和batchSize一致
+ int bufferSize = this.configuration.getInt(Key.BUFFER_SIZE, batchSize);
+ // insert缓冲，多个分区排序后insert合并发送到ads
+ List writeBuffer = new ArrayList(bufferSize);
+ List deleteBuffer = null;
+ if (this.writeMode.equalsIgnoreCase(Constant.STREAMMODE)) {
+ // delete缓冲，多个分区排序后delete合并发送到ads
+ deleteBuffer = new ArrayList(bufferSize);
+ }
+ try {
+ Record record;
+ while ((record = recordReceiver.getFromReader()) != null) {
+ if (this.writeMode.equalsIgnoreCase(Constant.INSERTMODE)) {
+ if (record.getColumnNumber() != columnNumber) {
+ // 源头读取字段列数与目的表字段写入列数不相等，直接报错
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.CONF_ERROR,
+ String.format(
+ "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.",
+ record.getColumnNumber(),
+ columnNumber));
+ }
+ writeBuffer.add(record);
+ if (writeBuffer.size() >= bufferSize) {
+ this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
+ writeBuffer.clear();
+ }
+ } else {
+ if (record.getColumnNumber() != columnNumber + 1) {
+ // 源头读取字段列数需要为目的表字段写入列数+1, 直接报错, 源头多了一列OP
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.CONF_ERROR,
+ String.format(
+ "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不满足源头多1列操作类型列. 请检查您的配置并作出修改.",
+ record.getColumnNumber(),
+ columnNumber));
+ }
+ String optionColumnValue = record.getColumn(this.opColumnIndex).asString();
+ OperationType operationType = OperationType.asOperationType(optionColumnValue);
+ if (operationType.isInsertTemplate()) {
+ writeBuffer.add(record);
+ if (this.lastDmlMode == null || this.lastDmlMode == Constant.INSERTMODE ) {
+ this.lastDmlMode = Constant.INSERTMODE;
+ if (writeBuffer.size() >= bufferSize) {
+ this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
+ writeBuffer.clear();
+ }
+ } else {
+ this.lastDmlMode = Constant.INSERTMODE;
+ // 模式变换触发一次提交ads delete, 并进入insert模式
+ this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
+ deleteBuffer.clear();
+ }
+ } else if (operationType.isDeleteTemplate()) {
+ deleteBuffer.add(record);
+ if (this.lastDmlMode == null || this.lastDmlMode == Constant.DELETEMODE ) {
+ this.lastDmlMode = Constant.DELETEMODE;
+ if (deleteBuffer.size() >= bufferSize) {
+ this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
+ deleteBuffer.clear();
+ }
+ } else {
+ this.lastDmlMode = Constant.DELETEMODE;
+ // 模式变换触发一次提交ads insert, 并进入delete模式
+ this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
+ writeBuffer.clear();
+ }
+ } else {
+ // 注意OP操作类型的脏数据, 这里不需要重试
+ this.taskPluginCollector.collectDirtyRecord(record, String.format("不支持您的更新类型:%s", optionColumnValue));
+ }
+ }
+ }
+
+ if (!writeBuffer.isEmpty()) {
+ //doOneRecord(writeBuffer, Constant.INSERTMODE);
+ this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
+ writeBuffer.clear();
+ }
+ // 2个缓冲最多一个不为空同时
+ if (null!= deleteBuffer && !deleteBuffer.isEmpty()) {
+ //doOneRecord(deleteBuffer, Constant.DELETEMODE);
+ this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
+ deleteBuffer.clear();
+ }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ writeBuffer.clear();
+ DBUtil.closeDBResources(null, null, connection);
+ }
+ }
+
+ /**
+ * @param bufferSize datax缓冲记录条数
+ * @param batchSize datax向ads系统一次发送数据条数
+ * @param buffer datax缓冲区
+ * @param mode 实时表模式insert 或者 stream
+ * */
+ private void doBatchRecordWithPartitionSort(List buffer, String mode, int bufferSize, int batchSize) throws SQLException{
+ //warn: 排序会影响数据插入顺序, 如果源头没有数据约束, 排序可能造成数据不一致, 快速排序是一种不稳定的排序算法
+ //warn: 不明确配置bufferSize或者小于batchSize的情况下，不要进行排序;如果缓冲区实际内容条数少于batchSize也不排序了，最后一次的余量
+ int recordBufferedNumber = buffer.size();
+ if (bufferSize > batchSize && recordBufferedNumber > batchSize && this.partitionColumnIndex >= 0) {
+ final int partitionColumnIndex = this.partitionColumnIndex;
+ final int partitionCount = this.partitionCount;
+ Collections.sort(buffer, new Comparator() {
+ @Override
+ public int compare(Record record1, Record record2) {
+ int hashPartition1 = AdsInsertProxy.getHashPartition(record1.getColumn(partitionColumnIndex).asString(), partitionCount);
+ int hashPartition2 = AdsInsertProxy.getHashPartition(record2.getColumn(partitionColumnIndex).asString(), partitionCount);
+ return hashPartition1 - hashPartition2;
+ }
+ });
+ }
+ // 将缓冲区的Record输出到ads, 使用recordBufferedNumber哦
+ for (int i = 0; i < recordBufferedNumber; i += batchSize) {
+ int toIndex = i + batchSize;
+ if (toIndex > recordBufferedNumber) {
+ toIndex = recordBufferedNumber;
+ }
+ this.doBatchRecord(buffer.subList(i, toIndex), mode);
+ }
+ }
+
+ private void doBatchRecord(final List buffer, final String mode) throws SQLException {
+ List> retryExceptionClasss = new ArrayList>();
+ retryExceptionClasss.add(com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class);
+ retryExceptionClasss.add(java.net.SocketException.class);
+ try {
+ RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public Boolean call() throws Exception {
+ doBatchRecordDml(buffer, mode);
+ return true;
+ }
+ }, this.retryTimeUpperLimit, 2000L, true, retryExceptionClasss);
+ }catch (SQLException e) {
+ LOG.warn(String.format("after retry %s times, doBatchRecord meet a exception: ", this.retryTimeUpperLimit), e);
+ LOG.info("try to re execute for each record...");
+ doOneRecord(buffer, mode);
+ // below is the old way
+ // for (Record eachRecord : buffer) {
+ // this.taskPluginCollector.collectDirtyRecord(eachRecord, e);
+ // }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ }
+ }
+
+ //warn: ADS 无法支持事物roll back都是不管用
+ @SuppressWarnings("resource")
+ private void doBatchRecordDml(List buffer, String mode) throws Exception {
+ Statement statement = null;
+ String sql = null;
+ try {
+ int bufferSize = buffer.size();
+ if (buffer.isEmpty()) {
+ return;
+ }
+ StringBuilder sqlSb = new StringBuilder();
+ // connection.setAutoCommit(true);
+ //mysql impl warn: if a database access error occurs or this method is called on a closed connection throw SQLException
+ statement = this.currentConnection.createStatement();
+ sqlSb.append(this.generateDmlSql(this.currentConnection, buffer.get(0), mode));
+ for (int i = 1; i < bufferSize; i++) {
+ Record record = buffer.get(i);
+ this.appendDmlSqlValues(this.currentConnection, record, sqlSb, mode);
+ }
+ sql = sqlSb.toString();
+ if (IS_DEBUG_ENABLE) {
+ LOG.debug(sql);
+ }
+ @SuppressWarnings("unused")
+ int status = statement.executeUpdate(sql);
+ sql = null;
+ } catch (SQLException e) {
+ LOG.warn("doBatchRecordDml meet a exception: " + sql, e);
+ Exception eachException = e;
+ int maxIter = 0;// 避免死循环
+ while (null != eachException && maxIter < AdsInsertProxy.MAX_EXCEPTION_CAUSE_ITER) {
+ if (this.isRetryable(eachException)) {
+ LOG.warn("doBatchRecordDml meet a retry exception: " + e.getMessage());
+ this.currentConnection = AdsUtil.getAdsConnect(this.configuration);
+ throw eachException;
+ } else {
+ try {
+ Throwable causeThrowable = eachException.getCause();
+ eachException = causeThrowable == null ? null : (Exception)causeThrowable;
+ } catch (Exception castException) {
+ LOG.warn("doBatchRecordDml meet a no! retry exception: " + e.getMessage());
+ throw e;
+ }
+ }
+ maxIter++;
+ }
+ throw e;
+ } catch (Exception e) {
+ LOG.error("插入异常, sql: " + sql);
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ DBUtil.closeDBResources(statement, null);
+ }
+ }
+
+ private void doOneRecord(List buffer, final String mode) {
+ List> retryExceptionClasss = new ArrayList>();
+ retryExceptionClasss.add(com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class);
+ retryExceptionClasss.add(java.net.SocketException.class);
+ for (final Record record : buffer) {
+ try {
+ RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public Boolean call() throws Exception {
+ doOneRecordDml(record, mode);
+ return true;
+ }
+ }, this.retryTimeUpperLimit, 2000L, true, retryExceptionClasss);
+ } catch (Exception e) {
+ // 不能重试的一行，记录脏数据
+ this.taskPluginCollector.collectDirtyRecord(record, e);
+ }
+ }
+ }
+
+ @SuppressWarnings("resource")
+ private void doOneRecordDml(Record record, String mode) throws Exception {
+ Statement statement = null;
+ String sql = null;
+ try {
+ // connection.setAutoCommit(true);
+ statement = this.currentConnection.createStatement();
+ sql = generateDmlSql(this.currentConnection, record, mode);
+ if (IS_DEBUG_ENABLE) {
+ LOG.debug(sql);
+ }
+ @SuppressWarnings("unused")
+ int status = statement.executeUpdate(sql);
+ sql = null;
+ } catch (SQLException e) {
+ LOG.error("doOneDml meet a exception: " + sql, e);
+ //need retry before record dirty data
+ //this.taskPluginCollector.collectDirtyRecord(record, e);
+ // 更新当前可用连接
+ Exception eachException = e;
+ int maxIter = 0;// 避免死循环
+ while (null != eachException && maxIter < AdsInsertProxy.MAX_EXCEPTION_CAUSE_ITER) {
+ if (this.isRetryable(eachException)) {
+ LOG.warn("doOneDml meet a retry exception: " + e.getMessage());
+ this.currentConnection = AdsUtil.getAdsConnect(this.configuration);
+ throw eachException;
+ } else {
+ try {
+ Throwable causeThrowable = eachException.getCause();
+ eachException = causeThrowable == null ? null : (Exception)causeThrowable;
+ } catch (Exception castException) {
+ LOG.warn("doOneDml meet a no! retry exception: " + e.getMessage());
+ throw e;
+ }
+ }
+ maxIter++;
+ }
+ throw e;
+ } catch (Exception e) {
+ LOG.error("插入异常, sql: " + sql);
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ DBUtil.closeDBResources(statement, null);
+ }
+ }
+
+ private boolean isRetryable(Throwable e) {
+ Class meetExceptionClass = e.getClass();
+ if (meetExceptionClass == com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class) {
+ return true;
+ }
+ if (meetExceptionClass == java.net.SocketException.class) {
+ return true;
+ }
+ return false;
+ }
+
+ private String generateDmlSql(Connection connection, Record record, String mode) throws SQLException {
+ String sql = null;
+ StringBuilder sqlSb = new StringBuilder();
+ if (mode.equalsIgnoreCase(Constant.INSERTMODE)) {
+ sqlSb.append(this.insertSqlPrefix);
+ sqlSb.append("(");
+ int columnsSize = this.columns.size();
+ for (int i = 0; i < columnsSize; i++) {
+ if((i + 1) != columnsSize) {
+ sqlSb.append("?,");
+ } else {
+ sqlSb.append("?");
+ }
+ }
+ sqlSb.append(")");
+ //mysql impl warn: if a database access error occurs or this method is called on a closed connection
+ PreparedStatement statement = connection.prepareStatement(sqlSb.toString());
+ for (int i = 0; i < this.columns.size(); i++) {
+ int preparedParamsIndex = i;
+ if (Constant.STREAMMODE.equalsIgnoreCase(this.writeMode)) {
+ if (preparedParamsIndex >= this.opColumnIndex) {
+ preparedParamsIndex = i + 1;
+ }
+ }
+ String columnName = this.columns.get(i);
+ int columnSqltype = this.userConfigColumnsMetaData.get(columnName).getLeft();
+ prepareColumnTypeValue(statement, columnSqltype, record.getColumn(preparedParamsIndex), i, columnName);
+ }
+ sql = ((JDBC4PreparedStatement) statement).asSql();
+ DBUtil.closeDBResources(statement, null);
+ } else {
+ sqlSb.append(this.deleteSqlPrefix);
+ sqlSb.append("(");
+ Set> primaryEntrySet = this.primaryKeyNameIndexMap.entrySet();
+ int entrySetSize = primaryEntrySet.size();
+ int i = 0;
+ for (Entry eachEntry : primaryEntrySet) {
+ if((i + 1) != entrySetSize) {
+ sqlSb.append(String.format(" (%s = ?) and ", eachEntry.getKey()));
+ } else {
+ sqlSb.append(String.format(" (%s = ?) ", eachEntry.getKey()));
+ }
+ i++;
+ }
+ sqlSb.append(")");
+ //mysql impl warn: if a database access error occurs or this method is called on a closed connection
+ PreparedStatement statement = connection.prepareStatement(sqlSb.toString());
+ i = 0;
+ //ads的real time表只能是1级分区、且分区列类型是long, 但是这里是需要主键删除的
+ for (Entry each : primaryEntrySet) {
+ String columnName = each.getKey();
+ int columnSqlType = this.userConfigColumnsMetaData.get(columnName).getLeft();
+ int primaryKeyInUserConfigIndex = this.primaryKeyNameIndexMap.get(columnName);
+ if (primaryKeyInUserConfigIndex >= this.opColumnIndex) {
+ primaryKeyInUserConfigIndex ++;
+ }
+ prepareColumnTypeValue(statement, columnSqlType, record.getColumn(primaryKeyInUserConfigIndex), i, columnName);
+ i++;
+ }
+ sql = ((JDBC4PreparedStatement) statement).asSql();
+ DBUtil.closeDBResources(statement, null);
+ }
+ return sql;
+ }
+
+ private void appendDmlSqlValues(Connection connection, Record record, StringBuilder sqlSb, String mode) throws SQLException {
+ String sqlResult = this.generateDmlSql(connection, record, mode);
+ if (mode.equalsIgnoreCase(Constant.INSERTMODE)) {
+ sqlSb.append(",");
+ sqlSb.append(sqlResult.substring(this.insertSqlPrefix.length()));
+ } else {
+ // 之前已经充分增加过括号了
+ sqlSb.append(" or ");
+ sqlSb.append(sqlResult.substring(this.deleteSqlPrefix.length()));
+ }
+ }
+
+ private void prepareColumnTypeValue(PreparedStatement statement, int columnSqltype, Column column, int preparedPatamIndex, String columnName) throws SQLException {
+ java.util.Date utilDate;
+ switch (columnSqltype) {
+ case Types.CHAR:
+ case Types.NCHAR:
+ case Types.CLOB:
+ case Types.NCLOB:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.NVARCHAR:
+ case Types.LONGNVARCHAR:
+ String strValue = column.asString();
+ statement.setString(preparedPatamIndex + 1, strValue);
+ break;
+
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ case Types.BIGINT:
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ case Types.REAL:
+ String numValue = column.asString();
+ if(emptyAsNull && "".equals(numValue) || numValue == null){
+ //statement.setObject(preparedPatamIndex + 1, null);
+ statement.setNull(preparedPatamIndex + 1, Types.BIGINT);
+ } else{
+ statement.setLong(preparedPatamIndex + 1, column.asLong());
+ }
+ break;
+
+ case Types.FLOAT:
+ case Types.DOUBLE:
+ String floatValue = column.asString();
+ if(emptyAsNull && "".equals(floatValue) || floatValue == null){
+ //statement.setObject(preparedPatamIndex + 1, null);
+ statement.setNull(preparedPatamIndex + 1, Types.DOUBLE);
+ } else{
+ statement.setDouble(preparedPatamIndex + 1, column.asDouble());
+ }
+ break;
+
+ //tinyint is a little special in some database like mysql {boolean->tinyint(1)}
+ case Types.TINYINT:
+ Long longValue = column.asLong();
+ if (null == longValue) {
+ statement.setNull(preparedPatamIndex + 1, Types.BIGINT);
+ } else {
+ statement.setLong(preparedPatamIndex + 1, longValue);
+ }
+
+ break;
+
+ case Types.DATE:
+ java.sql.Date sqlDate = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "Date 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlDate = new java.sql.Date(utilDate.getTime());
+ }
+ statement.setDate(preparedPatamIndex + 1, sqlDate);
+ break;
+
+ case Types.TIME:
+ java.sql.Time sqlTime = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "TIME 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTime = new java.sql.Time(utilDate.getTime());
+ }
+ statement.setTime(preparedPatamIndex + 1, sqlTime);
+ break;
+
+ case Types.TIMESTAMP:
+ java.sql.Timestamp sqlTimestamp = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "TIMESTAMP 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTimestamp = new java.sql.Timestamp(
+ utilDate.getTime());
+ }
+ statement.setTimestamp(preparedPatamIndex + 1, sqlTimestamp);
+ break;
+
+ case Types.BOOLEAN:
+ //case Types.BIT: ads 没有bit
+ Boolean booleanValue = column.asBoolean();
+ if (null == booleanValue) {
+ statement.setNull(preparedPatamIndex + 1, Types.BOOLEAN);
+ } else {
+ statement.setBoolean(preparedPatamIndex + 1, booleanValue);
+ }
+
+ break;
+ default:
+ Pair columnMetaPair = this.userConfigColumnsMetaData.get(columnName);
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.UNSUPPORTED_TYPE,
+ String.format(
+ "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库写入这种字段类型. 字段名:[%s], 字段类型:[%s], 字段Java类型:[%s]. 请修改表中该字段的类型或者不同步该字段.",
+ columnName, columnMetaPair.getRight(), columnMetaPair.getLeft()));
+ }
+ }
+
+ private static int getHashPartition(String value, int totalHashPartitionNum) {
+ long crc32 = (value == null ? getCRC32("-1") : getCRC32(value));
+ return (int) (crc32 % totalHashPartitionNum);
+ }
+
+ private static long getCRC32(String value) {
+ Checksum checksum = new CRC32();
+ byte[] bytes = value.getBytes();
+ checksum.update(bytes, 0, bytes.length);
+ return checksum.getValue();
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
new file mode 100644
index 0000000000..8e44e8c794
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
@@ -0,0 +1,153 @@
+package com.alibaba.datax.plugin.writer.adswriter.insert;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.ListUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.AdsException;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
+import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
+import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
+import com.alibaba.datax.plugin.writer.adswriter.util.Key;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class AdsInsertUtil {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsInsertUtil.class);
+
+ public static TableInfo getAdsTableInfo(Configuration conf) {
+ AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
+ TableInfo tableInfo= null;
+ try {
+ tableInfo = adsHelper.getTableInfo(conf.getString(Key.ADS_TABLE));
+ } catch (AdsException e) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
+ }
+ return tableInfo;
+ }
+
+ /*
+ * 返回列顺序为ads建表列顺序
+ * */
+ public static List getAdsTableColumnNames(Configuration conf) {
+ List tableColumns = new ArrayList();
+ AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
+ TableInfo tableInfo= null;
+ String adsTable = conf.getString(Key.ADS_TABLE);
+ try {
+ tableInfo = adsHelper.getTableInfo(adsTable);
+ } catch (AdsException e) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
+ }
+
+ List columnInfos = tableInfo.getColumns();
+ for(ColumnInfo columnInfo: columnInfos) {
+ tableColumns.add(columnInfo.getName());
+ }
+
+ LOG.info("table:[{}] all columns:[\n{}\n].", adsTable, StringUtils.join(tableColumns, ","));
+ return tableColumns;
+ }
+
+ public static Map> getColumnMetaData
+ (Configuration configuration, List userColumns) {
+ Map> columnMetaData = new HashMap>();
+ List columnInfoList = getAdsTableColumns(configuration);
+ for(String column : userColumns) {
+ if (column.startsWith(Constant.ADS_QUOTE_CHARACTER) && column.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
+ column = column.substring(1, column.length() - 1);
+ }
+ for (ColumnInfo columnInfo : columnInfoList) {
+ if(column.equalsIgnoreCase(columnInfo.getName())) {
+ Pair eachPair = new ImmutablePair(columnInfo.getDataType().sqlType, columnInfo.getDataType().name);
+ columnMetaData.put(columnInfo.getName(), eachPair);
+ }
+ }
+ }
+ return columnMetaData;
+ }
+
+ public static Map> getColumnMetaData(TableInfo tableInfo, List userColumns){
+ Map> columnMetaData = new HashMap>();
+ List columnInfoList = tableInfo.getColumns();
+ for(String column : userColumns) {
+ if (column.startsWith(Constant.ADS_QUOTE_CHARACTER) && column.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
+ column = column.substring(1, column.length() - 1);
+ }
+ for (ColumnInfo columnInfo : columnInfoList) {
+ if(column.equalsIgnoreCase(columnInfo.getName())) {
+ Pair eachPair = new ImmutablePair(columnInfo.getDataType().sqlType, columnInfo.getDataType().name);
+ columnMetaData.put(columnInfo.getName(), eachPair);
+ }
+ }
+ }
+ return columnMetaData;
+ }
+
+ /*
+ * 返回列顺序为ads建表列顺序
+ * */
+ public static List getAdsTableColumns(Configuration conf) {
+ AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
+ TableInfo tableInfo= null;
+ String adsTable = conf.getString(Key.ADS_TABLE);
+ try {
+ tableInfo = adsHelper.getTableInfo(adsTable);
+ } catch (AdsException e) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
+ }
+
+ List columnInfos = tableInfo.getColumns();
+
+ return columnInfos;
+ }
+
+ public static void dealColumnConf(Configuration originalConfig, List tableColumns) {
+ List userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class);
+ if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) {
+ throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
+ "您的配置文件中的列配置信息有误. 因为您未配置写入数据库表的列名称，DataX获取不到列信息. 请检查您的配置并作出修改.");
+ } else {
+ if (1 == userConfiguredColumns.size() && "*".equals(userConfiguredColumns.get(0))) {
+ LOG.warn("您的配置文件中的列配置信息存在风险. 因为您配置的写入数据库表的列为*，当您的表字段个数、类型有变动时，可能影响任务正确性甚至会运行出错。请检查您的配置并作出修改.");
+
+ // 回填其值，需要以 String 的方式转交后续处理
+ originalConfig.set(Key.COLUMN, tableColumns);
+ } else if (userConfiguredColumns.size() > tableColumns.size()) {
+ throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
+ String.format("您的配置文件中的列配置信息有误. 因为您所配置的写入数据库表的字段个数:%s 大于目的表的总字段总个数:%s. 请检查您的配置并作出修改.",
+ userConfiguredColumns.size(), tableColumns.size()));
+ } else {
+ // 确保用户配置的 column 不重复
+ ListUtil.makeSureNoValueDuplicate(userConfiguredColumns, false);
+ // 检查列是否都为数据库表中正确的列（通过执行一次 select column from table 进行判断）
+ // ListUtil.makeSureBInA(tableColumns, userConfiguredColumns, true);
+ // 支持关键字和保留字, ads列是不区分大小写的
+ List removeQuotedColumns = new ArrayList();
+ for (String each : userConfiguredColumns) {
+ if (each.startsWith(Constant.ADS_QUOTE_CHARACTER) && each.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
+ removeQuotedColumns.add(each.substring(1, each.length() - 1));
+ } else {
+ removeQuotedColumns.add(each);
+ }
+ }
+ ListUtil.makeSureBInA(tableColumns, removeQuotedColumns, false);
+ }
+ }
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java
new file mode 100644
index 0000000000..a689e70327
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java
@@ -0,0 +1,75 @@
+package com.alibaba.datax.plugin.writer.adswriter.insert;
+
+public enum OperationType {
+ // i: insert uo:before image uu:before image un: after image d: delete
+ // u:update
+ I("i"), UO("uo"), UU("uu"), UN("un"), D("d"), U("u"), UNKNOWN("unknown"), ;
+ private OperationType(String type) {
+ this.type = type;
+ }
+
+ private String type;
+
+ public String getType() {
+ return this.type;
+ }
+
+ public static OperationType asOperationType(String type) {
+ if ("i".equalsIgnoreCase(type)) {
+ return I;
+ } else if ("uo".equalsIgnoreCase(type)) {
+ return UO;
+ } else if ("uu".equalsIgnoreCase(type)) {
+ return UU;
+ } else if ("un".equalsIgnoreCase(type)) {
+ return UN;
+ } else if ("d".equalsIgnoreCase(type)) {
+ return D;
+ } else if ("u".equalsIgnoreCase(type)) {
+ return U;
+ } else {
+ return UNKNOWN;
+ }
+ }
+
+ public boolean isInsertTemplate() {
+ switch (this) {
+ // 建议merge 过后应该只有I和U两类
+ case I:
+ case UO:
+ case UU:
+ case UN:
+ case U:
+ return true;
+ case D:
+ return false;
+ default:
+ return false;
+ }
+ }
+
+ public boolean isDeleteTemplate() {
+ switch (this) {
+ // 建议merge 过后应该只有I和U两类
+ case I:
+ case UO:
+ case UU:
+ case UN:
+ case U:
+ return false;
+ case D:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public boolean isLegal() {
+ return this.type != UNKNOWN.getType();
+ }
+
+ @Override
+ public String toString() {
+ return this.name();
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
new file mode 100644
index 0000000000..924f6fcb61
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
@@ -0,0 +1,429 @@
+/**
+ *
+ */
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.RetryUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.writer.adswriter.AdsException;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+
+public class AdsHelper {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsHelper.class);
+
+ private String adsURL;
+ private String userName;
+ private String password;
+ private String schema;
+ private Long socketTimeout;
+ private String suffix;
+
+ public AdsHelper(String adsUrl, String userName, String password, String schema, Long socketTimeout, String suffix) {
+ this.adsURL = adsUrl;
+ this.userName = userName;
+ this.password = password;
+ this.schema = schema;
+ this.socketTimeout = socketTimeout;
+ this.suffix = suffix;
+ }
+
+ public String getAdsURL() {
+ return adsURL;
+ }
+
+ public void setAdsURL(String adsURL) {
+ this.adsURL = adsURL;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ /**
+ * Obtain the table meta information.
+ *
+ * @param table The table
+ * @return The table meta information
+ * @throws com.alibaba.datax.plugin.writer.adswriter.AdsException
+ */
+ public TableInfo getTableInfo(String table) throws AdsException {
+
+ if (table == null) {
+ throw new AdsException(AdsException.ADS_TABLEMETA_TABLE_NULL, "Table is null.", null);
+ }
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
+
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url, connectionProps);
+ statement = connection.createStatement();
+ // ads 表名、schema名不区分大小写, 提高用户易用性, 注意列顺序性
+ String columnMetaSql = String.format("select ordinal_position,column_name,data_type,type_name,column_comment from information_schema.columns where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'` order by ordinal_position", schema.toLowerCase(), table.toLowerCase());
+ LOG.info(String.format("检查列信息sql语句:%s", columnMetaSql));
+ rs = statement.executeQuery(columnMetaSql);
+
+ TableInfo tableInfo = new TableInfo();
+ List columnInfoList = new ArrayList();
+ while (DBUtil.asyncResultSetNext(rs)) {
+ ColumnInfo columnInfo = new ColumnInfo();
+ columnInfo.setOrdinal(rs.getInt(1));
+ columnInfo.setName(rs.getString(2));
+ //columnInfo.setDataType(ColumnDataType.getDataType(rs.getInt(3))); //for ads version < 0.7
+ //columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(3).toUpperCase())); //for ads version 0.8
+ columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(4).toUpperCase())); //for ads version 0.8 & 0.7
+ columnInfo.setComment(rs.getString(5));
+ columnInfoList.add(columnInfo);
+ }
+ if (columnInfoList.isEmpty()) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.NO_ADS_TABLE, table + "不存在或者查询不到列信息. ");
+ }
+ tableInfo.setColumns(columnInfoList);
+ tableInfo.setTableSchema(schema);
+ tableInfo.setTableName(table);
+ DBUtil.closeDBResources(rs, statement, null);
+
+ String tableMetaSql = String.format("select update_type, partition_type, partition_column, partition_count, primary_key_columns from information_schema.tables where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'`", schema.toLowerCase(), table.toLowerCase());
+ LOG.info(String.format("检查表信息sql语句:%s", tableMetaSql));
+ statement = connection.createStatement();
+ rs = statement.executeQuery(tableMetaSql);
+ while (DBUtil.asyncResultSetNext(rs)) {
+ tableInfo.setUpdateType(rs.getString(1));
+ tableInfo.setPartitionType(rs.getString(2));
+ tableInfo.setPartitionColumn(rs.getString(3));
+ tableInfo.setPartitionCount(rs.getInt(4));
+ //primary_key_columns ads主键是逗号分隔的，可以有多个
+ String primaryKeyColumns = rs.getString(5);
+ if (StringUtils.isNotBlank(primaryKeyColumns)) {
+ tableInfo.setPrimaryKeyColumns(Arrays.asList(StringUtils.split(primaryKeyColumns, ",")));
+ } else {
+ tableInfo.setPrimaryKeyColumns(null);
+ }
+ break;
+ }
+ DBUtil.closeDBResources(rs, statement, null);
+ return tableInfo;
+
+ } catch (ClassNotFoundException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch ( DataXException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Submit LOAD DATA command.
+ *
+ * @param table The target ADS table
+ * @param partition The partition option in the form of "(partition_name,...)"
+ * @param sourcePath The source path
+ * @param overwrite
+ * @return
+ * @throws AdsException
+ */
+ public String loadData(String table, String partition, String sourcePath, boolean overwrite) throws AdsException {
+
+ if (table == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_TABLE_NULL, "ADS LOAD DATA table is null.", null);
+ }
+
+ if (sourcePath == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_SOURCEPATH_NULL, "ADS LOAD DATA source path is null.",
+ null);
+ }
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("LOAD DATA FROM ");
+ if (sourcePath.startsWith("'") && sourcePath.endsWith("'")) {
+ sb.append(sourcePath);
+ } else {
+ sb.append("'" + sourcePath + "'");
+ }
+ if (overwrite) {
+ sb.append(" OVERWRITE");
+ }
+ sb.append(" INTO TABLE ");
+ sb.append(schema + "." + table);
+ if (partition != null && !partition.trim().equals("")) {
+ String partitionTrim = partition.trim();
+ if(partitionTrim.startsWith("(") && partitionTrim.endsWith(")")) {
+ sb.append(" PARTITION " + partition);
+ } else {
+ sb.append(" PARTITION " + "(" + partition + ")");
+ }
+ }
+
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url, connectionProps);
+ statement = connection.createStatement();
+ LOG.info("正在从ODPS数据库导数据到ADS中: "+sb.toString());
+ LOG.info("由于ADS的限制，ADS导数据最少需要20分钟，请耐心等待");
+ rs = statement.executeQuery(sb.toString());
+
+ String jobId = null;
+ while (DBUtil.asyncResultSetNext(rs)) {
+ jobId = rs.getString(1);
+ }
+
+ if (jobId == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_JOBID_NOT_AVAIL,
+ "Job id is not available for the submitted LOAD DATA." + jobId, null);
+ }
+
+ return jobId;
+
+ } catch (ClassNotFoundException e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Check the load data job status.
+ *
+ * @param jobId The job id to
+ * @return true if load data job succeeded, false if load data job failed.
+ * @throws AdsException
+ */
+ public boolean checkLoadDataJobStatus(String jobId) throws AdsException {
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ try {
+ String state = this.checkLoadDataJobStatusWithRetry(jobId);
+ if (state == null) {
+ throw new AdsException(AdsException.JOB_NOT_EXIST, "Target job does not exist for id: " + jobId, null);
+ }
+ if (state.equals("SUCCEEDED")) {
+ return true;
+ } else if (state.equals("FAILED")) {
+ throw new AdsException(AdsException.JOB_FAILED, "Target job failed for id: " + jobId, null);
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ }
+ }
+
+ private String checkLoadDataJobStatusWithRetry(final String jobId)
+ throws AdsException {
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ final String finalAdsUrl = this.adsURL;
+ final String finalSchema = this.schema;
+ final Long finalSocketTimeout = this.socketTimeout;
+ final String suffix = this.suffix;
+ return RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public String call() throws Exception {
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+
+ String url = AdsUtil.prepareJdbcUrl(finalAdsUrl, finalSchema, finalSocketTimeout, suffix);
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url,
+ connectionProps);
+ statement = connection.createStatement();
+
+ String sql = "select state from information_schema.job_instances where job_id like '"
+ + jobId + "'";
+ rs = statement.executeQuery(sql);
+ String state = null;
+ while (DBUtil.asyncResultSetNext(rs)) {
+ state = rs.getString(1);
+ }
+ return state;
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+ }
+ }, 3, 1000L, true);
+ } catch (Exception e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ }
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
new file mode 100644
index 0000000000..1ecad7561d
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
@@ -0,0 +1,87 @@
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.odps.DataType;
+import com.alibaba.datax.plugin.writer.adswriter.odps.FieldSchema;
+import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Table meta helper for ADS writer.
+ *
+ * @since 0.0.1
+ */
+public class TableMetaHelper {
+
+ private TableMetaHelper() {
+ }
+
+ /**
+ * Create temporary ODPS table.
+ *
+ * @param tableMeta table meta
+ * @param lifeCycle for temporary table
+ * @return ODPS temporary table meta
+ */
+ public static TableMeta createTempODPSTable(TableInfo tableMeta, int lifeCycle) {
+ TableMeta tempTable = new TableMeta();
+ tempTable.setComment(tableMeta.getComments());
+ tempTable.setLifeCycle(lifeCycle);
+ String tableSchema = tableMeta.getTableSchema();
+ String tableName = tableMeta.getTableName();
+ tempTable.setTableName(generateTempTableName(tableSchema, tableName));
+ List tempColumns = new ArrayList();
+ List columns = tableMeta.getColumns();
+ for (ColumnInfo column : columns) {
+ FieldSchema tempColumn = new FieldSchema();
+ tempColumn.setName(column.getName());
+ tempColumn.setType(toODPSDataType(column.getDataType()));
+ tempColumn.setComment(column.getComment());
+ tempColumns.add(tempColumn);
+ }
+ tempTable.setCols(tempColumns);
+ tempTable.setPartitionKeys(null);
+ return tempTable;
+ }
+
+ private static String toODPSDataType(ColumnDataType columnDataType) {
+ int type;
+ switch (columnDataType.type) {
+ case ColumnDataType.BOOLEAN:
+ type = DataType.STRING;
+ break;
+ case ColumnDataType.BYTE:
+ case ColumnDataType.SHORT:
+ case ColumnDataType.INT:
+ case ColumnDataType.LONG:
+ type = DataType.INTEGER;
+ break;
+ case ColumnDataType.DECIMAL:
+ case ColumnDataType.DOUBLE:
+ case ColumnDataType.FLOAT:
+ type = DataType.DOUBLE;
+ break;
+ case ColumnDataType.DATE:
+ case ColumnDataType.TIME:
+ case ColumnDataType.TIMESTAMP:
+ case ColumnDataType.STRING:
+ case ColumnDataType.MULTI_VALUE:
+ type = DataType.STRING;
+ break;
+ default:
+ throw new IllegalArgumentException("columnDataType=" + columnDataType);
+ }
+ return DataType.toString(type);
+ }
+
+ private static String generateTempTableName(String tableSchema, String tableName) {
+ int randNum = 1000 + new Random(System.currentTimeMillis()).nextInt(1000);
+ return tableSchema + "__" + tableName + "_" + System.currentTimeMillis() + randNum;
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
new file mode 100644
index 0000000000..bff4b7b900
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
@@ -0,0 +1,65 @@
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.common.util.Configuration;
+
+/**
+ * Created by xiafei.qiuxf on 15/4/13.
+ */
+public class TransferProjectConf {
+
+ public final static String KEY_ACCESS_ID = "odps.accessId";
+ public final static String KEY_ACCESS_KEY = "odps.accessKey";
+ public final static String KEY_ACCOUNT = "odps.account";
+ public final static String KEY_ODPS_SERVER = "odps.odpsServer";
+ public final static String KEY_ODPS_TUNNEL = "odps.tunnelServer";
+ public final static String KEY_ACCOUNT_TYPE = "odps.accountType";
+ public final static String KEY_PROJECT = "odps.project";
+
+ private String accessId;
+ private String accessKey;
+ private String account;
+ private String odpsServer;
+ private String odpsTunnel;
+ private String accountType;
+ private String project;
+
+ public static TransferProjectConf create(Configuration adsWriterConf) {
+ TransferProjectConf res = new TransferProjectConf();
+ res.accessId = adsWriterConf.getString(KEY_ACCESS_ID);
+ res.accessKey = adsWriterConf.getString(KEY_ACCESS_KEY);
+ res.account = adsWriterConf.getString(KEY_ACCOUNT);
+ res.odpsServer = adsWriterConf.getString(KEY_ODPS_SERVER);
+ res.odpsTunnel = adsWriterConf.getString(KEY_ODPS_TUNNEL);
+ res.accountType = adsWriterConf.getString(KEY_ACCOUNT_TYPE, "aliyun");
+ res.project = adsWriterConf.getString(KEY_PROJECT);
+ return res;
+ }
+
+ public String getAccessId() {
+ return accessId;
+ }
+
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ public String getAccount() {
+ return account;
+ }
+
+ public String getOdpsServer() {
+ return odpsServer;
+ }
+
+ public String getOdpsTunnel() {
+ return odpsTunnel;
+ }
+
+ public String getAccountType() {
+ return accountType;
+ }
+
+ public String getProject() {
+ return project;
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
new file mode 100644
index 0000000000..595b1dfd26
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
@@ -0,0 +1,77 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+/**
+ * ODPS 数据类型.
+ *
+ * 当前定义了如下类型：
+ *
+ * INTEGER
+ * DOUBLE
+ * BOOLEAN
+ * STRING
+ * DATETIME
+ *
+ *
+ *
+ * @since 0.0.1
+ */
+public class DataType {
+
+ public final static byte INTEGER = 0;
+ public final static byte DOUBLE = 1;
+ public final static byte BOOLEAN = 2;
+ public final static byte STRING = 3;
+ public final static byte DATETIME = 4;
+
+ public static String toString(int type) {
+ switch (type) {
+ case INTEGER:
+ return "bigint";
+ case DOUBLE:
+ return "double";
+ case BOOLEAN:
+ return "boolean";
+ case STRING:
+ return "string";
+ case DATETIME:
+ return "datetime";
+ default:
+ throw new IllegalArgumentException("type=" + type);
+ }
+ }
+
+ /**
+ * 字符串的数据类型转换为byte常量定义的数据类型.
+ *
+ * 转换规则：
+ *
+ * tinyint, int, bigint, long - {@link #INTEGER}
+ * double, float - {@link #DOUBLE}
+ * string - {@link #STRING}
+ * boolean, bool - {@link #BOOLEAN}
+ * datetime - {@link #DATETIME}
+ *
+ *
+ *
+ * @param type 字符串的数据类型
+ * @return byte常量定义的数据类型
+ * @throws IllegalArgumentException
+ */
+ public static byte convertToDataType(String type) throws IllegalArgumentException {
+ type = type.toLowerCase().trim();
+ if ("string".equals(type)) {
+ return STRING;
+ } else if ("bigint".equals(type) || "int".equals(type) || "tinyint".equals(type) || "long".equals(type)) {
+ return INTEGER;
+ } else if ("boolean".equals(type) || "bool".equals(type)) {
+ return BOOLEAN;
+ } else if ("double".equals(type) || "float".equals(type)) {
+ return DOUBLE;
+ } else if ("datetime".equals(type)) {
+ return DATETIME;
+ } else {
+ throw new IllegalArgumentException("unkown type: " + type);
+ }
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java
new file mode 100644
index 0000000000..701ee261cf
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java
@@ -0,0 +1,63 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+/**
+ * ODPS列属性，包含列名和类型 列名和类型与SQL的DESC表或分区显示的列名和类型一致
+ *
+ * @since 0.0.1
+ */
+public class FieldSchema {
+
+ /** 列名 */
+ private String name;
+
+ /** 列类型，如：string, bigint, boolean, datetime等等 */
+ private String type;
+
+ private String comment;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("FieldSchema [name=").append(name).append(", type=").append(type).append(", comment=")
+ .append(comment).append("]");
+ return builder.toString();
+ }
+
+ /**
+ * @return "col_name data_type [COMMENT col_comment]"
+ */
+ public String toDDL() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(name).append(" ").append(type);
+ String comment = this.comment;
+ if (comment != null && comment.length() > 0) {
+ builder.append(" ").append("COMMENT \"" + comment + "\"");
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java
new file mode 100644
index 0000000000..d0adc4eae6
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java
@@ -0,0 +1,114 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * ODPS table meta.
+ *
+ * @since 0.0.1
+ */
+public class TableMeta {
+
+ private String tableName;
+
+ private List cols;
+
+ private List partitionKeys;
+
+ private int lifeCycle;
+
+ private String comment;
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List getCols() {
+ return cols;
+ }
+
+ public void setCols(List cols) {
+ this.cols = cols;
+ }
+
+ public List getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ public void setPartitionKeys(List partitionKeys) {
+ this.partitionKeys = partitionKeys;
+ }
+
+ public int getLifeCycle() {
+ return lifeCycle;
+ }
+
+ public void setLifeCycle(int lifeCycle) {
+ this.lifeCycle = lifeCycle;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("TableMeta [tableName=").append(tableName).append(", cols=").append(cols)
+ .append(", partitionKeys=").append(partitionKeys).append(", lifeCycle=").append(lifeCycle)
+ .append(", comment=").append(comment).append("]");
+ return builder.toString();
+ }
+
+ /**
+ * @return cols = this.cols;
+ if (cols != null && cols.size() > 0) {
+ builder.append("(").append(toDDL(cols)).append(")").append(" ");
+ }
+ String comment = this.comment;
+ if (comment != null && comment.length() > 0) {
+ builder.append("COMMENT \"" + comment + "\" ");
+ }
+ List partitionKeys = this.partitionKeys;
+ if (partitionKeys != null && partitionKeys.size() > 0) {
+ builder.append("PARTITIONED BY ");
+ builder.append("(").append(toDDL(partitionKeys)).append(")").append(" ");
+ }
+ if (lifeCycle > 0) {
+ builder.append("LIFECYCLE " + lifeCycle).append(" ");
+ }
+ builder.append(";");
+ return builder.toString();
+ }
+
+ private String toDDL(List cols) {
+ StringBuilder builder = new StringBuilder();
+ Iterator iter = cols.iterator();
+ builder.append(iter.next().toDDL());
+ while (iter.hasNext()) {
+ builder.append(", ").append(iter.next().toDDL());
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java
new file mode 100644
index 0000000000..92dfd09da4
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ODPS meta.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter.odps;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java
new file mode 100644
index 0000000000..139a39106a
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ADS Writer.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java
new file mode 100644
index 0000000000..4336d4773f
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java
@@ -0,0 +1,175 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.load.TransferProjectConf;
+import com.alibaba.datax.plugin.writer.adswriter.odps.FieldSchema;
+import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AdsUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(AdsUtil.class);
+
+ /*检查配置文件中必填的配置项是否都已填
+ * */
+ public static void checkNecessaryConfig(Configuration originalConfig, String writeMode) {
+ //检查ADS必要参数
+ originalConfig.getNecessaryValue(Key.ADS_URL,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.USERNAME,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.PASSWORD,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.SCHEMA,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ if(Constant.LOADMODE.equals(writeMode)) {
+ originalConfig.getNecessaryValue(Key.Life_CYCLE,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ Integer lifeCycle = originalConfig.getInt(Key.Life_CYCLE);
+ if (lifeCycle <= 0) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE, "配置项[lifeCycle]的值必须大于零.");
+ }
+ originalConfig.getNecessaryValue(Key.ADS_TABLE,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ Boolean overwrite = originalConfig.getBool(Key.OVER_WRITE);
+ if (overwrite == null) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.REQUIRED_VALUE, "配置项[overWrite]是必填项.");
+ }
+ }
+ if (Constant.STREAMMODE.equalsIgnoreCase(writeMode)) {
+ originalConfig.getNecessaryValue(Key.OPIndex, AdsWriterErrorCode.REQUIRED_VALUE);
+ }
+ }
+
+ /*生成AdsHelp实例
+ * */
+ public static AdsHelper createAdsHelper(Configuration originalConfig){
+ //Get adsUrl,userName,password,schema等参数,创建AdsHelp实例
+ String adsUrl = originalConfig.getString(Key.ADS_URL);
+ String userName = originalConfig.getString(Key.USERNAME);
+ String password = originalConfig.getString(Key.PASSWORD);
+ String schema = originalConfig.getString(Key.SCHEMA);
+ Long socketTimeout = originalConfig.getLong(Key.SOCKET_TIMEOUT, Constant.DEFAULT_SOCKET_TIMEOUT);
+ String suffix = originalConfig.getString(Key.JDBC_URL_SUFFIX, "");
+ return new AdsHelper(adsUrl,userName,password,schema,socketTimeout,suffix);
+ }
+
+ public static AdsHelper createAdsHelperWithOdpsAccount(Configuration originalConfig) {
+ String adsUrl = originalConfig.getString(Key.ADS_URL);
+ String userName = originalConfig.getString(TransferProjectConf.KEY_ACCESS_ID);
+ String password = originalConfig.getString(TransferProjectConf.KEY_ACCESS_KEY);
+ String schema = originalConfig.getString(Key.SCHEMA);
+ Long socketTimeout = originalConfig.getLong(Key.SOCKET_TIMEOUT, Constant.DEFAULT_SOCKET_TIMEOUT);
+ String suffix = originalConfig.getString(Key.JDBC_URL_SUFFIX, "");
+ return new AdsHelper(adsUrl, userName, password, schema,socketTimeout,suffix);
+ }
+
+ /*生成ODPSWriter Plugin所需要的配置文件
+ * */
+ public static Configuration generateConf(Configuration originalConfig, String odpsTableName, TableMeta tableMeta, TransferProjectConf transConf){
+ Configuration newConfig = originalConfig.clone();
+ newConfig.set(Key.ODPSTABLENAME, odpsTableName);
+ newConfig.set(Key.ODPS_SERVER, transConf.getOdpsServer());
+ newConfig.set(Key.TUNNEL_SERVER,transConf.getOdpsTunnel());
+ newConfig.set(Key.ACCESS_ID,transConf.getAccessId());
+ newConfig.set(Key.ACCESS_KEY,transConf.getAccessKey());
+ newConfig.set(Key.PROJECT,transConf.getProject());
+ newConfig.set(Key.TRUNCATE, true);
+ newConfig.set(Key.PARTITION,null);
+// newConfig.remove(Key.PARTITION);
+ List cols = tableMeta.getCols();
+ List allColumns = new ArrayList();
+ if(cols != null && !cols.isEmpty()){
+ for(FieldSchema col:cols){
+ allColumns.add(col.getName());
+ }
+ }
+ newConfig.set(Key.COLUMN,allColumns);
+ return newConfig;
+ }
+
+ /*生成ADS数据导入时的source_path
+ * */
+ public static String generateSourcePath(String project, String tmpOdpsTableName, String odpsPartition){
+ StringBuilder builder = new StringBuilder();
+ String partition = transferOdpsPartitionToAds(odpsPartition);
+ builder.append("odps://").append(project).append("/").append(tmpOdpsTableName);
+ if(odpsPartition != null && !odpsPartition.isEmpty()){
+ builder.append("/").append(partition);
+ }
+ return builder.toString();
+ }
+
+ public static String transferOdpsPartitionToAds(String odpsPartition){
+ if(odpsPartition == null || odpsPartition.isEmpty())
+ return null;
+ String adsPartition = formatPartition(odpsPartition);;
+ String[] partitions = adsPartition.split("/");
+ for(int last = partitions.length; last > 0; last--){
+
+ String partitionPart = partitions[last-1];
+ String newPart = partitionPart.replace(".*", "*").replace("*", ".*");
+ if(newPart.split("=")[1].equals(".*")){
+ adsPartition = adsPartition.substring(0,adsPartition.length()-partitionPart.length());
+ }else{
+ break;
+ }
+ if(adsPartition.endsWith("/")){
+ adsPartition = adsPartition.substring(0,adsPartition.length()-1);
+ }
+ }
+ if (adsPartition.contains("*"))
+ throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_PARTITION_FAILED, "");
+ return adsPartition;
+ }
+
+ public static String formatPartition(String partition) {
+ return partition.trim().replaceAll(" *= *", "=")
+ .replaceAll(" */ *", ",").replaceAll(" *, *", ",")
+ .replaceAll("'", "").replaceAll(",", "/");
+ }
+
+ public static String prepareJdbcUrl(Configuration conf) {
+ String adsURL = conf.getString(Key.ADS_URL);
+ String schema = conf.getString(Key.SCHEMA);
+ Long socketTimeout = conf.getLong(Key.SOCKET_TIMEOUT,
+ Constant.DEFAULT_SOCKET_TIMEOUT);
+ String suffix = conf.getString(Key.JDBC_URL_SUFFIX, "");
+ return AdsUtil.prepareJdbcUrl(adsURL, schema, socketTimeout, suffix);
+ }
+
+ public static String prepareJdbcUrl(String adsURL, String schema,
+ Long socketTimeout, String suffix) {
+ String jdbcUrl = null;
+ // like autoReconnect=true&failOverReadOnly=false&maxReconnects=10
+ if (StringUtils.isNotBlank(suffix)) {
+ jdbcUrl = String
+ .format("jdbc:mysql://%s/%s?useUnicode=true&characterEncoding=UTF-8&socketTimeout=%s&%s",
+ adsURL, schema, socketTimeout, suffix);
+ } else {
+ jdbcUrl = String
+ .format("jdbc:mysql://%s/%s?useUnicode=true&characterEncoding=UTF-8&socketTimeout=%s",
+ adsURL, schema, socketTimeout);
+ }
+ return jdbcUrl;
+ }
+
+ public static Connection getAdsConnect(Configuration conf) {
+ String userName = conf.getString(Key.USERNAME);
+ String passWord = conf.getString(Key.PASSWORD);
+ String jdbcUrl = AdsUtil.prepareJdbcUrl(conf);
+ Connection connection = DBUtil.getConnection(DataBaseType.ADS, jdbcUrl, userName, passWord);
+ return connection;
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java
new file mode 100644
index 0000000000..f0ab71ec18
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java
@@ -0,0 +1,29 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+public class Constant {
+
+ public static final String LOADMODE = "load";
+
+ public static final String INSERTMODE = "insert";
+
+ public static final String DELETEMODE = "delete";
+
+ public static final String REPLACEMODE = "replace";
+
+ public static final String STREAMMODE = "stream";
+
+ public static final int DEFAULT_BATCH_SIZE = 32;
+
+ public static final long DEFAULT_SOCKET_TIMEOUT = 3600000L;
+
+ public static final int DEFAULT_RETRY_TIMES = 2;
+
+ public static final String INSERT_TEMPLATE = "insert into %s ( %s ) values ";
+
+ public static final String DELETE_TEMPLATE = "delete from %s where ";
+
+ public static final String ADS_TABLE_INFO = "adsTableInfo";
+
+ public static final String ADS_QUOTE_CHARACTER = "`";
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java
new file mode 100644
index 0000000000..3d31c8186f
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java
@@ -0,0 +1,62 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+
+public final class Key {
+
+ public final static String ADS_URL = "url";
+
+ public final static String USERNAME = "username";
+
+ public final static String PASSWORD = "password";
+
+ public final static String SCHEMA = "schema";
+
+ public final static String ADS_TABLE = "table";
+
+ public final static String Life_CYCLE = "lifeCycle";
+
+ public final static String OVER_WRITE = "overWrite";
+
+ public final static String WRITE_MODE = "writeMode";
+
+
+ public final static String COLUMN = "column";
+
+ public final static String OPIndex = "opIndex";
+
+ public final static String EMPTY_AS_NULL = "emptyAsNull";
+
+ public final static String BATCH_SIZE = "batchSize";
+
+ public final static String BUFFER_SIZE = "bufferSize";
+
+ public final static String PRE_SQL = "preSql";
+
+ public final static String POST_SQL = "postSql";
+
+ public final static String SOCKET_TIMEOUT = "socketTimeout";
+
+ public final static String RETRY_CONNECTION_TIME = "retryTimes";
+
+ public final static String JDBC_URL_SUFFIX = "urlSuffix";
+
+ /**
+ * 以下是odps writer的key
+ */
+ public final static String PARTITION = "partition";
+
+ public final static String ODPSTABLENAME = "table";
+
+ public final static String ODPS_SERVER = "odpsServer";
+
+ public final static String TUNNEL_SERVER = "tunnelServer";
+
+ public final static String ACCESS_ID = "accessId";
+
+ public final static String ACCESS_KEY = "accessKey";
+
+ public final static String PROJECT = "project";
+
+ public final static String TRUNCATE = "truncate";
+
+}
\ No newline at end of file
diff --git a/adswriter/src/main/resources/plugin.json b/adswriter/src/main/resources/plugin.json
new file mode 100644
index 0000000000..a70fb36462
--- /dev/null
+++ b/adswriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "adswriter",
+ "class": "com.alibaba.datax.plugin.writer.adswriter.AdsWriter",
+ "description": "",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/adswriter/src/main/resources/plugin_job_template.json b/adswriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 0000000000..0753a226e8
--- /dev/null
+++ b/adswriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,13 @@
+{
+ "name": "adswriter",
+ "parameter": {
+ "url": "",
+ "username": "",
+ "password": "",
+ "schema": "",
+ "table": "",
+ "partition": "",
+ "overWrite": "",
+ "lifeCycle": 2
+ }
+}
\ No newline at end of file
diff --git a/common/pom.xml b/common/pom.xml
new file mode 100755
index 0000000000..6cce789f23
--- /dev/null
+++ b/common/pom.xml
@@ -0,0 +1,75 @@
+
+ 4.0.0
+
+ com.alibaba.datax
+ datax-all
+ 0.0.1-SNAPSHOT
+
+
+ datax-common
+ datax-common
+ jar
+
+
+
+ org.apache.commons
+ commons-lang3
+
+
+ com.alibaba
+ fastjson
+
+
+ commons-io
+ commons-io
+
+
+
+ junit
+ junit
+ test
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.4
+ test
+
+
+ org.apache.httpcomponents
+ fluent-hc
+ 4.4
+ test
+
+
+ org.apache.commons
+ commons-math3
+ 3.1.1
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ 1.6
+ 1.6
+ ${project-sourceEncoding}
+
+
+
+
+
diff --git a/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java b/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java
new file mode 100755
index 0000000000..e7d06a9503
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java
@@ -0,0 +1,25 @@
+package com.alibaba.datax.common.base;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+public class BaseObject {
+
+ @Override
+ public int hashCode() {
+ return HashCodeBuilder.reflectionHashCode(this, false);
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ return EqualsBuilder.reflectionEquals(this, object, false);
+ }
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this,
+ ToStringStyle.MULTI_LINE_STYLE);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java b/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java
new file mode 100755
index 0000000000..423e16f926
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java
@@ -0,0 +1,9 @@
+package com.alibaba.datax.common.constant;
+
+public final class CommonConstant {
+ /**
+ * 用于插件对自身 split 的每个 task 标识其使用的资源，以告知core 对 reader/writer split 之后的 task 进行拼接时需要根据资源标签进行更有意义的 shuffle 操作
+ */
+ public static String LOAD_BALANCE_RESOURCE_MARK = "loadBalanceResourceMark";
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java b/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java
new file mode 100755
index 0000000000..ceee089e9e
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java
@@ -0,0 +1,20 @@
+package com.alibaba.datax.common.constant;
+
+/**
+ * Created by jingxing on 14-8-31.
+ */
+public enum PluginType {
+ //pluginType还代表了资源目录，很难扩展,或者说需要足够必要才扩展。先mark Handler（其实和transformer一样），再讨论
+ READER("reader"), TRANSFORMER("transformer"), WRITER("writer"), HANDLER("handler");
+
+ private String pluginType;
+
+ private PluginType(String pluginType) {
+ this.pluginType = pluginType;
+ }
+
+ @Override
+ public String toString() {
+ return this.pluginType;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java b/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java
new file mode 100755
index 0000000000..7699e152ae
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java
@@ -0,0 +1,115 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class BoolColumn extends Column {
+
+ public BoolColumn(Boolean bool) {
+ super(bool, Column.Type.BOOL, 1);
+ }
+
+ public BoolColumn(final String data) {
+ this(true);
+ this.validate(data);
+ if (null == data) {
+ this.setRawData(null);
+ this.setByteSize(0);
+ } else {
+ this.setRawData(Boolean.valueOf(data));
+ this.setByteSize(1);
+ }
+ return;
+ }
+
+ public BoolColumn() {
+ super(null, Column.Type.BOOL, 1);
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ if (null == super.getRawData()) {
+ return null;
+ }
+
+ return (Boolean) super.getRawData();
+ }
+
+ @Override
+ public Long asLong() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? 1L : 0L;
+ }
+
+ @Override
+ public Double asDouble() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? 1.0d : 0.0d;
+ }
+
+ @Override
+ public String asString() {
+ if (null == super.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? "true" : "false";
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return BigInteger.valueOf(this.asLong());
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return BigDecimal.valueOf(this.asLong());
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool类型不能转为Date .");
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Boolean类型不能转为Bytes .");
+ }
+
+ private void validate(final String data) {
+ if (null == data) {
+ return;
+ }
+
+ if ("true".equalsIgnoreCase(data) || "false".equalsIgnoreCase(data)) {
+ return;
+ }
+
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s]不能转为Bool .", data));
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java b/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java
new file mode 100755
index 0000000000..d3cc599361
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java
@@ -0,0 +1,84 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.ArrayUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class BytesColumn extends Column {
+
+ public BytesColumn() {
+ this(null);
+ }
+
+ public BytesColumn(byte[] bytes) {
+ super(ArrayUtils.clone(bytes), Column.Type.BYTES, null == bytes ? 0
+ : bytes.length);
+ }
+
+ @Override
+ public byte[] asBytes() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return (byte[]) this.getRawData();
+ }
+
+ @Override
+ public String asString() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ try {
+ return ColumnCast.bytes2String(this);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("Bytes[%s]不能转为String .", this.toString()));
+ }
+ }
+
+ @Override
+ public Long asLong() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Long .");
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为BigDecimal .");
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为BigInteger .");
+ }
+
+ @Override
+ public Double asDouble() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Long .");
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Date .");
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Boolean .");
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/Column.java b/common/src/main/java/com/alibaba/datax/common/element/Column.java
new file mode 100755
index 0000000000..ed68e88d6b
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/Column.java
@@ -0,0 +1,75 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.fastjson.JSON;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ *
+ */
+public abstract class Column {
+
+ private Type type;
+
+ private Object rawData;
+
+ private int byteSize;
+
+ public Column(final Object object, final Type type, int byteSize) {
+ this.rawData = object;
+ this.type = type;
+ this.byteSize = byteSize;
+ }
+
+ public Object getRawData() {
+ return this.rawData;
+ }
+
+ public Type getType() {
+ return this.type;
+ }
+
+ public int getByteSize() {
+ return this.byteSize;
+ }
+
+ protected void setType(Type type) {
+ this.type = type;
+ }
+
+ protected void setRawData(Object rawData) {
+ this.rawData = rawData;
+ }
+
+ protected void setByteSize(int byteSize) {
+ this.byteSize = byteSize;
+ }
+
+ public abstract Long asLong();
+
+ public abstract Double asDouble();
+
+ public abstract String asString();
+
+ public abstract Date asDate();
+
+ public abstract byte[] asBytes();
+
+ public abstract Boolean asBoolean();
+
+ public abstract BigDecimal asBigDecimal();
+
+ public abstract BigInteger asBigInteger();
+
+ @Override
+ public String toString() {
+ return JSON.toJSONString(this);
+ }
+
+ public enum Type {
+ BAD, NULL, INT, LONG, DOUBLE, STRING, BOOL, DATE, BYTES
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
new file mode 100755
index 0000000000..89d0a7c627
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
@@ -0,0 +1,199 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
+
+import java.io.UnsupportedEncodingException;
+import java.text.ParseException;
+import java.util.*;
+
+public final class ColumnCast {
+
+ public static void bind(final Configuration configuration) {
+ StringCast.init(configuration);
+ DateCast.init(configuration);
+ BytesCast.init(configuration);
+ }
+
+ public static Date string2Date(final StringColumn column)
+ throws ParseException {
+ return StringCast.asDate(column);
+ }
+
+ public static byte[] string2Bytes(final StringColumn column)
+ throws UnsupportedEncodingException {
+ return StringCast.asBytes(column);
+ }
+
+ public static String date2String(final DateColumn column) {
+ return DateCast.asString(column);
+ }
+
+ public static String bytes2String(final BytesColumn column)
+ throws UnsupportedEncodingException {
+ return BytesCast.asString(column);
+ }
+}
+
+class StringCast {
+ static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+ static String dateFormat = "yyyy-MM-dd";
+
+ static String timeFormat = "HH:mm:ss";
+
+ static List extraFormats = Collections.emptyList();
+
+ static String timeZone = "GMT+8";
+
+ static FastDateFormat dateFormatter;
+
+ static FastDateFormat timeFormatter;
+
+ static FastDateFormat datetimeFormatter;
+
+ static TimeZone timeZoner;
+
+ static String encoding = "UTF-8";
+
+ static void init(final Configuration configuration) {
+ StringCast.datetimeFormat = configuration.getString(
+ "common.column.datetimeFormat", StringCast.datetimeFormat);
+ StringCast.dateFormat = configuration.getString(
+ "common.column.dateFormat", StringCast.dateFormat);
+ StringCast.timeFormat = configuration.getString(
+ "common.column.timeFormat", StringCast.timeFormat);
+ StringCast.extraFormats = configuration.getList(
+ "common.column.extraFormats", Collections.emptyList(), String.class);
+
+ StringCast.timeZone = configuration.getString("common.column.timeZone",
+ StringCast.timeZone);
+ StringCast.timeZoner = TimeZone.getTimeZone(StringCast.timeZone);
+
+ StringCast.datetimeFormatter = FastDateFormat.getInstance(
+ StringCast.datetimeFormat, StringCast.timeZoner);
+ StringCast.dateFormatter = FastDateFormat.getInstance(
+ StringCast.dateFormat, StringCast.timeZoner);
+ StringCast.timeFormatter = FastDateFormat.getInstance(
+ StringCast.timeFormat, StringCast.timeZoner);
+
+ StringCast.encoding = configuration.getString("common.column.encoding",
+ StringCast.encoding);
+ }
+
+ static Date asDate(final StringColumn column) throws ParseException {
+ if (null == column.asString()) {
+ return null;
+ }
+
+ try {
+ return StringCast.datetimeFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ }
+
+ try {
+ return StringCast.dateFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ }
+
+ ParseException e;
+ try {
+ return StringCast.timeFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ e = ignored;
+ }
+
+ for (String format : StringCast.extraFormats) {
+ try{
+ return FastDateFormat.getInstance(format, StringCast.timeZoner).parse(column.asString());
+ } catch (ParseException ignored){
+ e = ignored;
+ }
+ }
+ throw e;
+ }
+
+ static byte[] asBytes(final StringColumn column)
+ throws UnsupportedEncodingException {
+ if (null == column.asString()) {
+ return null;
+ }
+
+ return column.asString().getBytes(StringCast.encoding);
+ }
+}
+
+/**
+ * 后续为了可维护性，可以考虑直接使用 apache 的DateFormatUtils.
+ *
+ * 迟南已经修复了该问题，但是为了维护性，还是直接使用apache的内置函数
+ */
+class DateCast {
+
+ static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+ static String dateFormat = "yyyy-MM-dd";
+
+ static String timeFormat = "HH:mm:ss";
+
+ static String timeZone = "GMT+8";
+
+ static TimeZone timeZoner = TimeZone.getTimeZone(DateCast.timeZone);
+
+ static void init(final Configuration configuration) {
+ DateCast.datetimeFormat = configuration.getString(
+ "common.column.datetimeFormat", datetimeFormat);
+ DateCast.timeFormat = configuration.getString(
+ "common.column.timeFormat", timeFormat);
+ DateCast.dateFormat = configuration.getString(
+ "common.column.dateFormat", dateFormat);
+ DateCast.timeZone = configuration.getString("common.column.timeZone",
+ DateCast.timeZone);
+ DateCast.timeZoner = TimeZone.getTimeZone(DateCast.timeZone);
+ return;
+ }
+
+ static String asString(final DateColumn column) {
+ if (null == column.asDate()) {
+ return null;
+ }
+
+ switch (column.getSubType()) {
+ case DATE:
+ return DateFormatUtils.format(column.asDate(), DateCast.dateFormat,
+ DateCast.timeZoner);
+ case TIME:
+ return DateFormatUtils.format(column.asDate(), DateCast.timeFormat,
+ DateCast.timeZoner);
+ case DATETIME:
+ return DateFormatUtils.format(column.asDate(),
+ DateCast.datetimeFormat, DateCast.timeZoner);
+ default:
+ throw DataXException
+ .asDataXException(CommonErrorCode.CONVERT_NOT_SUPPORT,
+ "时间类型出现不支持类型，目前仅支持DATE/TIME/DATETIME。该类型属于编程错误，请反馈给DataX开发团队 .");
+ }
+ }
+}
+
+class BytesCast {
+ static String encoding = "utf-8";
+
+ static void init(final Configuration configuration) {
+ BytesCast.encoding = configuration.getString("common.column.encoding",
+ BytesCast.encoding);
+ return;
+ }
+
+ static String asString(final BytesColumn column)
+ throws UnsupportedEncodingException {
+ if (null == column.asBytes()) {
+ return null;
+ }
+
+ return new String(column.asBytes(), encoding);
+ }
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java
new file mode 100755
index 0000000000..6626a6fbdd
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java
@@ -0,0 +1,130 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class DateColumn extends Column {
+
+ private DateType subType = DateType.DATETIME;
+
+ public static enum DateType {
+ DATE, TIME, DATETIME
+ }
+
+ /**
+ * 构建值为null的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn() {
+ this((Long)null);
+ }
+
+ /**
+ * 构建值为stamp(Unix时间戳)的DateColumn，使用Date子类型为DATETIME
+ * 实际存储有date改为long的ms，节省存储
+ * */
+ public DateColumn(final Long stamp) {
+ super(stamp, Column.Type.DATE, (null == stamp ? 0 : 8));
+ }
+
+ /**
+ * 构建值为date(java.util.Date)的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn(final Date date) {
+ this(date == null ? null : date.getTime());
+ }
+
+ /**
+ * 构建值为date(java.sql.Date)的DateColumn，使用Date子类型为DATE，只有日期，没有时间
+ * */
+ public DateColumn(final java.sql.Date date) {
+ this(date == null ? null : date.getTime());
+ this.setSubType(DateType.DATE);
+ }
+
+ /**
+ * 构建值为time(java.sql.Time)的DateColumn，使用Date子类型为TIME，只有时间，没有日期
+ * */
+ public DateColumn(final java.sql.Time time) {
+ this(time == null ? null : time.getTime());
+ this.setSubType(DateType.TIME);
+ }
+
+ /**
+ * 构建值为ts(java.sql.Timestamp)的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn(final java.sql.Timestamp ts) {
+ this(ts == null ? null : ts.getTime());
+ this.setSubType(DateType.DATETIME);
+ }
+
+ @Override
+ public Long asLong() {
+
+ return (Long)this.getRawData();
+ }
+
+ @Override
+ public String asString() {
+ try {
+ return ColumnCast.date2String(this);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("Date[%s]类型不能转为String .", this.toString()));
+ }
+ }
+
+ @Override
+ public Date asDate() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return new Date((Long)this.getRawData());
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Bytes .");
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Boolean .");
+ }
+
+ @Override
+ public Double asDouble() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Double .");
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为BigInteger .");
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为BigDecimal .");
+ }
+
+ public DateType getSubType() {
+ return subType;
+ }
+
+ public void setSubType(DateType subType) {
+ this.subType = subType;
+ }
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java
new file mode 100755
index 0000000000..17170ea6c4
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java
@@ -0,0 +1,161 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+public class DoubleColumn extends Column {
+
+ public DoubleColumn(final String data) {
+ this(data, null == data ? 0 : data.length());
+ this.validate(data);
+ }
+
+ public DoubleColumn(Long data) {
+ this(data == null ? (String) null : String.valueOf(data));
+ }
+
+ public DoubleColumn(Integer data) {
+ this(data == null ? (String) null : String.valueOf(data));
+ }
+
+ /**
+ * Double无法表示准确的小数数据，我们不推荐使用该方法保存Double数据，建议使用String作为构造入参
+ *
+ * */
+ public DoubleColumn(final Double data) {
+ this(data == null ? (String) null
+ : new BigDecimal(String.valueOf(data)).toPlainString());
+ }
+
+ /**
+ * Float无法表示准确的小数数据，我们不推荐使用该方法保存Float数据，建议使用String作为构造入参
+ *
+ * */
+ public DoubleColumn(final Float data) {
+ this(data == null ? (String) null
+ : new BigDecimal(String.valueOf(data)).toPlainString());
+ }
+
+ public DoubleColumn(final BigDecimal data) {
+ this(null == data ? (String) null : data.toPlainString());
+ }
+
+ public DoubleColumn(final BigInteger data) {
+ this(null == data ? (String) null : data.toString());
+ }
+
+ public DoubleColumn() {
+ this((String) null);
+ }
+
+ private DoubleColumn(final String data, int byteSize) {
+ super(data, Column.Type.DOUBLE, byteSize);
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ try {
+ return new BigDecimal((String) this.getRawData());
+ } catch (NumberFormatException e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s] 无法转换为Double类型 .",
+ (String) this.getRawData()));
+ }
+ }
+
+ @Override
+ public Double asDouble() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ String string = (String) this.getRawData();
+
+ boolean isDoubleSpecific = string.equals("NaN")
+ || string.equals("-Infinity") || string.equals("+Infinity");
+ if (isDoubleSpecific) {
+ return Double.valueOf(string);
+ }
+
+ BigDecimal result = this.asBigDecimal();
+ OverFlowUtil.validateDoubleNotOverFlow(result);
+
+ return result.doubleValue();
+ }
+
+ @Override
+ public Long asLong() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ BigDecimal result = this.asBigDecimal();
+ OverFlowUtil.validateLongNotOverFlow(result.toBigInteger());
+
+ return result.longValue();
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBigDecimal().toBigInteger();
+ }
+
+ @Override
+ public String asString() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+ return (String) this.getRawData();
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Bool .");
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Date类型 .");
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Bytes类型 .");
+ }
+
+ private void validate(final String data) {
+ if (null == data) {
+ return;
+ }
+
+ if (data.equalsIgnoreCase("NaN") || data.equalsIgnoreCase("-Infinity")
+ || data.equalsIgnoreCase("Infinity")) {
+ return;
+ }
+
+ try {
+ new BigDecimal(data);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s]无法转为Double类型 .", data));
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java b/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java
new file mode 100755
index 0000000000..d8113f7c05
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java
@@ -0,0 +1,135 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+public class LongColumn extends Column {
+
+ /**
+ * 从整形字符串表示转为LongColumn，支持Java科学计数法
+ *
+ * NOTE: > getMessage();
+
+ /**
+ * 从Task获取自定义收集信息
+ *
+ * */
+ List getMessage(String key);
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java b/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java
new file mode 100755
index 0000000000..f2af398dd3
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java
@@ -0,0 +1,9 @@
+package com.alibaba.datax.common.plugin;
+
+
+/**
+ * 这里只是一个标示类
+ * */
+public interface PluginCollector {
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java b/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java
new file mode 100755
index 0000000000..ac28f6a294
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java
@@ -0,0 +1,30 @@
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.util.Configuration;
+
+public interface Pluginable {
+ String getDeveloper();
+
+ String getDescription();
+
+ void setPluginConf(Configuration pluginConf);
+
+ void init();
+
+ void destroy();
+
+ String getPluginName();
+
+ Configuration getPluginJobConf();
+
+ Configuration getPeerPluginJobConf();
+
+ public String getPeerPluginName();
+
+ void setPluginJobConf(Configuration jobConf);
+
+ void setPeerPluginJobConf(Configuration peerPluginJobConf);
+
+ public void setPeerPluginName(String peerPluginName);
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java b/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java
new file mode 100755
index 0000000000..74f236f371
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java
@@ -0,0 +1,26 @@
+/**
+ * (C) 2010-2013 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+public interface RecordReceiver {
+
+ public Record getFromReader();
+
+ public void shutdown();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java b/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java
new file mode 100755
index 0000000000..0d6926098f
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java
@@ -0,0 +1,32 @@
+/**
+ * (C) 2010-2013 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+public interface RecordSender {
+
+ public Record createRecord();
+
+ public void sendToWriter(Record record);
+
+ public void flush();
+
+ public void terminate();
+
+ public void shutdown();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java b/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java
new file mode 100755
index 0000000000..f0c85fe6ce
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java
@@ -0,0 +1,57 @@
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+/**
+ *
+ * 该接口提供给Task Plugin用来记录脏数据和自定义信息。
+ *
+ * @Override
+ * public String toString() {
+ * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
+ * }
+ *
+ *
+ */
+public interface ErrorCode {
+ // 错误码编号
+ String getCode();
+
+ // 错误码描述
+ String getDescription();
+
+ /** 必须提供toString的实现
+ *
+ *
+ * @Override
+ * public String toString() {
+ * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
+ * }
+ *
+ *
+ */
+ String toString();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Hook.java b/common/src/main/java/com/alibaba/datax/common/spi/Hook.java
new file mode 100755
index 0000000000..d510f57c18
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Hook.java
@@ -0,0 +1,27 @@
+package com.alibaba.datax.common.spi;
+
+import com.alibaba.datax.common.util.Configuration;
+
+import java.util.Map;
+
+/**
+ * Created by xiafei.qiuxf on 14/12/17.
+ */
+public interface Hook {
+
+ /**
+ * 返回名字
+ *
+ * @return
+ */
+ public String getName();
+
+ /**
+ * TODO 文档
+ *
+ * @param jobConf
+ * @param msg
+ */
+ public void invoke(Configuration jobConf, Map msg);
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Reader.java b/common/src/main/java/com/alibaba/datax/common/spi/Reader.java
new file mode 100755
index 0000000000..fec41a9f03
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Reader.java
@@ -0,0 +1,52 @@
+package com.alibaba.datax.common.spi;
+
+import java.util.List;
+
+import com.alibaba.datax.common.base.BaseObject;
+import com.alibaba.datax.common.plugin.AbstractJobPlugin;
+import com.alibaba.datax.common.plugin.AbstractTaskPlugin;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.plugin.RecordSender;
+
+/**
+ * 每个Reader插件在其内部内部实现Job、Task两个内部类。
+ *
+ *
+ * */
+public abstract class Reader extends BaseObject {
+
+ /**
+ * 每个Reader插件必须实现Job内部类。
+ *
+ * */
+ public static abstract class Job extends AbstractJobPlugin {
+
+ /**
+ * 切分任务
+ *
+ * @param adviceNumber
+ *
+ * 着重说明下，adviceNumber是框架建议插件切分的任务数，插件开发人员最好切分出来的任务数>=
+ * adviceNumber。 split(int adviceNumber);
+ }
+
+ public static abstract class Task extends AbstractTaskPlugin {
+ public abstract void startRead(RecordSender recordSender);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Writer.java b/common/src/main/java/com/alibaba/datax/common/spi/Writer.java
new file mode 100755
index 0000000000..457eb6860c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Writer.java
@@ -0,0 +1,40 @@
+package com.alibaba.datax.common.spi;
+
+import com.alibaba.datax.common.base.BaseObject;
+import com.alibaba.datax.common.plugin.AbstractJobPlugin;
+import com.alibaba.datax.common.plugin.AbstractTaskPlugin;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+
+import java.util.List;
+
+/**
+ * 每个Writer插件需要实现Writer类，并在其内部实现Job、Task两个内部类。
+ *
+ *
+ * */
+public abstract class Writer extends BaseObject {
+ /**
+ * 每个Writer插件必须实现Job内部类
+ */
+ public abstract static class Job extends AbstractJobPlugin {
+ /**
+ * 切分任务。 split(int mandatoryNumber);
+ }
+
+ /**
+ * 每个Writer插件必须实现Task内部类
+ */
+ public abstract static class Task extends AbstractTaskPlugin {
+
+ public abstract void startWrite(RecordReceiver lineReceiver);
+
+ public boolean supportFailOver(){return false;}
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java b/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java
new file mode 100644
index 0000000000..74b26eeb60
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java
@@ -0,0 +1,258 @@
+package com.alibaba.datax.common.statistics;
+
+import com.alibaba.datax.common.util.HostUtils;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+
+/**
+ * Created by liqiang on 15/8/23.
+ */
+@SuppressWarnings("NullableProblems")
+public class PerfRecord implements Comparable {
+ private static Logger perf = LoggerFactory.getLogger(PerfRecord.class);
+ private static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+
+ public enum PHASE {
+ /**
+ * task total运行的时间，前10为框架统计，后面为部分插件的个性统计
+ */
+ TASK_TOTAL(0),
+
+ READ_TASK_INIT(1),
+ READ_TASK_PREPARE(2),
+ READ_TASK_DATA(3),
+ READ_TASK_POST(4),
+ READ_TASK_DESTROY(5),
+
+ WRITE_TASK_INIT(6),
+ WRITE_TASK_PREPARE(7),
+ WRITE_TASK_DATA(8),
+ WRITE_TASK_POST(9),
+ WRITE_TASK_DESTROY(10),
+
+ /**
+ * SQL_QUERY: sql query阶段, 部分reader的个性统计
+ */
+ SQL_QUERY(100),
+ /**
+ * 数据从sql全部读出来
+ */
+ RESULT_NEXT_ALL(101),
+
+ /**
+ * only odps block close
+ */
+ ODPS_BLOCK_CLOSE(102),
+
+ WAIT_READ_TIME(103),
+
+ WAIT_WRITE_TIME(104),
+
+ TRANSFORMER_TIME(201);
+
+ private int val;
+
+ PHASE(int val) {
+ this.val = val;
+ }
+
+ public int toInt(){
+ return val;
+ }
+ }
+
+ public enum ACTION{
+ start,
+ end
+ }
+
+ private final int taskGroupId;
+ private final int taskId;
+ private final PHASE phase;
+ private volatile ACTION action;
+ private volatile Date startTime;
+ private volatile long elapsedTimeInNs = -1;
+ private volatile long count = 0;
+ private volatile long size = 0;
+
+ private volatile long startTimeInNs;
+ private volatile boolean isReport = false;
+
+ public PerfRecord(int taskGroupId, int taskId, PHASE phase) {
+ this.taskGroupId = taskGroupId;
+ this.taskId = taskId;
+ this.phase = phase;
+ }
+
+ public static void addPerfRecord(int taskGroupId, int taskId, PHASE phase, long startTime,long elapsedTimeInNs) {
+ if(PerfTrace.getInstance().isEnable()) {
+ PerfRecord perfRecord = new PerfRecord(taskGroupId, taskId, phase);
+ perfRecord.elapsedTimeInNs = elapsedTimeInNs;
+ perfRecord.action = ACTION.end;
+ perfRecord.startTime = new Date(startTime);
+ //在PerfTrace里注册
+ PerfTrace.getInstance().tracePerfRecord(perfRecord);
+ perf.info(perfRecord.toString());
+ }
+ }
+
+ public void start() {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.startTime = new Date();
+ this.startTimeInNs = System.nanoTime();
+ this.action = ACTION.start;
+ //在PerfTrace里注册
+ PerfTrace.getInstance().tracePerfRecord(this);
+ perf.info(toString());
+ }
+ }
+
+ public void addCount(long count) {
+ this.count += count;
+ }
+
+ public void addSize(long size) {
+ this.size += size;
+ }
+
+ public void end() {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.elapsedTimeInNs = System.nanoTime() - startTimeInNs;
+ this.action = ACTION.end;
+ PerfTrace.getInstance().tracePerfRecord(this);
+ perf.info(toString());
+ }
+ }
+
+ public void end(long elapsedTimeInNs) {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.elapsedTimeInNs = elapsedTimeInNs;
+ this.action = ACTION.end;
+ PerfTrace.getInstance().tracePerfRecord(this);
+ perf.info(toString());
+ }
+ }
+
+ public String toString() {
+ return String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
+ , getInstId(), taskGroupId, taskId, phase, action,
+ DateFormatUtils.format(startTime, datetimeFormat), elapsedTimeInNs, count, size,getHostIP());
+ }
+
+
+ @Override
+ public int compareTo(PerfRecord o) {
+ if (o == null) {
+ return 1;
+ }
+ return this.elapsedTimeInNs > o.elapsedTimeInNs ? 1 : this.elapsedTimeInNs == o.elapsedTimeInNs ? 0 : -1;
+ }
+
+ @Override
+ public int hashCode() {
+ long jobId = getInstId();
+ int result = (int) (jobId ^ (jobId >>> 32));
+ result = 31 * result + taskGroupId;
+ result = 31 * result + taskId;
+ result = 31 * result + phase.toInt();
+ result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if(!(o instanceof PerfRecord)){
+ return false;
+ }
+
+ PerfRecord dst = (PerfRecord)o;
+
+ if (this.getInstId() != dst.getInstId()) return false;
+ if (this.taskGroupId != dst.taskGroupId) return false;
+ if (this.taskId != dst.taskId) return false;
+ if (phase != null ? !phase.equals(dst.phase) : dst.phase != null) return false;
+ if (startTime != null ? !startTime.equals(dst.startTime) : dst.startTime != null) return false;
+ return true;
+ }
+
+ public PerfRecord copy() {
+ PerfRecord copy = new PerfRecord(this.taskGroupId, this.getTaskId(), this.phase);
+ copy.action = this.action;
+ copy.startTime = this.startTime;
+ copy.elapsedTimeInNs = this.elapsedTimeInNs;
+ copy.count = this.count;
+ copy.size = this.size;
+ return copy;
+ }
+ public int getTaskGroupId() {
+ return taskGroupId;
+ }
+
+ public int getTaskId() {
+ return taskId;
+ }
+
+ public PHASE getPhase() {
+ return phase;
+ }
+
+ public ACTION getAction() {
+ return action;
+ }
+
+ public long getElapsedTimeInNs() {
+ return elapsedTimeInNs;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public long getInstId(){
+ return PerfTrace.getInstance().getInstId();
+ }
+
+ public String getHostIP(){
+ return HostUtils.IP;
+ }
+
+ public String getHostName(){
+ return HostUtils.HOSTNAME;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public long getStartTimeInMs() {
+ return startTime.getTime();
+ }
+
+ public long getStartTimeInNs() {
+ return startTimeInNs;
+ }
+
+ public String getDatetime(){
+ if(startTime == null){
+ return "null time";
+ }
+ return DateFormatUtils.format(startTime, datetimeFormat);
+ }
+
+ public boolean isReport() {
+ return isReport;
+ }
+
+ public void setIsReport(boolean isReport) {
+ this.isReport = isReport;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java b/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java
new file mode 100644
index 0000000000..ea9aa42110
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java
@@ -0,0 +1,907 @@
+package com.alibaba.datax.common.statistics;
+
+import com.alibaba.datax.common.statistics.PerfRecord.PHASE;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.HostUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * PerfTrace 记录 job（local模式），taskGroup（distribute模式），因为这2种都是jvm，即一个jvm里只需要有1个PerfTrace。
+ */
+
+public class PerfTrace {
+
+ private static Logger LOG = LoggerFactory.getLogger(PerfTrace.class);
+ private static PerfTrace instance;
+ private static final Object lock = new Object();
+ private String perfTraceId;
+ private volatile boolean enable;
+ private volatile boolean isJob;
+ private long instId;
+ private long jobId;
+ private long jobVersion;
+ private int taskGroupId;
+ private int channelNumber;
+
+ private int priority;
+ private int batchSize = 500;
+ private volatile boolean perfReportEnable = true;
+
+ //jobid_jobversion,instanceid,taskid, src_mark, dst_mark,
+ private Map taskDetails = new ConcurrentHashMap();
+ //PHASE => PerfRecord
+ private ConcurrentHashMap perfRecordMaps4print = new ConcurrentHashMap();
+ // job_phase => SumPerf4Report
+ private SumPerf4Report sumPerf4Report = new SumPerf4Report();
+ private SumPerf4Report sumPerf4Report4NotEnd;
+ private Configuration jobInfo;
+ private final Set needReportPool4NotEnd = new HashSet();
+ private final List totalEndReport = new ArrayList();
+
+ /**
+ * 单实例
+ *
+ * @param isJob
+ * @param jobId
+ * @param taskGroupId
+ * @return
+ */
+ public static PerfTrace getInstance(boolean isJob, long jobId, int taskGroupId, int priority, boolean enable) {
+
+ if (instance == null) {
+ synchronized (lock) {
+ if (instance == null) {
+ instance = new PerfTrace(isJob, jobId, taskGroupId, priority, enable);
+ }
+ }
+ }
+ return instance;
+ }
+
+ /**
+ * 因为一个JVM只有一个，因此在getInstance(isJob,jobId,taskGroupId)调用完成实例化后，方便后续调用，直接返回该实例
+ *
+ * @return
+ */
+ public static PerfTrace getInstance() {
+ if (instance == null) {
+ LOG.error("PerfTrace instance not be init! must have some error! ");
+ synchronized (lock) {
+ if (instance == null) {
+ instance = new PerfTrace(false, -1111, -1111, 0, false);
+ }
+ }
+ }
+ return instance;
+ }
+
+ private PerfTrace(boolean isJob, long jobId, int taskGroupId, int priority, boolean enable) {
+ try {
+ this.perfTraceId = isJob ? "job_" + jobId : String.format("taskGroup_%s_%s", jobId, taskGroupId);
+ this.enable = enable;
+ this.isJob = isJob;
+ this.taskGroupId = taskGroupId;
+ this.instId = jobId;
+ this.priority = priority;
+ LOG.info(String.format("PerfTrace traceId=%s, isEnable=%s, priority=%s", this.perfTraceId, this.enable, this.priority));
+
+ } catch (Exception e) {
+ // do nothing
+ this.enable = false;
+ }
+ }
+
+ public void addTaskDetails(int taskId, String detail) {
+ if (enable) {
+ String before = "";
+ int index = detail.indexOf("?");
+ String current = detail.substring(0, index == -1 ? detail.length() : index);
+ if (current.indexOf("[") >= 0) {
+ current += "]";
+ }
+ if (taskDetails.containsKey(taskId)) {
+ before = taskDetails.get(taskId).trim();
+ }
+ if (StringUtils.isEmpty(before)) {
+ before = "";
+ } else {
+ before += ",";
+ }
+ this.taskDetails.put(taskId, before + current);
+ }
+ }
+
+ public void tracePerfRecord(PerfRecord perfRecord) {
+ try {
+ if (enable) {
+ long curNanoTime = System.nanoTime();
+ //ArrayList非线程安全
+ switch (perfRecord.getAction()) {
+ case end:
+ synchronized (totalEndReport) {
+ totalEndReport.add(perfRecord);
+
+ if (totalEndReport.size() > batchSize * 10) {
+ sumPerf4EndPrint(totalEndReport);
+ }
+ }
+
+ if (perfReportEnable && needReport(perfRecord)) {
+ synchronized (needReportPool4NotEnd) {
+ sumPerf4Report.add(curNanoTime,perfRecord);
+ needReportPool4NotEnd.remove(perfRecord);
+ }
+ }
+
+ break;
+ case start:
+ if (perfReportEnable && needReport(perfRecord)) {
+ synchronized (needReportPool4NotEnd) {
+ needReportPool4NotEnd.add(perfRecord);
+ }
+ }
+ break;
+ }
+ }
+ } catch (Exception e) {
+ // do nothing
+ }
+ }
+
+ private boolean needReport(PerfRecord perfRecord) {
+ switch (perfRecord.getPhase()) {
+ case TASK_TOTAL:
+ case SQL_QUERY:
+ case RESULT_NEXT_ALL:
+ case ODPS_BLOCK_CLOSE:
+ return true;
+ }
+ return false;
+ }
+
+ public String summarizeNoException() {
+ String res;
+ try {
+ res = summarize();
+ } catch (Exception e) {
+ res = "PerfTrace summarize has Exception " + e.getMessage();
+ }
+ return res;
+ }
+
+ //任务结束时，对当前的perf总汇总统计
+ private synchronized String summarize() {
+ if (!enable) {
+ return "PerfTrace not enable!";
+ }
+
+ if (totalEndReport.size() > 0) {
+ sumPerf4EndPrint(totalEndReport);
+ }
+
+ StringBuilder info = new StringBuilder();
+ info.append("\n === total summarize info === \n");
+ info.append("\n 1. all phase average time info and max time task info: \n\n");
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %-100s\n", "PHASE", "AVERAGE USED TIME", "ALL TASK NUM", "MAX USED TIME", "MAX TASK ID", "MAX TASK INFO"));
+
+ List keys = new ArrayList(perfRecordMaps4print.keySet());
+ Collections.sort(keys, new Comparator() {
+ @Override
+ public int compare(PHASE o1, PHASE o2) {
+ return o1.toInt() - o2.toInt();
+ }
+ });
+ for (PHASE phase : keys) {
+ SumPerfRecord4Print sumPerfRecord = perfRecordMaps4print.get(phase);
+ if (sumPerfRecord == null) {
+ continue;
+ }
+ long averageTime = sumPerfRecord.getAverageTime();
+ long maxTime = sumPerfRecord.getMaxTime();
+ int maxTaskId = sumPerfRecord.maxTaskId;
+ int maxTaskGroupId = sumPerfRecord.getMaxTaskGroupId();
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %-100s\n",
+ phase, unitTime(averageTime), sumPerfRecord.totalCount, unitTime(maxTime), jobId + "-" + maxTaskGroupId + "-" + maxTaskId, taskDetails.get(maxTaskId)));
+ }
+
+ //SumPerfRecord4Print countSumPerf = Optional.fromNullable(perfRecordMaps4print.get(PHASE.READ_TASK_DATA)).or(new SumPerfRecord4Print());
+
+ SumPerfRecord4Print countSumPerf = perfRecordMaps4print.get(PHASE.READ_TASK_DATA);
+ if(countSumPerf == null){
+ countSumPerf = new SumPerfRecord4Print();
+ }
+
+ long averageRecords = countSumPerf.getAverageRecords();
+ long averageBytes = countSumPerf.getAverageBytes();
+ long maxRecord = countSumPerf.getMaxRecord();
+ long maxByte = countSumPerf.getMaxByte();
+ int maxTaskId4Records = countSumPerf.getMaxTaskId4Records();
+ int maxTGID4Records = countSumPerf.getMaxTGID4Records();
+
+ info.append("\n\n 2. record average count and max count task info :\n\n");
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %18s | %-100s\n", "PHASE", "AVERAGE RECORDS", "AVERAGE BYTES", "MAX RECORDS", "MAX RECORD`S BYTES", "MAX TASK ID", "MAX TASK INFO"));
+ if (maxTaskId4Records > -1) {
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %18s | %-100s\n"
+ , PHASE.READ_TASK_DATA, averageRecords, unitSize(averageBytes), maxRecord, unitSize(maxByte), jobId + "-" + maxTGID4Records + "-" + maxTaskId4Records, taskDetails.get(maxTaskId4Records)));
+
+ }
+ return info.toString();
+ }
+
+ //缺省传入的时间是nano
+ public static String unitTime(long time) {
+ return unitTime(time, TimeUnit.NANOSECONDS);
+ }
+
+ public static String unitTime(long time, TimeUnit timeUnit) {
+ return String.format("%,.3fs", ((float) timeUnit.toNanos(time)) / 1000000000);
+ }
+
+ public static String unitSize(long size) {
+ if (size > 1000000000) {
+ return String.format("%,.2fG", (float) size / 1000000000);
+ } else if (size > 1000000) {
+ return String.format("%,.2fM", (float) size / 1000000);
+ } else if (size > 1000) {
+ return String.format("%,.2fK", (float) size / 1000);
+ } else {
+ return size + "B";
+ }
+ }
+
+
+ public synchronized ConcurrentHashMap getPerfRecordMaps4print() {
+ if (totalEndReport.size() > 0) {
+ sumPerf4EndPrint(totalEndReport);
+ }
+ return perfRecordMaps4print;
+ }
+
+ public SumPerf4Report getSumPerf4Report() {
+ return sumPerf4Report;
+ }
+
+ public Set getNeedReportPool4NotEnd() {
+ return needReportPool4NotEnd;
+ }
+
+ public List getTotalEndReport() {
+ return totalEndReport;
+ }
+
+ public Map getTaskDetails() {
+ return taskDetails;
+ }
+
+ public boolean isEnable() {
+ return enable;
+ }
+
+ public boolean isJob() {
+ return isJob;
+ }
+
+ private String cluster;
+ private String jobDomain;
+ private String srcType;
+ private String dstType;
+ private String srcGuid;
+ private String dstGuid;
+ private Date windowStart;
+ private Date windowEnd;
+ private Date jobStartTime;
+
+ public void setJobInfo(Configuration jobInfo, boolean perfReportEnable, int channelNumber) {
+ try {
+ this.jobInfo = jobInfo;
+ if (jobInfo != null && perfReportEnable) {
+
+ cluster = jobInfo.getString("cluster");
+
+ String srcDomain = jobInfo.getString("srcDomain", "null");
+ String dstDomain = jobInfo.getString("dstDomain", "null");
+ jobDomain = srcDomain + "|" + dstDomain;
+ srcType = jobInfo.getString("srcType");
+ dstType = jobInfo.getString("dstType");
+ srcGuid = jobInfo.getString("srcGuid");
+ dstGuid = jobInfo.getString("dstGuid");
+ windowStart = getWindow(jobInfo.getString("windowStart"), true);
+ windowEnd = getWindow(jobInfo.getString("windowEnd"), false);
+ String jobIdStr = jobInfo.getString("jobId");
+ jobId = StringUtils.isEmpty(jobIdStr) ? (long) -5 : Long.parseLong(jobIdStr);
+ String jobVersionStr = jobInfo.getString("jobVersion");
+ jobVersion = StringUtils.isEmpty(jobVersionStr) ? (long) -4 : Long.parseLong(jobVersionStr);
+ jobStartTime = new Date();
+ }
+ this.perfReportEnable = perfReportEnable;
+ this.channelNumber = channelNumber;
+ } catch (Exception e) {
+ this.perfReportEnable = false;
+ }
+ }
+
+ private Date getWindow(String windowStr, boolean startWindow) {
+ SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
+ if (StringUtils.isNotEmpty(windowStr)) {
+ try {
+ return sdf1.parse(windowStr);
+ } catch (ParseException e) {
+ // do nothing
+ }
+ }
+
+ if (startWindow) {
+ try {
+ return sdf2.parse(sdf2.format(new Date()));
+ } catch (ParseException e1) {
+ //do nothing
+ }
+ }
+
+ return null;
+ }
+
+ public long getInstId() {
+ return instId;
+ }
+
+ public Configuration getJobInfo() {
+ return jobInfo;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public synchronized JobStatisticsDto2 getReports(String mode) {
+
+ try {
+ if (!enable || !perfReportEnable) {
+ return null;
+ }
+
+ if (("job".equalsIgnoreCase(mode) && !isJob) || "tg".equalsIgnoreCase(mode) && isJob) {
+ return null;
+ }
+
+ //每次将未完成的task的统计清空
+ sumPerf4Report4NotEnd = new SumPerf4Report();
+ Set needReportPool4NotEndTmp = null;
+ synchronized (needReportPool4NotEnd) {
+ needReportPool4NotEndTmp = new HashSet(needReportPool4NotEnd);
+ }
+
+ long curNanoTime = System.nanoTime();
+ for (PerfRecord perfRecord : needReportPool4NotEndTmp) {
+ sumPerf4Report4NotEnd.add(curNanoTime, perfRecord);
+ }
+
+ JobStatisticsDto2 jdo = new JobStatisticsDto2();
+ jdo.setInstId(this.instId);
+ if (isJob) {
+ jdo.setTaskGroupId(-6);
+ } else {
+ jdo.setTaskGroupId(this.taskGroupId);
+ }
+ jdo.setJobId(this.jobId);
+ jdo.setJobVersion(this.jobVersion);
+ jdo.setWindowStart(this.windowStart);
+ jdo.setWindowEnd(this.windowEnd);
+ jdo.setJobStartTime(jobStartTime);
+ jdo.setJobRunTimeMs(System.currentTimeMillis() - jobStartTime.getTime());
+ jdo.setJobPriority(this.priority);
+ jdo.setChannelNum(this.channelNumber);
+ jdo.setCluster(this.cluster);
+ jdo.setJobDomain(this.jobDomain);
+ jdo.setSrcType(this.srcType);
+ jdo.setDstType(this.dstType);
+ jdo.setSrcGuid(this.srcGuid);
+ jdo.setDstGuid(this.dstGuid);
+ jdo.setHostAddress(HostUtils.IP);
+
+ //sum
+ jdo.setTaskTotalTimeMs(sumPerf4Report4NotEnd.totalTaskRunTimeInMs + sumPerf4Report.totalTaskRunTimeInMs);
+ jdo.setOdpsBlockCloseTimeMs(sumPerf4Report4NotEnd.odpsCloseTimeInMs + sumPerf4Report.odpsCloseTimeInMs);
+ jdo.setSqlQueryTimeMs(sumPerf4Report4NotEnd.sqlQueryTimeInMs + sumPerf4Report.sqlQueryTimeInMs);
+ jdo.setResultNextTimeMs(sumPerf4Report4NotEnd.resultNextTimeInMs + sumPerf4Report.resultNextTimeInMs);
+
+ return jdo;
+ } catch (Exception e) {
+ // do nothing
+ }
+
+ return null;
+ }
+
+ private void sumPerf4EndPrint(List totalEndReport) {
+ if (!enable || totalEndReport == null) {
+ return;
+ }
+
+ for (PerfRecord perfRecord : totalEndReport) {
+ perfRecordMaps4print.putIfAbsent(perfRecord.getPhase(), new SumPerfRecord4Print());
+ perfRecordMaps4print.get(perfRecord.getPhase()).add(perfRecord);
+ }
+
+ totalEndReport.clear();
+ }
+
+ public void setChannelNumber(int needChannelNumber) {
+ this.channelNumber = needChannelNumber;
+ }
+
+
+ public static class SumPerf4Report {
+ long totalTaskRunTimeInMs = 0L;
+ long odpsCloseTimeInMs = 0L;
+ long sqlQueryTimeInMs = 0L;
+ long resultNextTimeInMs = 0L;
+
+ public void add(long curNanoTime,PerfRecord perfRecord) {
+ try {
+ long runTimeEndInMs;
+ if (perfRecord.getElapsedTimeInNs() == -1) {
+ runTimeEndInMs = (curNanoTime - perfRecord.getStartTimeInNs()) / 1000000;
+ } else {
+ runTimeEndInMs = perfRecord.getElapsedTimeInNs() / 1000000;
+ }
+ switch (perfRecord.getPhase()) {
+ case TASK_TOTAL:
+ totalTaskRunTimeInMs += runTimeEndInMs;
+ break;
+ case SQL_QUERY:
+ sqlQueryTimeInMs += runTimeEndInMs;
+ break;
+ case RESULT_NEXT_ALL:
+ resultNextTimeInMs += runTimeEndInMs;
+ break;
+ case ODPS_BLOCK_CLOSE:
+ odpsCloseTimeInMs += runTimeEndInMs;
+ break;
+ }
+ }catch (Exception e){
+ //do nothing
+ }
+ }
+
+ public long getTotalTaskRunTimeInMs() {
+ return totalTaskRunTimeInMs;
+ }
+
+ public long getOdpsCloseTimeInMs() {
+ return odpsCloseTimeInMs;
+ }
+
+ public long getSqlQueryTimeInMs() {
+ return sqlQueryTimeInMs;
+ }
+
+ public long getResultNextTimeInMs() {
+ return resultNextTimeInMs;
+ }
+ }
+
+ public static class SumPerfRecord4Print {
+ private long perfTimeTotal = 0;
+ private long averageTime = 0;
+ private long maxTime = 0;
+ private int maxTaskId = -1;
+ private int maxTaskGroupId = -1;
+ private int totalCount = 0;
+
+ private long recordsTotal = 0;
+ private long sizesTotal = 0;
+ private long averageRecords = 0;
+ private long averageBytes = 0;
+ private long maxRecord = 0;
+ private long maxByte = 0;
+ private int maxTaskId4Records = -1;
+ private int maxTGID4Records = -1;
+
+ public void add(PerfRecord perfRecord) {
+ if (perfRecord == null) {
+ return;
+ }
+ perfTimeTotal += perfRecord.getElapsedTimeInNs();
+ if (perfRecord.getElapsedTimeInNs() >= maxTime) {
+ maxTime = perfRecord.getElapsedTimeInNs();
+ maxTaskId = perfRecord.getTaskId();
+ maxTaskGroupId = perfRecord.getTaskGroupId();
+ }
+
+ recordsTotal += perfRecord.getCount();
+ sizesTotal += perfRecord.getSize();
+ if (perfRecord.getCount() >= maxRecord) {
+ maxRecord = perfRecord.getCount();
+ maxByte = perfRecord.getSize();
+ maxTaskId4Records = perfRecord.getTaskId();
+ maxTGID4Records = perfRecord.getTaskGroupId();
+ }
+
+ totalCount++;
+ }
+
+ public long getPerfTimeTotal() {
+ return perfTimeTotal;
+ }
+
+ public long getAverageTime() {
+ if (totalCount > 0) {
+ averageTime = perfTimeTotal / totalCount;
+ }
+ return averageTime;
+ }
+
+ public long getMaxTime() {
+ return maxTime;
+ }
+
+ public int getMaxTaskId() {
+ return maxTaskId;
+ }
+
+ public int getMaxTaskGroupId() {
+ return maxTaskGroupId;
+ }
+
+ public long getRecordsTotal() {
+ return recordsTotal;
+ }
+
+ public long getSizesTotal() {
+ return sizesTotal;
+ }
+
+ public long getAverageRecords() {
+ if (totalCount > 0) {
+ averageRecords = recordsTotal / totalCount;
+ }
+ return averageRecords;
+ }
+
+ public long getAverageBytes() {
+ if (totalCount > 0) {
+ averageBytes = sizesTotal / totalCount;
+ }
+ return averageBytes;
+ }
+
+ public long getMaxRecord() {
+ return maxRecord;
+ }
+
+ public long getMaxByte() {
+ return maxByte;
+ }
+
+ public int getMaxTaskId4Records() {
+ return maxTaskId4Records;
+ }
+
+ public int getMaxTGID4Records() {
+ return maxTGID4Records;
+ }
+
+ public int getTotalCount() {
+ return totalCount;
+ }
+ }
+ class JobStatisticsDto2 {
+
+ private Long id;
+ private Date gmtCreate;
+ private Date gmtModified;
+ private Long instId;
+ private Long jobId;
+ private Long jobVersion;
+ private Integer taskGroupId;
+ private Date windowStart;
+ private Date windowEnd;
+ private Date jobStartTime;
+ private Date jobEndTime;
+ private Long jobRunTimeMs;
+ private Integer jobPriority;
+ private Integer channelNum;
+ private String cluster;
+ private String jobDomain;
+ private String srcType;
+ private String dstType;
+ private String srcGuid;
+ private String dstGuid;
+ private Long records;
+ private Long bytes;
+ private Long speedRecord;
+ private Long speedByte;
+ private String stagePercent;
+ private Long errorRecord;
+ private Long errorBytes;
+ private Long waitReadTimeMs;
+ private Long waitWriteTimeMs;
+ private Long odpsBlockCloseTimeMs;
+ private Long sqlQueryTimeMs;
+ private Long resultNextTimeMs;
+ private Long taskTotalTimeMs;
+ private String hostAddress;
+
+ public Long getId() {
+ return id;
+ }
+
+ public Date getGmtCreate() {
+ return gmtCreate;
+ }
+
+ public Date getGmtModified() {
+ return gmtModified;
+ }
+
+ public Long getInstId() {
+ return instId;
+ }
+
+ public Long getJobId() {
+ return jobId;
+ }
+
+ public Long getJobVersion() {
+ return jobVersion;
+ }
+
+ public Integer getTaskGroupId() {
+ return taskGroupId;
+ }
+
+ public Date getWindowStart() {
+ return windowStart;
+ }
+
+ public Date getWindowEnd() {
+ return windowEnd;
+ }
+
+ public Date getJobStartTime() {
+ return jobStartTime;
+ }
+
+ public Date getJobEndTime() {
+ return jobEndTime;
+ }
+
+ public Long getJobRunTimeMs() {
+ return jobRunTimeMs;
+ }
+
+ public Integer getJobPriority() {
+ return jobPriority;
+ }
+
+ public Integer getChannelNum() {
+ return channelNum;
+ }
+
+ public String getCluster() {
+ return cluster;
+ }
+
+ public String getJobDomain() {
+ return jobDomain;
+ }
+
+ public String getSrcType() {
+ return srcType;
+ }
+
+ public String getDstType() {
+ return dstType;
+ }
+
+ public String getSrcGuid() {
+ return srcGuid;
+ }
+
+ public String getDstGuid() {
+ return dstGuid;
+ }
+
+ public Long getRecords() {
+ return records;
+ }
+
+ public Long getBytes() {
+ return bytes;
+ }
+
+ public Long getSpeedRecord() {
+ return speedRecord;
+ }
+
+ public Long getSpeedByte() {
+ return speedByte;
+ }
+
+ public String getStagePercent() {
+ return stagePercent;
+ }
+
+ public Long getErrorRecord() {
+ return errorRecord;
+ }
+
+ public Long getErrorBytes() {
+ return errorBytes;
+ }
+
+ public Long getWaitReadTimeMs() {
+ return waitReadTimeMs;
+ }
+
+ public Long getWaitWriteTimeMs() {
+ return waitWriteTimeMs;
+ }
+
+ public Long getOdpsBlockCloseTimeMs() {
+ return odpsBlockCloseTimeMs;
+ }
+
+ public Long getSqlQueryTimeMs() {
+ return sqlQueryTimeMs;
+ }
+
+ public Long getResultNextTimeMs() {
+ return resultNextTimeMs;
+ }
+
+ public Long getTaskTotalTimeMs() {
+ return taskTotalTimeMs;
+ }
+
+ public String getHostAddress() {
+ return hostAddress;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public void setGmtCreate(Date gmtCreate) {
+ this.gmtCreate = gmtCreate;
+ }
+
+ public void setGmtModified(Date gmtModified) {
+ this.gmtModified = gmtModified;
+ }
+
+ public void setInstId(Long instId) {
+ this.instId = instId;
+ }
+
+ public void setJobId(Long jobId) {
+ this.jobId = jobId;
+ }
+
+ public void setJobVersion(Long jobVersion) {
+ this.jobVersion = jobVersion;
+ }
+
+ public void setTaskGroupId(Integer taskGroupId) {
+ this.taskGroupId = taskGroupId;
+ }
+
+ public void setWindowStart(Date windowStart) {
+ this.windowStart = windowStart;
+ }
+
+ public void setWindowEnd(Date windowEnd) {
+ this.windowEnd = windowEnd;
+ }
+
+ public void setJobStartTime(Date jobStartTime) {
+ this.jobStartTime = jobStartTime;
+ }
+
+ public void setJobEndTime(Date jobEndTime) {
+ this.jobEndTime = jobEndTime;
+ }
+
+ public void setJobRunTimeMs(Long jobRunTimeMs) {
+ this.jobRunTimeMs = jobRunTimeMs;
+ }
+
+ public void setJobPriority(Integer jobPriority) {
+ this.jobPriority = jobPriority;
+ }
+
+ public void setChannelNum(Integer channelNum) {
+ this.channelNum = channelNum;
+ }
+
+ public void setCluster(String cluster) {
+ this.cluster = cluster;
+ }
+
+ public void setJobDomain(String jobDomain) {
+ this.jobDomain = jobDomain;
+ }
+
+ public void setSrcType(String srcType) {
+ this.srcType = srcType;
+ }
+
+ public void setDstType(String dstType) {
+ this.dstType = dstType;
+ }
+
+ public void setSrcGuid(String srcGuid) {
+ this.srcGuid = srcGuid;
+ }
+
+ public void setDstGuid(String dstGuid) {
+ this.dstGuid = dstGuid;
+ }
+
+ public void setRecords(Long records) {
+ this.records = records;
+ }
+
+ public void setBytes(Long bytes) {
+ this.bytes = bytes;
+ }
+
+ public void setSpeedRecord(Long speedRecord) {
+ this.speedRecord = speedRecord;
+ }
+
+ public void setSpeedByte(Long speedByte) {
+ this.speedByte = speedByte;
+ }
+
+ public void setStagePercent(String stagePercent) {
+ this.stagePercent = stagePercent;
+ }
+
+ public void setErrorRecord(Long errorRecord) {
+ this.errorRecord = errorRecord;
+ }
+
+ public void setErrorBytes(Long errorBytes) {
+ this.errorBytes = errorBytes;
+ }
+
+ public void setWaitReadTimeMs(Long waitReadTimeMs) {
+ this.waitReadTimeMs = waitReadTimeMs;
+ }
+
+ public void setWaitWriteTimeMs(Long waitWriteTimeMs) {
+ this.waitWriteTimeMs = waitWriteTimeMs;
+ }
+
+ public void setOdpsBlockCloseTimeMs(Long odpsBlockCloseTimeMs) {
+ this.odpsBlockCloseTimeMs = odpsBlockCloseTimeMs;
+ }
+
+ public void setSqlQueryTimeMs(Long sqlQueryTimeMs) {
+ this.sqlQueryTimeMs = sqlQueryTimeMs;
+ }
+
+ public void setResultNextTimeMs(Long resultNextTimeMs) {
+ this.resultNextTimeMs = resultNextTimeMs;
+ }
+
+ public void setTaskTotalTimeMs(Long taskTotalTimeMs) {
+ this.taskTotalTimeMs = taskTotalTimeMs;
+ }
+
+ public void setHostAddress(String hostAddress) {
+ this.hostAddress = hostAddress;
+ }
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java b/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java
new file mode 100644
index 0000000000..cab42a4b94
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java
@@ -0,0 +1,412 @@
+package com.alibaba.datax.common.statistics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by liqiang on 15/11/12.
+ */
+public class VMInfo {
+ private static final Logger LOG = LoggerFactory.getLogger(VMInfo.class);
+ static final long MB = 1024 * 1024;
+ static final long GB = 1024 * 1024 * 1024;
+ public static Object lock = new Object();
+ private static VMInfo vmInfo;
+
+ /**
+ * @return null or vmInfo. null is something error, job no care it.
+ */
+ public static VMInfo getVmInfo() {
+ if (vmInfo == null) {
+ synchronized (lock) {
+ if (vmInfo == null) {
+ try {
+ vmInfo = new VMInfo();
+ } catch (Exception e) {
+ LOG.warn("no need care, the fail is ignored : vmInfo init failed " + e.getMessage(), e);
+ }
+ }
+ }
+
+ }
+ return vmInfo;
+ }
+
+ // 数据的MxBean
+ private final OperatingSystemMXBean osMXBean;
+ private final RuntimeMXBean runtimeMXBean;
+ private final List garbageCollectorMXBeanList;
+ private final List memoryPoolMXBeanList;
+ /**
+ * 静态信息
+ */
+ private final String osInfo;
+ private final String jvmInfo;
+
+ /**
+ * cpu个数
+ */
+ private final int totalProcessorCount;
+
+ /**
+ * 机器的各个状态，用于中间打印和统计上报
+ */
+ private final PhyOSStatus startPhyOSStatus;
+ private final ProcessCpuStatus processCpuStatus = new ProcessCpuStatus();
+ private final ProcessGCStatus processGCStatus = new ProcessGCStatus();
+ private final ProcessMemoryStatus processMomoryStatus = new ProcessMemoryStatus();
+ //ms
+ private long lastUpTime = 0;
+ //nano
+ private long lastProcessCpuTime = 0;
+
+
+ private VMInfo() {
+ //初始化静态信息
+ osMXBean = java.lang.management.ManagementFactory.getOperatingSystemMXBean();
+ runtimeMXBean = java.lang.management.ManagementFactory.getRuntimeMXBean();
+ garbageCollectorMXBeanList = java.lang.management.ManagementFactory.getGarbageCollectorMXBeans();
+ memoryPoolMXBeanList = java.lang.management.ManagementFactory.getMemoryPoolMXBeans();
+
+ osInfo = runtimeMXBean.getVmVendor() + " " + runtimeMXBean.getSpecVersion() + " " + runtimeMXBean.getVmVersion();
+ jvmInfo = osMXBean.getName() + " " + osMXBean.getArch() + " " + osMXBean.getVersion();
+ totalProcessorCount = osMXBean.getAvailableProcessors();
+
+ //构建startPhyOSStatus
+ startPhyOSStatus = new PhyOSStatus();
+ LOG.info("VMInfo# operatingSystem class => " + osMXBean.getClass().getName());
+ if (VMInfo.isSunOsMBean(osMXBean)) {
+ {
+ startPhyOSStatus.totalPhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getTotalPhysicalMemorySize");
+ startPhyOSStatus.freePhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getFreePhysicalMemorySize");
+ startPhyOSStatus.maxFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getMaxFileDescriptorCount");
+ startPhyOSStatus.currentOpenFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getOpenFileDescriptorCount");
+ }
+ }
+
+ //初始化processGCStatus;
+ for (GarbageCollectorMXBean garbage : garbageCollectorMXBeanList) {
+ GCStatus gcStatus = new GCStatus();
+ gcStatus.name = garbage.getName();
+ processGCStatus.gcStatusMap.put(garbage.getName(), gcStatus);
+ }
+
+ //初始化processMemoryStatus
+ if (memoryPoolMXBeanList != null && !memoryPoolMXBeanList.isEmpty()) {
+ for (MemoryPoolMXBean pool : memoryPoolMXBeanList) {
+ MemoryStatus memoryStatus = new MemoryStatus();
+ memoryStatus.name = pool.getName();
+ memoryStatus.initSize = pool.getUsage().getInit();
+ memoryStatus.maxSize = pool.getUsage().getMax();
+ processMomoryStatus.memoryStatusMap.put(pool.getName(), memoryStatus);
+ }
+ }
+ }
+
+ public String toString() {
+ return "the machine info => \n\n"
+ + "\tosInfo:\t" + osInfo + "\n"
+ + "\tjvmInfo:\t" + jvmInfo + "\n"
+ + "\tcpu num:\t" + totalProcessorCount + "\n\n"
+ + startPhyOSStatus.toString() + "\n"
+ + processGCStatus.toString() + "\n"
+ + processMomoryStatus.toString() + "\n";
+ }
+
+ public String totalString() {
+ return (processCpuStatus.getTotalString() + processGCStatus.getTotalString());
+ }
+
+ public void getDelta() {
+ getDelta(true);
+ }
+
+ public synchronized void getDelta(boolean print) {
+
+ try {
+ if (VMInfo.isSunOsMBean(osMXBean)) {
+ long curUptime = runtimeMXBean.getUptime();
+ long curProcessTime = getLongFromOperatingSystem(osMXBean, "getProcessCpuTime");
+ //百分比， uptime是ms，processTime是nano
+ if ((curUptime > lastUpTime) && (curProcessTime >= lastProcessCpuTime)) {
+ float curDeltaCpu = (float) (curProcessTime - lastProcessCpuTime) / ((curUptime - lastUpTime) * totalProcessorCount * 10000);
+ processCpuStatus.setMaxMinCpu(curDeltaCpu);
+ processCpuStatus.averageCpu = (float) curProcessTime / (curUptime * totalProcessorCount * 10000);
+
+ lastUpTime = curUptime;
+ lastProcessCpuTime = curProcessTime;
+ }
+ }
+
+ for (GarbageCollectorMXBean garbage : garbageCollectorMXBeanList) {
+
+ GCStatus gcStatus = processGCStatus.gcStatusMap.get(garbage.getName());
+ if (gcStatus == null) {
+ gcStatus = new GCStatus();
+ gcStatus.name = garbage.getName();
+ processGCStatus.gcStatusMap.put(garbage.getName(), gcStatus);
+ }
+
+ long curTotalGcCount = garbage.getCollectionCount();
+ gcStatus.setCurTotalGcCount(curTotalGcCount);
+
+ long curtotalGcTime = garbage.getCollectionTime();
+ gcStatus.setCurTotalGcTime(curtotalGcTime);
+ }
+
+ if (memoryPoolMXBeanList != null && !memoryPoolMXBeanList.isEmpty()) {
+ for (MemoryPoolMXBean pool : memoryPoolMXBeanList) {
+
+ MemoryStatus memoryStatus = processMomoryStatus.memoryStatusMap.get(pool.getName());
+ if (memoryStatus == null) {
+ memoryStatus = new MemoryStatus();
+ memoryStatus.name = pool.getName();
+ processMomoryStatus.memoryStatusMap.put(pool.getName(), memoryStatus);
+ }
+ memoryStatus.commitedSize = pool.getUsage().getCommitted();
+ memoryStatus.setMaxMinUsedSize(pool.getUsage().getUsed());
+ long maxMemory = memoryStatus.commitedSize > 0 ? memoryStatus.commitedSize : memoryStatus.maxSize;
+ memoryStatus.setMaxMinPercent(maxMemory > 0 ? (float) 100 * memoryStatus.usedSize / maxMemory : -1);
+ }
+ }
+
+ if (print) {
+ LOG.info(processCpuStatus.getDeltaString() + processMomoryStatus.getDeltaString() + processGCStatus.getDeltaString());
+ }
+
+ } catch (Exception e) {
+ LOG.warn("no need care, the fail is ignored : vmInfo getDelta failed " + e.getMessage(), e);
+ }
+ }
+
+ public static boolean isSunOsMBean(OperatingSystemMXBean operatingSystem) {
+ final String className = operatingSystem.getClass().getName();
+
+ return "com.sun.management.UnixOperatingSystem".equals(className);
+ }
+
+ public static long getLongFromOperatingSystem(OperatingSystemMXBean operatingSystem, String methodName) {
+ try {
+ final Method method = operatingSystem.getClass().getMethod(methodName, (Class[]) null);
+ method.setAccessible(true);
+ return (Long) method.invoke(operatingSystem, (Object[]) null);
+ } catch (final Exception e) {
+ LOG.info(String.format("OperatingSystemMXBean %s failed, Exception = %s ", methodName, e.getMessage()));
+ }
+
+ return -1;
+ }
+
+ private class PhyOSStatus {
+ long totalPhysicalMemory = -1;
+ long freePhysicalMemory = -1;
+ long maxFileDescriptorCount = -1;
+ long currentOpenFileDescriptorCount = -1;
+
+ public String toString() {
+ return String.format("\ttotalPhysicalMemory:\t%,.2fG\n"
+ + "\tfreePhysicalMemory:\t%,.2fG\n"
+ + "\tmaxFileDescriptorCount:\t%s\n"
+ + "\tcurrentOpenFileDescriptorCount:\t%s\n",
+ (float) totalPhysicalMemory / GB, (float) freePhysicalMemory / GB, maxFileDescriptorCount, currentOpenFileDescriptorCount);
+ }
+ }
+
+ private class ProcessCpuStatus {
+ // 百分比的值 比如30.0 表示30.0%
+ float maxDeltaCpu = -1;
+ float minDeltaCpu = -1;
+ float curDeltaCpu = -1;
+ float averageCpu = -1;
+
+ public void setMaxMinCpu(float curCpu) {
+ this.curDeltaCpu = curCpu;
+ if (maxDeltaCpu < curCpu) {
+ maxDeltaCpu = curCpu;
+ }
+
+ if (minDeltaCpu == -1 || minDeltaCpu > curCpu) {
+ minDeltaCpu = curCpu;
+ }
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta cpu info] => \n");
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s \n", "curDeltaCpu", "averageCpu", "maxDeltaCpu", "minDeltaCpu"));
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s \n",
+ String.format("%,.2f%%", processCpuStatus.curDeltaCpu),
+ String.format("%,.2f%%", processCpuStatus.averageCpu),
+ String.format("%,.2f%%", processCpuStatus.maxDeltaCpu),
+ String.format("%,.2f%%\n", processCpuStatus.minDeltaCpu)));
+
+ return sb.toString();
+ }
+
+ public String getTotalString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [total cpu info] => \n");
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n", "averageCpu", "maxDeltaCpu", "minDeltaCpu"));
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n",
+ String.format("%,.2f%%", processCpuStatus.averageCpu),
+ String.format("%,.2f%%", processCpuStatus.maxDeltaCpu),
+ String.format("%,.2f%%\n", processCpuStatus.minDeltaCpu)));
+
+ return sb.toString();
+ }
+
+ }
+
+ private class ProcessGCStatus {
+ final Map gcStatusMap = new HashMap();
+
+ public String toString() {
+ return "\tGC Names\t" + gcStatusMap.keySet() + "\n";
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta gc info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n", "NAME", "curDeltaGCCount", "totalGCCount", "maxDeltaGCCount", "minDeltaGCCount", "curDeltaGCTime", "totalGCTime", "maxDeltaGCTime", "minDeltaGCTime"));
+ for (GCStatus gc : gcStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n",
+ gc.name, gc.curDeltaGCCount, gc.totalGCCount, gc.maxDeltaGCCount, gc.minDeltaGCCount,
+ String.format("%,.3fs",(float)gc.curDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.totalGCTime/1000),
+ String.format("%,.3fs",(float)gc.maxDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.minDeltaGCTime/1000)));
+
+ }
+ return sb.toString();
+ }
+
+ public String getTotalString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [total gc info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n", "NAME", "totalGCCount", "maxDeltaGCCount", "minDeltaGCCount", "totalGCTime", "maxDeltaGCTime", "minDeltaGCTime"));
+ for (GCStatus gc : gcStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n",
+ gc.name, gc.totalGCCount, gc.maxDeltaGCCount, gc.minDeltaGCCount,
+ String.format("%,.3fs",(float)gc.totalGCTime/1000),
+ String.format("%,.3fs",(float)gc.maxDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.minDeltaGCTime/1000)));
+
+ }
+ return sb.toString();
+ }
+ }
+
+ private class ProcessMemoryStatus {
+ final Map memoryStatusMap = new HashMap();
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n", "MEMORY_NAME", "allocation_size", "init_size"));
+ for (MemoryStatus ms : memoryStatusMap.values()) {
+ sb.append("\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n",
+ ms.name, String.format("%,.2fMB", (float) ms.maxSize / MB), String.format("%,.2fMB", (float) ms.initSize / MB)));
+ }
+ return sb.toString();
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta memory info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s | %-30s \n", "NAME", "used_size", "used_percent", "max_used_size", "max_percent"));
+ for (MemoryStatus ms : memoryStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s | %-30s \n",
+ ms.name, String.format("%,.2f", (float) ms.usedSize / MB) + "MB",
+ String.format("%,.2f", (float) ms.percent) + "%",
+ String.format("%,.2f", (float) ms.maxUsedSize / MB) + "MB",
+ String.format("%,.2f", (float) ms.maxpercent) + "%"));
+
+ }
+ return sb.toString();
+ }
+ }
+
+ private class GCStatus {
+ String name;
+ long maxDeltaGCCount = -1;
+ long minDeltaGCCount = -1;
+ long curDeltaGCCount;
+ long totalGCCount = 0;
+ long maxDeltaGCTime = -1;
+ long minDeltaGCTime = -1;
+ long curDeltaGCTime;
+ long totalGCTime = 0;
+
+ public void setCurTotalGcCount(long curTotalGcCount) {
+ this.curDeltaGCCount = curTotalGcCount - totalGCCount;
+ this.totalGCCount = curTotalGcCount;
+
+ if (maxDeltaGCCount < curDeltaGCCount) {
+ maxDeltaGCCount = curDeltaGCCount;
+ }
+
+ if (minDeltaGCCount == -1 || minDeltaGCCount > curDeltaGCCount) {
+ minDeltaGCCount = curDeltaGCCount;
+ }
+ }
+
+ public void setCurTotalGcTime(long curTotalGcTime) {
+ this.curDeltaGCTime = curTotalGcTime - totalGCTime;
+ this.totalGCTime = curTotalGcTime;
+
+ if (maxDeltaGCTime < curDeltaGCTime) {
+ maxDeltaGCTime = curDeltaGCTime;
+ }
+
+ if (minDeltaGCTime == -1 || minDeltaGCTime > curDeltaGCTime) {
+ minDeltaGCTime = curDeltaGCTime;
+ }
+ }
+ }
+
+ private class MemoryStatus {
+ String name;
+ long initSize;
+ long maxSize;
+ long commitedSize;
+ long usedSize;
+ float percent;
+ long maxUsedSize = -1;
+ float maxpercent = 0;
+
+ void setMaxMinUsedSize(long curUsedSize) {
+ if (maxUsedSize < curUsedSize) {
+ maxUsedSize = curUsedSize;
+ }
+ this.usedSize = curUsedSize;
+ }
+
+ void setMaxMinPercent(float curPercent) {
+ if (maxpercent < curPercent) {
+ maxpercent = curPercent;
+ }
+ this.percent = curPercent;
+ }
+ }
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/Configuration.java b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
new file mode 100755
index 0000000000..f570dd00c2
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
@@ -0,0 +1,1078 @@
+package com.alibaba.datax.common.util;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.spi.ErrorCode;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.CharUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Configuration 提供多级JSON配置信息无损存储
+ * 实例代码:
+ * 获取job的配置信息
+ *
+ *
+ *
+ *
+ *
+ * Configuration 存在两种较好地实现方式
+ * 目前使用的第二种实现方式，使用第一种的问题在于: secretKeyPathSet =
+ new HashSet();
+
+ private Object root = null;
+
+ /**
+ * 初始化空白的Configuration
+ */
+ public static Configuration newDefault() {
+ return Configuration.from("{}");
+ }
+
+ /**
+ * 从JSON字符串加载Configuration
+ */
+ public static Configuration from(String json) {
+ json = StrUtil.replaceVariable(json);
+ checkJSON(json);
+
+ try {
+ return new Configuration(json);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ e);
+ }
+
+ }
+
+ /**
+ * 从包括json的File对象加载Configuration
+ */
+ public static Configuration from(File file) {
+ try {
+ return Configuration.from(IOUtils
+ .toString(new FileInputStream(file)));
+ } catch (FileNotFoundException e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误，您提供的配置文件[%s]不存在. 请检查您的配置文件.", file.getAbsolutePath()));
+ } catch (IOException e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误. 您提供配置文件[%s]读取失败，错误原因: %s. 请检查您的配置文件的权限设置.",
+ file.getAbsolutePath(), e));
+ }
+ }
+
+ /**
+ * 从包括json的InputStream对象加载Configuration
+ */
+ public static Configuration from(InputStream is) {
+ try {
+ return Configuration.from(IOUtils.toString(is));
+ } catch (IOException e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("请检查您的配置文件. 您提供的配置文件读取失败，错误原因: %s. 请检查您的配置文件的权限设置.", e));
+ }
+ }
+
+ /**
+ * 从Map对象加载Configuration
+ */
+ public static Configuration from(final Map object) {
+ return Configuration.from(Configuration.toJSONString(object));
+ }
+
+ /**
+ * 从List对象加载Configuration
+ */
+ public static Configuration from(final List object) {
+ return Configuration.from(Configuration.toJSONString(object));
+ }
+
+ public String getNecessaryValue(String key, ErrorCode errorCode) {
+ String value = this.getString(key, null);
+ if (StringUtils.isBlank(value)) {
+ throw DataXException.asDataXException(errorCode,
+ String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
+ }
+
+ return value;
+ }
+
+ public String getUnnecessaryValue(String key,String defaultValue,ErrorCode errorCode) {
+ String value = this.getString(key, defaultValue);
+ if (StringUtils.isBlank(value)) {
+ value = defaultValue;
+ }
+ return value;
+ }
+
+ public Boolean getNecessaryBool(String key, ErrorCode errorCode) {
+ Boolean value = this.getBool(key);
+ if (value == null) {
+ throw DataXException.asDataXException(errorCode,
+ String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
+ }
+
+ return value;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址具体的对象。
+ *
+ *
+ * NOTE: 目前仅支持Map以及List下标寻址, 例如:
+ *
+ *
+ * 对于如下JSON
+ *
+ * {"a": {"b": {"c": [0,1,2,3]}}}
+ *
+ * config.get("") 返回整个Map
+ * getList(final String path) {
+ List list = this.get(path, List.class);
+ if (null == list) {
+ return null;
+ }
+ return list;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
+ */
+ @SuppressWarnings("unchecked")
+ public List getList(final String path, Class t) {
+ Object object = this.get(path, List.class);
+ if (null == object) {
+ return null;
+ }
+
+ List result = new ArrayList();
+
+ List origin = (List) object;
+ for (final Object each : origin) {
+ result.add((T) each);
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
+ */
+ @SuppressWarnings("unchecked")
+ public List getList(final String path,
+ final List defaultList) {
+ Object object = this.getList(path);
+ if (null == object) {
+ return defaultList;
+ }
+ return (List) object;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
+ */
+ public List getList(final String path, final List defaultList,
+ Class t) {
+ List list = this.getList(path, t);
+ if (null == list) {
+ return defaultList;
+ }
+ return list;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址包含Configuration的List，如果对象不存在，返回默认null
+ */
+ public List getListConfiguration(final String path) {
+ List lists = getList(path);
+ if (lists == null) {
+ return null;
+ }
+
+ List result = new ArrayList();
+ for (final Object object : lists) {
+ result.add(Configuration.from(Configuration.toJSONString(object)));
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path) {
+ Map result = this.get(path, Map.class);
+ if (null == result) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null;
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path, Class t) {
+ Map map = this.get(path, Map.class);
+ if (null == map) {
+ return null;
+ }
+
+ Map result = new HashMap();
+ for (final String key : map.keySet()) {
+ result.put(key, (T) map.get(key));
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path,
+ final Map defaultMap) {
+ Object object = this.getMap(path);
+ if (null == object) {
+ return defaultMap;
+ }
+ return (Map) object;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
+ */
+ public Map getMap(final String path,
+ final Map defaultMap, Class t) {
+ Map result = getMap(path, t);
+ if (null == result) {
+ return defaultMap;
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址包含Configuration的Map，如果对象不存在，返回默认null
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMapConfiguration(final String path) {
+ Map map = this.get(path, Map.class);
+ if (null == map) {
+ return null;
+ }
+
+ Map result = new HashMap();
+ for (final String key : map.keySet()) {
+ result.put(key, Configuration.from(Configuration.toJSONString(map
+ .get(key))));
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址具体的对象，并转为用户提供的类型
+ *
+ *
+ * NOTE: 目前仅支持Map以及List下标寻址, 例如:
+ *
+ *
+ * 对于如下JSON
+ *
+ * {"a": {"b": {"c": [0,1,2,3]}}}
+ *
+ * config.get("") 返回整个Map T get(final String path, Class clazz) {
+ this.checkPath(path);
+ return (T) this.get(path);
+ }
+
+ /**
+ * 格式化Configuration输出
+ */
+ public String beautify() {
+ return JSON.toJSONString(this.getInternal(),
+ SerializerFeature.PrettyFormat);
+ }
+
+ /**
+ * 根据用户提供的json path，插入指定对象，并返回之前存在的对象(如果存在)
+ *
+ *
+ * 目前仅支持.以及数组下标寻址, 例如:
+ *
+ *
+ * config.set("a.b.c[3]", object);
+ *
+ * 、List)，不要使用自定义对象，否则后续对于JSON序列化等情况会出现未定义行为。
+ *
+ * @param path
+ * JSON path对象
+ * @param object
+ * 需要插入的对象
+ * @return Java表示的JSON对象
+ */
+ public Object set(final String path, final Object object) {
+ checkPath(path);
+
+ Object result = this.get(path);
+
+ setObject(path, extractConfiguration(object));
+
+ return result;
+ }
+
+ /**
+ * 获取Configuration下所有叶子节点的key
+ *
+ *
+ * 对于
+ * {"a": {"b": {"c": [0,1,2,3]}}, "x": "y"}
+ *
+ * 下属的key包括: a.b.c[0],a.b.c[1],a.b.c[2],a.b.c[3],x
+ */
+ public Set getKeys() {
+ Set collect = new HashSet();
+ this.getKeysRecursive(this.getInternal(), "", collect);
+ return collect;
+ }
+
+ /**
+ * 删除path对应的值，如果path不存在，将抛出异常。
+ */
+ public Object remove(final String path) {
+ final Object result = this.get(path);
+ if (null == result) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR,
+ String.format("配置文件对应Key[%s]并不存在，该情况是代码编程错误. 请联系DataX团队的同学.", path));
+ }
+
+ this.set(path, null);
+ return result;
+ }
+
+ /**
+ * 合并其他Configuration，并修改两者冲突的KV配置
+ *
+ * @param another
+ * 合并加入的第三方Configuration
+ * @param updateWhenConflict
+ * 当合并双方出现KV冲突时候，选择更新当前KV，或者忽略该KV
+ * @return 返回合并后对象
+ */
+ public Configuration merge(final Configuration another,
+ boolean updateWhenConflict) {
+ Set keys = another.getKeys();
+
+ for (final String key : keys) {
+ // 如果使用更新策略，凡是another存在的key，均需要更新
+ if (updateWhenConflict) {
+ this.set(key, another.get(key));
+ continue;
+ }
+
+ // 使用忽略策略，只有another Configuration存在但是当前Configuration不存在的key，才需要更新
+ boolean isCurrentExists = this.get(key) != null;
+ if (isCurrentExists) {
+ continue;
+ }
+
+ this.set(key, another.get(key));
+ }
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return this.toJSON();
+ }
+
+ /**
+ * 将Configuration作为JSON输出
+ */
+ public String toJSON() {
+ return Configuration.toJSONString(this.getInternal());
+ }
+
+ /**
+ * 拷贝当前Configuration，注意，这里使用了深拷贝，避免冲突
+ */
+ public Configuration clone() {
+ Configuration config = Configuration
+ .from(Configuration.toJSONString(this.getInternal()));
+ config.addSecretKeyPath(this.secretKeyPathSet);
+ return config;
+ }
+
+ /**
+ * 按照configuration要求格式的path
+ * 比如：
+ * a.b.c
+ * a.b[2].c
+ * @param path
+ */
+ public void addSecretKeyPath(String path) {
+ if(StringUtils.isNotBlank(path)) {
+ this.secretKeyPathSet.add(path);
+ }
+ }
+
+ public void addSecretKeyPath(Set pathSet) {
+ if(pathSet != null) {
+ this.secretKeyPathSet.addAll(pathSet);
+ }
+ }
+
+ public void setSecretKeyPathSet(Set keyPathSet) {
+ if(keyPathSet != null) {
+ this.secretKeyPathSet = keyPathSet;
+ }
+ }
+
+ public boolean isSecretPath(String path) {
+ return this.secretKeyPathSet.contains(path);
+ }
+
+ @SuppressWarnings("unchecked")
+ void getKeysRecursive(final Object current, String path, Set collect) {
+ boolean isRegularElement = !(current instanceof Map || current instanceof List);
+ if (isRegularElement) {
+ collect.add(path);
+ return;
+ }
+
+ boolean isMap = current instanceof Map;
+ if (isMap) {
+ Map mapping = ((Map) current);
+ for (final String key : mapping.keySet()) {
+ if (StringUtils.isBlank(path)) {
+ getKeysRecursive(mapping.get(key), key.trim(), collect);
+ } else {
+ getKeysRecursive(mapping.get(key), path + "." + key.trim(),
+ collect);
+ }
+ }
+ return;
+ }
+
+ boolean isList = current instanceof List;
+ if (isList) {
+ List lists = (List) current;
+ for (int i = 0; i < lists.size(); i++) {
+ getKeysRecursive(lists.get(i), path + String.format("[%d]", i),
+ collect);
+ }
+ return;
+ }
+
+ return;
+ }
+
+ public Object getInternal() {
+ return this.root;
+ }
+
+ private void setObject(final String path, final Object object) {
+ Object newRoot = setObjectRecursive(this.root, split2List(path), 0,
+ object);
+
+ if (isSuitForRoot(newRoot)) {
+ this.root = newRoot;
+ return;
+ }
+
+ throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
+ String.format("值[%s]无法适配您提供[%s]， 该异常代表系统编程错误, 请联系DataX开发团队!",
+ ToStringBuilder.reflectionToString(object), path));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object extractConfiguration(final Object object) {
+ if (object instanceof Configuration) {
+ return extractFromConfiguration(object);
+ }
+
+ if (object instanceof List) {
+ List result = new ArrayList();
+ for (final Object each : (List) object) {
+ result.add(extractFromConfiguration(each));
+ }
+ return result;
+ }
+
+ if (object instanceof Map) {
+ Map result = new HashMap();
+ for (final String key : ((Map) object).keySet()) {
+ result.put(key,
+ extractFromConfiguration(((Map) object)
+ .get(key)));
+ }
+ return result;
+ }
+
+ return object;
+ }
+
+ private Object extractFromConfiguration(final Object object) {
+ if (object instanceof Configuration) {
+ return ((Configuration) object).getInternal();
+ }
+
+ return object;
+ }
+
+ Object buildObject(final List paths, final Object object) {
+ if (null == paths) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR,
+ "Path不能为null，该异常代表系统编程错误, 请联系DataX开发团队 !");
+ }
+
+ if (1 == paths.size() && StringUtils.isBlank(paths.get(0))) {
+ return object;
+ }
+
+ Object child = object;
+ for (int i = paths.size() - 1; i >= 0; i--) {
+ String path = paths.get(i);
+
+ if (isPathMap(path)) {
+ Map mapping = new HashMap();
+ mapping.put(path, child);
+ child = mapping;
+ continue;
+ }
+
+ if (isPathList(path)) {
+ List lists = new ArrayList(
+ this.getIndex(path) + 1);
+ expand(lists, this.getIndex(path) + 1);
+ lists.set(this.getIndex(path), child);
+ child = lists;
+ continue;
+ }
+
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR, String.format(
+ "路径[%s]出现非法值类型[%s]，该异常代表系统编程错误, 请联系DataX开发团队! .",
+ StringUtils.join(paths, "."), path));
+ }
+
+ return child;
+ }
+
+ @SuppressWarnings("unchecked")
+ Object setObjectRecursive(Object current, final List paths,
+ int index, final Object value) {
+
+ // 如果是已经超出path，我们就返回value即可，作为最底层叶子节点
+ boolean isLastIndex = index == paths.size();
+ if (isLastIndex) {
+ return value;
+ }
+
+ String path = paths.get(index).trim();
+ boolean isNeedMap = isPathMap(path);
+ if (isNeedMap) {
+ Map mapping;
+
+ // 当前不是map，因此全部替换为map，并返回新建的map对象
+ boolean isCurrentMap = current instanceof Map;
+ if (!isCurrentMap) {
+ mapping = new HashMap();
+ mapping.put(
+ path,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return mapping;
+ }
+
+ // 当前是map，但是没有对应的key，也就是我们需要新建对象插入该map，并返回该map
+ mapping = ((Map) current);
+ boolean hasSameKey = mapping.containsKey(path);
+ if (!hasSameKey) {
+ mapping.put(
+ path,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return mapping;
+ }
+
+ // 当前是map，而且还竟然存在这个值，好吧，继续递归遍历
+ current = mapping.get(path);
+ mapping.put(path,
+ setObjectRecursive(current, paths, index + 1, value));
+ return mapping;
+ }
+
+ boolean isNeedList = isPathList(path);
+ if (isNeedList) {
+ List lists;
+ int listIndexer = getIndex(path);
+
+ // 当前是list，直接新建并返回即可
+ boolean isCurrentList = current instanceof List;
+ if (!isCurrentList) {
+ lists = expand(new ArrayList(), listIndexer + 1);
+ lists.set(
+ listIndexer,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return lists;
+ }
+
+ // 当前是list，但是对应的indexer是没有具体的值，也就是我们新建对象然后插入到该list，并返回该List
+ lists = (List) current;
+ lists = expand(lists, listIndexer + 1);
+
+ boolean hasSameIndex = lists.get(listIndexer) != null;
+ if (!hasSameIndex) {
+ lists.set(
+ listIndexer,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return lists;
+ }
+
+ // 当前是list，并且存在对应的index，没有办法继续递归寻找
+ current = lists.get(listIndexer);
+ lists.set(listIndexer,
+ setObjectRecursive(current, paths, index + 1, value));
+ return lists;
+ }
+
+ throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
+ "该异常代表系统编程错误, 请联系DataX开发团队 !");
+ }
+
+ private Object findObject(final String path) {
+ boolean isRootQuery = StringUtils.isBlank(path);
+ if (isRootQuery) {
+ return this.root;
+ }
+
+ Object target = this.root;
+
+ for (final String each : split2List(path)) {
+ if (isPathMap(each)) {
+ target = findObjectInMap(target, each);
+ continue;
+ } else {
+ target = findObjectInList(target, each);
+ continue;
+ }
+ }
+
+ return target;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object findObjectInMap(final Object target, final String index) {
+ boolean isMap = (target instanceof Map);
+ if (!isMap) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
+ index, target.getClass().toString()));
+ }
+
+ Object result = ((Map) target).get(index);
+ if (null == result) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]值为null，datax无法识别该配置. 请检查您的配置并作出修改.", index));
+ }
+
+ return result;
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ private Object findObjectInList(final Object target, final String each) {
+ boolean isList = (target instanceof List);
+ if (!isList) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
+ each, target.getClass().toString()));
+ }
+
+ String index = each.replace("[", "").replace("]", "");
+ if (!StringUtils.isNumeric(index)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "系统编程错误，列表下标必须为数字类型，但该节点发现实际类型是[%s] ，该异常代表系统编程错误, 请联系DataX开发团队 !",
+ index));
+ }
+
+ return ((List) target).get(Integer.valueOf(index));
+ }
+
+ private List expand(List list, int size) {
+ int expand = size - list.size();
+ while (expand-- > 0) {
+ list.add(null);
+ }
+ return list;
+ }
+
+ private boolean isPathList(final String path) {
+ return path.contains("[") && path.contains("]");
+ }
+
+ private boolean isPathMap(final String path) {
+ return StringUtils.isNotBlank(path) && !isPathList(path);
+ }
+
+ private int getIndex(final String index) {
+ return Integer.valueOf(index.replace("[", "").replace("]", ""));
+ }
+
+ private boolean isSuitForRoot(final Object object) {
+ if (null != object && (object instanceof List || object instanceof Map)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private String split(final String path) {
+ return StringUtils.replace(path, "[", ".[");
+ }
+
+ private List split2List(final String path) {
+ return Arrays.asList(StringUtils.split(split(path), "."));
+ }
+
+ private void checkPath(final String path) {
+ if (null == path) {
+ throw new IllegalArgumentException(
+ "系统编程错误, 该异常代表系统编程错误, 请联系DataX开发团队!.");
+ }
+
+ for (final String each : StringUtils.split(".")) {
+ if (StringUtils.isBlank(each)) {
+ throw new IllegalArgumentException(String.format(
+ "系统编程错误, 路径[%s]不合法, 路径层次之间不能出现空白字符 .", path));
+ }
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private String toJSONPath(final String path) {
+ return (StringUtils.isBlank(path) ? "$" : "$." + path).replace("$.[",
+ "$[");
+ }
+
+ private static void checkJSON(final String json) {
+ if (StringUtils.isBlank(json)) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ "配置信息错误. 因为您提供的配置信息不是合法的JSON格式, JSON不能为空白. 请按照标准json格式提供配置信息. ");
+ }
+ }
+
+ private Configuration(final String json) {
+ try {
+ this.root = JSON.parse(json);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误. 您提供的配置信息不是合法的JSON格式: %s . 请按照标准json格式提供配置信息. ", e.getMessage()));
+ }
+ }
+
+ private static String toJSONString(final Object object) {
+ return JSON.toJSONString(object);
+ }
+
+ public Set getSecretKeyPathSet() {
+ return secretKeyPathSet;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java b/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java
new file mode 100755
index 0000000000..37b319a194
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java
@@ -0,0 +1,52 @@
+package com.alibaba.datax.common.util;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+/**
+ * 提供从 List 中根据 regular 过滤的通用工具(返回值已经去重). 使用场景，比如：odpsreader
+ * 的分区筛选，hdfsreader/txtfilereader的路径筛选等
+ */
+public final class FilterUtil {
+
+ //已经去重
+ public static List filterByRegular(List allStrs,
+ String regular) {
+ List matchedValues = new ArrayList();
+
+ // 语法习惯上的兼容处理(pt=* 实际正则应该是：pt=.*)
+ String newReqular = regular.replace(".*", "*").replace("*", ".*");
+
+ Pattern p = Pattern.compile(newReqular);
+
+ for (String partition : allStrs) {
+ if (p.matcher(partition).matches()) {
+ if (!matchedValues.contains(partition)) {
+ matchedValues.add(partition);
+ }
+ }
+ }
+
+ return matchedValues;
+ }
+
+ //已经去重
+ public static List filterByRegulars(List allStrs,
+ List regulars) {
+ List matchedValues = new ArrayList();
+
+ List tempMatched = null;
+ for (String regular : regulars) {
+ tempMatched = filterByRegular(allStrs, regular);
+ if (null != tempMatched && !tempMatched.isEmpty()) {
+ for (String temp : tempMatched) {
+ if (!matchedValues.contains(temp)) {
+ matchedValues.add(temp);
+ }
+ }
+ }
+ }
+
+ return matchedValues;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java b/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java
new file mode 100644
index 0000000000..2ed8f1019c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java
@@ -0,0 +1,49 @@
+package com.alibaba.datax.common.util;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Created by liqiang on 15/8/25.
+ */
+public class HostUtils {
+
+ public static final String IP;
+ public static final String HOSTNAME;
+ private static final Logger log = LoggerFactory.getLogger(HostUtils.class);
+
+ static {
+ String ip;
+ String hostname;
+ try {
+ InetAddress addr = InetAddress.getLocalHost();
+ ip = addr.getHostAddress();
+ hostname = addr.getHostName();
+ } catch (UnknownHostException e) {
+ log.error("Can't find out address: " + e.getMessage());
+ ip = "UNKNOWN";
+ hostname = "UNKNOWN";
+ }
+ if (ip.equals("127.0.0.1") || ip.equals("::1") || ip.equals("UNKNOWN")) {
+ try {
+ Process process = Runtime.getRuntime().exec("hostname -i");
+ if (process.waitFor() == 0) {
+ ip = new String(IOUtils.toByteArray(process.getInputStream()), "UTF8");
+ }
+ process = Runtime.getRuntime().exec("hostname");
+ if (process.waitFor() == 0) {
+ hostname = (new String(IOUtils.toByteArray(process.getInputStream()), "UTF8")).trim();
+ }
+ } catch (Exception e) {
+ log.warn("get hostname failed {}", e.getMessage());
+ }
+ }
+ IP = ip;
+ HOSTNAME = hostname;
+ log.info("IP {} HOSTNAME {}", IP, HOSTNAME);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java
new file mode 100755
index 0000000000..d7a5b76462
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java
@@ -0,0 +1,139 @@
+package com.alibaba.datax.common.util;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * 提供针对 DataX中使用的 List 较为常见的一些封装。 比如：checkIfValueDuplicate 可以用于检查用户配置的 writer
+ * 的列不能重复。makeSureNoValueDuplicate亦然，只是会严格报错。
+ */
+public final class ListUtil {
+
+ public static boolean checkIfValueDuplicate(List aList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty()) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ "您提供的作业配置有误，List不能为空.");
+ }
+
+ try {
+ makeSureNoValueDuplicate(aList, caseSensitive);
+ } catch (Exception e) {
+ return true;
+ }
+ return false;
+ }
+
+ public static void makeSureNoValueDuplicate(List aList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ if (1 == aList.size()) {
+ return;
+ } else {
+ List list = null;
+ if (!caseSensitive) {
+ list = valueToLowerCase(aList);
+ } else {
+ list = new ArrayList(aList);
+ }
+
+ Collections.sort(list);
+
+ for (int i = 0, len = list.size() - 1; i < len; i++) {
+ if (list.get(i).equals(list.get(i + 1))) {
+ throw DataXException
+ .asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format(
+ "您提供的作业配置信息有误, String:[%s] 不允许重复出现在列表中: [%s].",
+ list.get(i),
+ StringUtils.join(aList, ",")));
+ }
+ }
+ }
+ }
+
+ public static boolean checkIfBInA(List aList, List bList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty() || null == bList
+ || bList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ try {
+ makeSureBInA(aList, bList, caseSensitive);
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ public static void makeSureBInA(List aList, List bList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty() || null == bList
+ || bList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ List all = null;
+ List part = null;
+
+ if (!caseSensitive) {
+ all = valueToLowerCase(aList);
+ part = valueToLowerCase(bList);
+ } else {
+ all = new ArrayList(aList);
+ part = new ArrayList(bList);
+ }
+
+ for (String oneValue : part) {
+ if (!all.contains(oneValue)) {
+ throw DataXException
+ .asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format(
+ "您提供的作业配置信息有误, String:[%s] 不存在于列表中:[%s].",
+ oneValue, StringUtils.join(aList, ",")));
+ }
+ }
+
+ }
+
+ public static boolean checkIfValueSame(List aList) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ if (1 == aList.size()) {
+ return true;
+ } else {
+ Boolean firstValue = aList.get(0);
+ for (int i = 1, len = aList.size(); i < len; i++) {
+ if (firstValue.booleanValue() != aList.get(i).booleanValue()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public static List valueToLowerCase(List aList) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+ List result = new ArrayList(aList.size());
+ for (String oneValue : aList) {
+ result.add(null != oneValue ? oneValue.toLowerCase() : null);
+ }
+
+ return result;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java b/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java
new file mode 100755
index 0000000000..791f9ea12c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java
@@ -0,0 +1,209 @@
+package com.alibaba.datax.common.util;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.math.BigInteger;
+import java.util.*;
+
+/**
+ * 提供通用的根据数字范围、字符串范围等进行切分的通用功能.
+ */
+public final class RangeSplitUtil {
+
+ public static String[] doAsciiStringSplit(String left, String right, int expectSliceNumber) {
+ int radix = 128;
+
+ BigInteger[] tempResult = doBigIntegerSplit(stringToBigInteger(left, radix),
+ stringToBigInteger(right, radix), expectSliceNumber);
+ String[] result = new String[tempResult.length];
+
+ //处理第一个字符串（因为：在转换为数字，再还原的时候，如果首字符刚好是 basic,则不知道应该添加多少个 basic）
+ result[0] = left;
+ result[tempResult.length - 1] = right;
+
+ for (int i = 1, len = tempResult.length - 1; i < len; i++) {
+ result[i] = bigIntegerToString(tempResult[i], radix);
+ }
+
+ return result;
+ }
+
+
+ public static long[] doLongSplit(long left, long right, int expectSliceNumber) {
+ BigInteger[] result = doBigIntegerSplit(BigInteger.valueOf(left),
+ BigInteger.valueOf(right), expectSliceNumber);
+ long[] returnResult = new long[result.length];
+ for (int i = 0, len = result.length; i < len; i++) {
+ returnResult[i] = result[i].longValue();
+ }
+ return returnResult;
+ }
+
+ public static BigInteger[] doBigIntegerSplit(BigInteger left, BigInteger right, int expectSliceNumber) {
+ if (expectSliceNumber < 1) {
+ throw new IllegalArgumentException(String.format(
+ "切分份数不能小于1. 此处:expectSliceNumber=[%s].", expectSliceNumber));
+ }
+
+ if (null == left || null == right) {
+ throw new IllegalArgumentException(String.format(
+ "对 BigInteger 进行切分时，其左右区间不能为 null. 此处:left=[%s],right=[%s].", left, right));
+ }
+
+ if (left.compareTo(right) == 0) {
+ return new BigInteger[]{left, right};
+ } else {
+ // 调整大小顺序，确保 left < right
+ if (left.compareTo(right) > 0) {
+ BigInteger temp = left;
+ left = right;
+ right = temp;
+ }
+
+ //left < right
+ BigInteger endAndStartGap = right.subtract(left);
+
+ BigInteger step = endAndStartGap.divide(BigInteger.valueOf(expectSliceNumber));
+ BigInteger remainder = endAndStartGap.remainder(BigInteger.valueOf(expectSliceNumber));
+
+ //remainder 不可能超过expectSliceNumber,所以不需要检查remainder的 Integer 的范围
+
+ // 这里不能 step.intValue()==0，因为可能溢出
+ if (step.compareTo(BigInteger.ZERO) == 0) {
+ expectSliceNumber = remainder.intValue();
+ }
+
+ BigInteger[] result = new BigInteger[expectSliceNumber + 1];
+ result[0] = left;
+ result[expectSliceNumber] = right;
+
+ BigInteger lowerBound;
+ BigInteger upperBound = left;
+ for (int i = 1; i < expectSliceNumber; i++) {
+ lowerBound = upperBound;
+ upperBound = lowerBound.add(step);
+ upperBound = upperBound.add((remainder.compareTo(BigInteger.valueOf(i)) >= 0)
+ ? BigInteger.ONE : BigInteger.ZERO);
+ result[i] = upperBound;
+ }
+
+ return result;
+ }
+ }
+
+ private static void checkIfBetweenRange(int value, int left, int right) {
+ if (value < left || value > right) {
+ throw new IllegalArgumentException(String.format("parameter can not <[%s] or >[%s].",
+ left, right));
+ }
+ }
+
+ /**
+ * 由于只支持 ascii 码对应字符，所以radix 范围为[1,128]
+ */
+ public static BigInteger stringToBigInteger(String aString, int radix) {
+ if (null == aString) {
+ throw new IllegalArgumentException("参数 bigInteger 不能为空.");
+ }
+
+ checkIfBetweenRange(radix, 1, 128);
+
+ BigInteger result = BigInteger.ZERO;
+ BigInteger radixBigInteger = BigInteger.valueOf(radix);
+
+ int tempChar;
+ int k = 0;
+
+ for (int i = aString.length() - 1; i >= 0; i--) {
+ tempChar = aString.charAt(i);
+ if (tempChar >= 128) {
+ throw new IllegalArgumentException(String.format("根据字符串进行切分时仅支持 ASCII 字符串，而字符串:[%s]非 ASCII 字符串.", aString));
+ }
+ result = result.add(BigInteger.valueOf(tempChar).multiply(radixBigInteger.pow(k)));
+ k++;
+ }
+
+ return result;
+ }
+
+ /**
+ * 把BigInteger 转换为 String.注意：radix 和 basic 范围都为[1,128], radix + basic 的范围也必须在[1,128].
+ */
+ private static String bigIntegerToString(BigInteger bigInteger, int radix) {
+ if (null == bigInteger) {
+ throw new IllegalArgumentException("参数 bigInteger 不能为空.");
+ }
+
+ checkIfBetweenRange(radix, 1, 128);
+
+ StringBuilder resultStringBuilder = new StringBuilder();
+
+ List list = new ArrayList();
+ BigInteger radixBigInteger = BigInteger.valueOf(radix);
+ BigInteger currentValue = bigInteger;
+
+ BigInteger quotient = currentValue.divide(radixBigInteger);
+ while (quotient.compareTo(BigInteger.ZERO) > 0) {
+ list.add(currentValue.remainder(radixBigInteger).intValue());
+ currentValue = currentValue.divide(radixBigInteger);
+ quotient = currentValue;
+ }
+ Collections.reverse(list);
+
+ if (list.isEmpty()) {
+ list.add(0, bigInteger.remainder(radixBigInteger).intValue());
+ }
+
+ Map map = new HashMap();
+ for (int i = 0; i < radix; i++) {
+ map.put(i, (char) (i));
+ }
+
+// String msg = String.format("%s 转为 %s 进制，结果为：%s", bigInteger.longValue(), radix, list);
+// System.out.println(msg);
+
+ for (Integer aList : list) {
+ resultStringBuilder.append(map.get(aList));
+ }
+
+ return resultStringBuilder.toString();
+ }
+
+ /**
+ * 获取字符串中的最小字符和最大字符（依据 ascii 进行判断）.要求字符串必须非空，并且为 ascii 字符串.
+ * 返回的Pair，left=最小字符，right=最大字符.
+ */
+ public static Pair getMinAndMaxCharacter(String aString) {
+ if (!isPureAscii(aString)) {
+ throw new IllegalArgumentException(String.format("根据字符串进行切分时仅支持 ASCII 字符串，而字符串:[%s]非 ASCII 字符串.", aString));
+ }
+
+ char min = aString.charAt(0);
+ char max = min;
+
+ char temp;
+ for (int i = 1, len = aString.length(); i < len; i++) {
+ temp = aString.charAt(i);
+ min = min < temp ? min : temp;
+ max = max > temp ? max : temp;
+ }
+
+ return new ImmutablePair