From 8bd01ac795d1ebb0e4a7e7380e4a66e2f0a48cde Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Wed, 12 Feb 2020 16:33:26 -0800 Subject: [PATCH 01/16] expose date fields using a standardized date format --- build.gradle | 1 + .../semantic/visitor/ESMappingLoader.java | 11 ++ .../sql/esdomain/LocalClusterState.java | 11 ++ .../sql/executor/format/DateFormat.java | 116 ++++++++++++++++++ .../sql/executor/format/Protocol.java | 6 +- .../sql/executor/format/SelectResultSet.java | 54 +++++++- .../sql/plugin/RestSqlAction.java | 5 + .../sql/query/QueryAction.java | 12 ++ 8 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java diff --git a/build.gradle b/build.gradle index 6f9fa1eba8..8898057aeb 100644 --- a/build.gradle +++ b/build.gradle @@ -223,6 +223,7 @@ dependencies { compile group: "org.elasticsearch.plugin", name: 'reindex-client', version: "${es_version}" compile group: 'com.google.guava', name: 'guava', version:'15.0' compile group: 'org.json', name: 'json', version:'20180813' + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9' // ANTLR gradle plugin and runtime dependency antlr "org.antlr:antlr4:4.7.1" diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java index 34c1ebc801..f749988a67 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java @@ -105,9 +105,20 @@ private void defineIndexType(String indexName) { private void loadAllFieldsWithType(String indexName) { FieldMappings mappings = getFieldMappings(indexName); + saveDateFormats(mappings); mappings.flat(this::defineFieldName); } + private void saveDateFormats(FieldMappings mappings) + { + for (Map.Entry> data : mappings.data().entrySet()) { + String type = data.getValue().get("type").toString(); + if ("date".equals(type)) { + clusterState.pushDateFieldFormat(data.getKey(), data.getValue().get("format").toString()); + } + } + } + /* * 3.1 Define with alias if given: ex."SELECT * FROM accounts a". * 'accounts' -> INDEX diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index 190c827c06..68d760ea55 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -91,6 +92,7 @@ public class LocalClusterState { */ private final Map latestSettings = new ConcurrentHashMap<>(); + private Map dateFieldFormatMap = new HashMap<>(); public static synchronized LocalClusterState state() { if (INSTANCE == null) { @@ -242,4 +244,13 @@ private List sortToList(T[] array) { return Arrays.asList(array); } + public void pushDateFieldFormat(String fieldName, String dateFormat) + { + dateFieldFormatMap.put(fieldName, dateFormat); + } + + public Map getDateFieldFormatMap() + { + return dateFieldFormatMap; + } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java new file mode 100644 index 0000000000..bd8a45260d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -0,0 +1,116 @@ +package com.amazon.opendistroforelasticsearch.sql.executor.format; + +import java.text.SimpleDateFormat; + +public enum DateFormat +{ + // Special cases that are parsed separately + DATE_OPTIONAL_TIME(""), + EPOCH_MILLIS(""), + EPOCH_SECOND(""), + + BASIC_DATE(Date.BASIC_DATE), + BASIC_DATE_TIME(Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), + BASIC_DATE_TIME_NO_MILLIS(Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.TZ), + + BASIC_ORDINAL_DATE(Date.BASIC_ORDINAL_DATE), + BASIC_ORDINAL_DATE_TIME(Date.BASIC_ORDINAL_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), + BASIC_ORDINAL_DATE_TIME_NO_MILLIS(Date.BASIC_ORDINAL_DATE+ Time.T + Time.BASIC_TIME + Time.TZ), + + BASIC_TIME(Time.BASIC_TIME + Time.MILLIS + Time.TZ), + BASIC_TIME_NO_MILLIS(Time.BASIC_TIME + Time.TZ), + + BASIC_T_TIME(Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), + BASIC_T_TIME_NO_MILLIS(Time.T + Time.BASIC_TIME + Time.TZ), + + BASIC_WEEK_DATE(Date.BASIC_WEEK_DATE), + BASIC_WEEK_DATE_TIME(Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), + BASIC_WEEK_DATE_TIME_NO_MILLIS(Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.TZ), + + DATE(Date.DATE), + DATE_HOUR(Date.DATE + Time.T + Time.HOUR), + DATE_HOUR_MINUTE(Date.DATE + Time.T + Time.HOUR_MINUTE), + DATE_HOUR_MINUTE_SECOND(Date.DATE + Time.T + Time.TIME), + DATE_HOUR_MINUTE_SECOND_FRACTION(Date.DATE + Time.T + Time.TIME + Time.MILLIS), + DATE_HOUR_MINUTE_SECOND_MILLIS(Date.DATE + Time.T + Time.TIME + Time.MILLIS), + DATE_TIME(Date.DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), + DATE_TIME_NO_MILLIS(Date.DATE + Time.T + Time.TIME + Time.TZZ), + + HOUR(Time.HOUR), + HOUR_MINUTE(Time.HOUR_MINUTE), + HOUR_MINUTE_SECOND(Time.TIME), + HOUR_MINUTE_SECOND_FRACTION(Time.TIME + Time.MILLIS), + HOUR_MINUTE_SECOND_MILLIS(Time.TIME + Time.MILLIS), + + ORDINAL_DATE(Date.ORDINAL_DATE), + ORDINAL_DATE_TIME(Date.ORDINAL_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), + ORDINAL_DATE_TIME_NO_MILLIS(Date.ORDINAL_DATE + Time.T + Time.TIME + Time.TZZ), + + TIME(Time.TIME + Time.MILLIS + Time.TZZ), + TIME_NO_MILLIS(Time.TIME + Time.TZZ), + + T_TIME(Time.T + Time.TIME + Time.MILLIS + Time.TZZ), + T_TIME_NO_MILLIS(Time.T + Time.TIME + Time.TZZ), + + WEEK_DATE(Date.WEEK_DATE), + WEEK_DATE_TIME(Date.WEEK_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), + WEEK_DATE_TIME_NO_MILLIS(Date.WEEK_DATE + Time.T + Time.TIME + Time.TZZ), + + // Note: input mapping is "weekyear", but output value is "week_year" + WEEK_YEAR(Date.WEEKYEAR), + WEEKYEAR_WEEK(Date.WEEKYEAR_WEEK), + WEEKYEAR_WEEK_DAY(Date.WEEK_DATE), + + YEAR(Date.YEAR), + YEAR_MONTH(Date.YEAR_MONTH), + YEAR_MONTH_DAY(Date.DATE); + + private static class Date { + static String BASIC_DATE = "yyyyMMdd"; + static String BASIC_ORDINAL_DATE = "yyyyDDD"; + static String BASIC_WEEK_DATE = "YYYY'W'wwu"; + + static String DATE = "yyyy-MM-dd"; + static String ORDINAL_DATE = "yyyy-DDD"; + + static String YEAR = "yyyy"; + static String YEAR_MONTH = "yyyy-MM"; + + static String WEEK_DATE = "YYYY-'W'ww-u"; + static String WEEKYEAR = "YYYY"; + static String WEEKYEAR_WEEK = "YYYY-'W'ww"; + } + + private static class Time { + static String T = "'T'"; + static String BASIC_TIME = "HHmmss"; + static String TIME = "HH:mm:ss"; + + static String HOUR = "HH"; + static String HOUR_MINUTE = "HH:mm"; + + static String MILLIS = ".SSS"; + static String TZ = "Z"; + static String TZZ = "XX"; + } + + private String formatString; + + DateFormat(String formatString) + { + this.formatString = formatString; + } + + public String getFormatString() { + return formatString; + } + + public String getFormattedDate(java.util.Date date, String dateFormat) { + return new SimpleDateFormat(dateFormat).format(date); + } + + public String nameLowerCase() + { + return name().toLowerCase(); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java index c533e03582..0aadafe619 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java @@ -30,7 +30,9 @@ import org.json.JSONArray; import org.json.JSONObject; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -42,6 +44,7 @@ public class Protocol { static final int ERROR_STATUS = 500; private final String formatType; + private Map dateFieldFormatMap = new HashMap<>(); private int status; private long size; private long total; @@ -56,6 +59,7 @@ public Protocol(Client client, QueryAction queryAction, Object queryResult, Stri } this.formatType = formatType; QueryStatement query = queryAction.getQueryStatement(); + this.dateFieldFormatMap = queryAction.getDateFieldFormatMap(); this.status = OK_STATUS; this.resultSet = loadResultSet(client, query, queryResult); this.size = resultSet.getDataRows().getSize(); @@ -75,7 +79,7 @@ private ResultSet loadResultSet(Client client, QueryStatement queryStatement, Ob if (queryStatement instanceof Delete) { return new DeleteResultSet(client, (Delete) queryStatement, queryResult); } else if (queryStatement instanceof Query) { - return new SelectResultSet(client, (Query) queryStatement, queryResult); + return new SelectResultSet(client, (Query) queryStatement, queryResult, dateFieldFormatMap); } else if (queryStatement instanceof IndexStatement) { IndexStatement statement = (IndexStatement) queryStatement; StatementType statementType = statement.getStatementType(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index be436b3e3a..a1a30f52f1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -25,6 +25,9 @@ import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMapping; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.client.Client; @@ -39,8 +42,10 @@ import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.Percentiles; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -56,6 +61,8 @@ public class SelectResultSet extends ResultSet { + private static final Logger LOG = LogManager.getLogger(SelectResultSet.class); + public static final String SCORE = "_score"; private Query query; @@ -71,11 +78,14 @@ public class SelectResultSet extends ResultSet { private long totalHits; private List rows; - public SelectResultSet(Client client, Query query, Object queryResult) { + private Map dateFieldFormatMap; + + public SelectResultSet(Client client, Query query, Object queryResult, Map dateFieldFormatMap) { this.client = client; this.query = query; this.queryResult = queryResult; this.selectAll = false; + this.dateFieldFormatMap = dateFieldFormatMap; if (isJoinQuery()) { JoinSelect joinQuery = (JoinSelect) query; @@ -515,6 +525,8 @@ private List populateRows(SearchHits searchHits) { Set newKeys = new HashSet<>(head); for (SearchHit hit : searchHits) { Map rowSource = hit.getSourceAsMap(); + applyClientDateFormat(rowSource); + List result; if (!isJoinQuery()) { @@ -537,6 +549,46 @@ private List populateRows(SearchHits searchHits) { return rows; } + private void applyClientDateFormat(Map rowSource) + { + for (Schema.Column column : columns) { + String columnType = column.getType(); + String columnName = column.getName(); + + if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { + String columnFormat = dateFieldFormatMap.get(columnName); + DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); + + String columnOriginalDate = rowSource.get(columnName).toString(); + Date date = parseDateString(format, columnOriginalDate); + + String clientFormattedDate = format.getFormattedDate(date, "yyyy-MM-dd HH:mm:ss.SSS"); + String dataString = String.format("[YES] %s -> %s", columnOriginalDate, clientFormattedDate); + rowSource.put(columnName, dataString); + } + } + } + + private Date parseDateString(DateFormat format, String columnOriginalDate) + { + try { + switch (format) { + case DATE_OPTIONAL_TIME: + return DateUtils.parseDate(columnOriginalDate, "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd"); + case EPOCH_MILLIS: + return new Date(Long.parseLong(columnOriginalDate)); + case EPOCH_SECOND: + return new Date(Long.parseLong(columnOriginalDate) * 1000); + default: + return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); + } + } + catch (ParseException e) { + LOG.error(String.format("Could not parse date string %s", columnOriginalDate), e); + } + return null; + } + private List populateRows(Aggregations aggregations) { List rows = new ArrayList<>(); List aggs = aggregations.asList(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java index 8c175c943d..c1fb884e73 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java @@ -72,6 +72,7 @@ public class RestSqlAction extends BaseRestHandler { private static final Logger LOG = LogManager.getLogger(RestSqlAction.class); + private static Map dateFieldFormatMap = new HashMap<>(); private final boolean allowExplicitIndex; @@ -150,6 +151,7 @@ private static QueryAction explainRequest(final NodeClient client, final SqlRequ final QueryAction queryAction = new SearchDao(client) .explain(new QueryActionRequest(sqlRequest.getSql(), typeProvider, format)); queryAction.setSqlRequest(sqlRequest); + queryAction.setDateFieldFormatMap(dateFieldFormatMap); return queryAction; } @@ -219,6 +221,9 @@ private static ColumnTypeProvider performAnalysis(String sql) { OpenDistroSqlAnalyzer analyzer = new OpenDistroSqlAnalyzer(config); Optional outputColumnType = analyzer.analyze(sql, clusterState); + + dateFieldFormatMap = clusterState.getDateFieldFormatMap(); + if (outputColumnType.isPresent()) { return new ColumnTypeProvider(outputColumnType.get()); } else { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java index d83031eb25..5050b63876 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -48,6 +49,7 @@ public abstract class QueryAction { protected Query query; protected Client client; protected SqlRequest sqlRequest = SqlRequest.NULL; + private Map dateFieldFormatMap = new HashMap<>(); public QueryAction(Client client, Query query) { this.client = client; @@ -217,4 +219,14 @@ private char[] fromArrayListToCharArray(ArrayList arrayList) { * @throws SqlParseException */ public abstract SqlElasticRequestBuilder explain() throws SqlParseException; + + public Map getDateFieldFormatMap() + { + return dateFieldFormatMap; + } + + public void setDateFieldFormatMap(Map dateFieldFormatMap) + { + this.dateFieldFormatMap = dateFieldFormatMap; + } } From c9c797314f8160d667d9a2e4312647eb5e5d7aca Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 14 Feb 2020 13:25:09 -0800 Subject: [PATCH 02/16] move date format logic to new class --- .../semantic/visitor/ESMappingLoader.java | 12 ++- .../executor/format/DateFieldFormatter.java | 84 +++++++++++++++++++ .../sql/executor/format/SelectResultSet.java | 53 +----------- 3 files changed, 97 insertions(+), 52 deletions(-) create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java index f749988a67..c899b671ec 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java @@ -28,6 +28,8 @@ import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; @@ -112,9 +114,15 @@ private void loadAllFieldsWithType(String indexName) { private void saveDateFormats(FieldMappings mappings) { for (Map.Entry> data : mappings.data().entrySet()) { - String type = data.getValue().get("type").toString(); + String fieldName = data.getKey(); + Object type = data.getValue().get("type"); if ("date".equals(type)) { - clusterState.pushDateFieldFormat(data.getKey(), data.getValue().get("format").toString()); + Object fieldDateFormat = data.getValue().get("format"); + if (fieldDateFormat == null) { + // Default format when not specified is "date_optional_time" + fieldDateFormat = "date_optional_time"; + } + clusterState.pushDateFieldFormat(fieldName, fieldDateFormat.toString()); } } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java new file mode 100644 index 0000000000..2cbab05382 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -0,0 +1,84 @@ +package com.amazon.opendistroforelasticsearch.sql.executor.format; + +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.text.ParseException; +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class DateFieldFormatter +{ + private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); + public static final String JDBC_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + + private final Map dateFieldFormatMap; + private List columns; + + public DateFieldFormatter(Map dateFieldFormatMap, List columns) + { + this.dateFieldFormatMap = dateFieldFormatMap; + this.columns = columns; + } + + public void applyJDBCDateFormat(Map rowSource) { + for (Schema.Column column : columns) { + String columnType = column.getType(); + String columnName = column.getName(); + + LOG.info("{} ({})", columnName, columnType); + if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { + String columnFormat = dateFieldFormatMap.get(columnName); + LOG.info("(CFFF) {} : {}", columnName, columnFormat); + DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); + + LOG.info("(FORMATTING) {} using {}", rowSource.get(columnName), format.getFormatString()); + Object columnOriginalDate = rowSource.get(columnName); + if (columnOriginalDate == null) { + continue; + } + + Date date = parseDateString(format, columnOriginalDate.toString()); + + if (date != null) { + LOG.info("(FORMATTING) {} using {}", columnOriginalDate.toString(), format.getFormatString()); + String clientFormattedDate = format.getFormattedDate(date, JDBC_DATE_FORMAT); + // String dataString = String.format("[YES] %s -> %s", columnOriginalDate.toString(), clientFormattedDate); + String dataString = String.format("%s", clientFormattedDate); + rowSource.put(columnName, dataString); + } else { + LOG.warn("Could not parse date value; returning original value"); + } + } + } + } + + private Date parseDateString(DateFormat format, String columnOriginalDate) + { + try { + switch (format) { + case DATE_OPTIONAL_TIME: + return DateUtils.parseDate( + columnOriginalDate, + "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ssXXX", + "yyyy-MM-dd"); + case EPOCH_MILLIS: + return new Date(Long.parseLong(columnOriginalDate)); + case EPOCH_SECOND: + return new Date(Long.parseLong(columnOriginalDate) * 1000); + default: + return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); + } + } + catch (ParseException e) { + LOG.error(String.format("Error parsing date string %s as %s", columnOriginalDate, format.nameLowerCase()), e); + } + return null; + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index a1a30f52f1..a0e4e66257 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -25,9 +25,6 @@ import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMapping; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; -import org.apache.commons.lang3.time.DateUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.client.Client; @@ -42,10 +39,8 @@ import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.Percentiles; -import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -61,8 +56,6 @@ public class SelectResultSet extends ResultSet { - private static final Logger LOG = LogManager.getLogger(SelectResultSet.class); - public static final String SCORE = "_score"; private Query query; @@ -78,14 +71,13 @@ public class SelectResultSet extends ResultSet { private long totalHits; private List rows; - private Map dateFieldFormatMap; + private DateFieldFormatter dateFieldFormatter; public SelectResultSet(Client client, Query query, Object queryResult, Map dateFieldFormatMap) { this.client = client; this.query = query; this.queryResult = queryResult; this.selectAll = false; - this.dateFieldFormatMap = dateFieldFormatMap; if (isJoinQuery()) { JoinSelect joinQuery = (JoinSelect) query; @@ -96,6 +88,7 @@ public SelectResultSet(Client client, Query query, Object queryResult, Map populateRows(SearchHits searchHits) { Set newKeys = new HashSet<>(head); for (SearchHit hit : searchHits) { Map rowSource = hit.getSourceAsMap(); - applyClientDateFormat(rowSource); + dateFieldFormatter.applyJDBCDateFormat(rowSource); List result; @@ -549,46 +542,6 @@ private List populateRows(SearchHits searchHits) { return rows; } - private void applyClientDateFormat(Map rowSource) - { - for (Schema.Column column : columns) { - String columnType = column.getType(); - String columnName = column.getName(); - - if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { - String columnFormat = dateFieldFormatMap.get(columnName); - DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - - String columnOriginalDate = rowSource.get(columnName).toString(); - Date date = parseDateString(format, columnOriginalDate); - - String clientFormattedDate = format.getFormattedDate(date, "yyyy-MM-dd HH:mm:ss.SSS"); - String dataString = String.format("[YES] %s -> %s", columnOriginalDate, clientFormattedDate); - rowSource.put(columnName, dataString); - } - } - } - - private Date parseDateString(DateFormat format, String columnOriginalDate) - { - try { - switch (format) { - case DATE_OPTIONAL_TIME: - return DateUtils.parseDate(columnOriginalDate, "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd"); - case EPOCH_MILLIS: - return new Date(Long.parseLong(columnOriginalDate)); - case EPOCH_SECOND: - return new Date(Long.parseLong(columnOriginalDate) * 1000); - default: - return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); - } - } - catch (ParseException e) { - LOG.error(String.format("Could not parse date string %s", columnOriginalDate), e); - } - return null; - } - private List populateRows(Aggregations aggregations) { List rows = new ArrayList<>(); List aggs = aggregations.asList(); From 795a0bb91a034adb768c365b2d56a581c6f54b29 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 14 Feb 2020 15:39:18 -0800 Subject: [PATCH 03/16] add tests for DateTimeFormatter --- .../executor/format/DateFieldFormatter.java | 29 +- .../sql/executor/format/DateFormat.java | 8 +- .../format/DateFieldFormatterTest.java | 597 ++++++++++++++++++ 3 files changed, 615 insertions(+), 19 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 2cbab05382..b624c06ec1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -12,7 +12,13 @@ public class DateFieldFormatter { private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); - public static final String JDBC_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; + + private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; + private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; private final Map dateFieldFormatMap; private List columns; @@ -28,26 +34,19 @@ public void applyJDBCDateFormat(Map rowSource) { String columnType = column.getType(); String columnName = column.getName(); - LOG.info("{} ({})", columnName, columnType); if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { String columnFormat = dateFieldFormatMap.get(columnName); - LOG.info("(CFFF) {} : {}", columnName, columnFormat); DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - LOG.info("(FORMATTING) {} using {}", rowSource.get(columnName), format.getFormatString()); Object columnOriginalDate = rowSource.get(columnName); if (columnOriginalDate == null) { + // Don't try to parse null date values continue; } Date date = parseDateString(format, columnOriginalDate.toString()); - if (date != null) { - LOG.info("(FORMATTING) {} using {}", columnOriginalDate.toString(), format.getFormatString()); - String clientFormattedDate = format.getFormattedDate(date, JDBC_DATE_FORMAT); - // String dataString = String.format("[YES] %s -> %s", columnOriginalDate.toString(), clientFormattedDate); - String dataString = String.format("%s", clientFormattedDate); - rowSource.put(columnName, dataString); + rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); } else { LOG.warn("Could not parse date value; returning original value"); } @@ -62,11 +61,11 @@ private Date parseDateString(DateFormat format, String columnOriginalDate) case DATE_OPTIONAL_TIME: return DateUtils.parseDate( columnOriginalDate, - "yyyy-MM-dd'T'HH:mm:ss.SSSZ", - "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", - "yyyy-MM-dd'T'HH:mm:ss", - "yyyy-MM-dd'T'HH:mm:ssXXX", - "yyyy-MM-dd"); + FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); case EPOCH_MILLIS: return new Date(Long.parseLong(columnOriginalDate)); case EPOCH_SECOND: diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java index bd8a45260d..78202d5394 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -105,12 +105,12 @@ public String getFormatString() { return formatString; } - public String getFormattedDate(java.util.Date date, String dateFormat) { - return new SimpleDateFormat(dateFormat).format(date); - } - public String nameLowerCase() { return name().toLowerCase(); } + + public static String getFormattedDate(java.util.Date date, String dateFormat) { + return new SimpleDateFormat(dateFormat).format(date); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java new file mode 100644 index 0000000000..ab31cacabd --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -0,0 +1,597 @@ +package com.amazon.opendistroforelasticsearch.sql.executor.format; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class DateFieldFormatterTest +{ + + @Test + public void testApplyJDBCDateFormat_kibana_sample_data_ecommerce_order_date() + { + String columnName = "order_date"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "2020-02-24T09:28:48+00:00"; + String expectedDateValue = "2020-02-24 01:28:48.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_kibana_sample_data_flights_timestamp() + { + String columnName = "timestamp"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "2020-02-03T00:00:00"; + String expectedDateValue = "2020-02-03 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_kibana_sample_data_logs_utc_date() + { + String columnName = "utc_date"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "2020-02-02T00:39:02.912Z"; + String expectedDateValue = "2020-02-02 00:39:02.912"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_epochMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.EPOCH_MILLIS; + String originalDateValue = "727430805000"; + String expectedDateValue = "1993-01-19 00:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_epochSecond() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.EPOCH_SECOND; + String originalDateValue = "727430805"; + String expectedDateValue = "1993-01-19 00:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateOptionalTime_date() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateOptionalTime_dateAndTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "1993-01-19T00:06:45.123-0800"; + String expectedDateValue = "1993-01-19 00:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicDate() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_DATE; + String originalDateValue = "19930119"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicDateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_DATE_TIME; + String originalDateValue = "19930119T120645.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicDateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_DATE_TIME_NO_MILLIS; + String originalDateValue = "19930119T120645-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicOrdinalDate() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE; + String originalDateValue = "1993019"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicOrdinalDateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME; + String originalDateValue = "1993019T120645.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicOrdinalDateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME_NO_MILLIS; + String originalDateValue = "1993019T120645-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_TIME; + String originalDateValue = "120645.123-0800"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_TIME_NO_MILLIS; + String originalDateValue = "120645-0800"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicTTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_T_TIME; + String originalDateValue = "T120645.123-0800"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicTTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_T_TIME_NO_MILLIS; + String originalDateValue = "T120645-0800"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicWeekDate() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE; + String originalDateValue = "1993W042"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicWeekDateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME; + String originalDateValue = "1993W042T120645.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_basicWeekDateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME_NO_MILLIS; + String originalDateValue = "1993W042T120645-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_date() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateHour() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_HOUR; + String originalDateValue = "1993-01-19T12"; + String expectedDateValue = "1993-01-19 12:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateHourMinute() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE; + String originalDateValue = "1993-01-19T12:06"; + String expectedDateValue = "1993-01-19 12:06:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateHourMinuteSecond() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND; + String originalDateValue = "1993-01-19T12:06:45"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateHourMinuteSecondFraction() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_FRACTION; + String originalDateValue = "1993-01-19T12:06:45.123"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateHourMinuteSecondMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_MILLIS; + String originalDateValue = "1993-01-19T12:06:45.123"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_TIME; + String originalDateValue = "1993-01-19T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_dateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_TIME_NO_MILLIS; + String originalDateValue = "1993-01-19T12:06:45-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_hour() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.HOUR; + String originalDateValue = "12"; + String expectedDateValue = "1970-01-01 12:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_hourMinute() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.HOUR_MINUTE; + String originalDateValue = "12:06"; + String expectedDateValue = "1970-01-01 12:06:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_hourMinuteSecond() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND; + String originalDateValue = "12:06:45"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_hourMinuteSecondFraction() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_FRACTION; + String originalDateValue = "12:06:45.123"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_hourMinuteSecondMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_MILLIS; + String originalDateValue = "12:06:45.123"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + + + @Test + public void testApplyJDBCDateFormat_ordinalDate() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.ORDINAL_DATE; + String originalDateValue = "1993-019"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_ordinalDateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME; + String originalDateValue = "1993-019T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_ordinalDateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME_NO_MILLIS; + String originalDateValue = "1993-019T12:06:45-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_time() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.TIME; + String originalDateValue = "12:06:45.123-0800"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_timeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.TIME_NO_MILLIS; + String originalDateValue = "12:06:45-0800"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_tTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.T_TIME; + String originalDateValue = "T12:06:45.123-0800"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_tTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.T_TIME_NO_MILLIS; + String originalDateValue = "T12:06:45-0800"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_weekDate() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEK_DATE; + String originalDateValue = "1993-W04-2"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_weekDateTime() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEK_DATE_TIME; + String originalDateValue = "1993-W04-2T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_weekDateTimeNoMillis() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEK_DATE_TIME_NO_MILLIS; + String originalDateValue = "1993-W04-2T12:06:45-0800"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + +// "field_weekyear" : "1993", +// "field_weekyear_week" : "1993-W04", +// "field_weekyear_week_day" : "1993-W04-2", +// "field_year" : "1993", +// "field_year_month" : "1993-01", +// "field_year_month_day" : "1993-01-19" + + @Test + public void testApplyJDBCDateFormat_weekyear() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEK_YEAR; + String originalDateValue = "1993"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_weekyearWeek() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK; + String originalDateValue = "1993-W04"; + String expectedDateValue = "1993-01-17 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_weekyearWeekDay() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK_DAY; + String originalDateValue = "1993-W04-2"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_year() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.YEAR; + String originalDateValue = "1993"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_yearMonth() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.YEAR_MONTH; + String originalDateValue = "1993-01"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_yearMonthDay() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.YEAR_MONTH_DAY; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + private void verifyFormatting(String columnName, DateFormat dateFormat, String originalDateValue, String expectedDateValue) + { + List columns = buildColumnList(columnName); + Map dateFieldFormatMap = buildDateFieldFormatMap(columnName, dateFormat); + + Map rowSource = new HashMap<>(); + rowSource.put(columnName, originalDateValue); + + DateFieldFormatter dateFieldFormatter = new DateFieldFormatter(dateFieldFormatMap, columns); + executeFormattingAndCompare(dateFieldFormatter, rowSource, columnName, expectedDateValue); + } + + private void executeFormattingAndCompare( + DateFieldFormatter formatter, + Map rowSource, + String columnToCheck, + String expectedDateValue) { + formatter.applyJDBCDateFormat(rowSource); + assertEquals(expectedDateValue, rowSource.get(columnToCheck)); + } + + private List buildColumnList(String columnName) { + return ImmutableList.builder() + .add(new Schema.Column(columnName, null, Schema.Type.DATE)) + .build(); + } + + private Map buildDateFieldFormatMap(String columnName, DateFormat dateFormat) { + return ImmutableMap.builder() + .put(columnName, dateFormat.nameLowerCase()) + .build(); + } + +} \ No newline at end of file From 1afa19283cc0cfc57e50a398df1fe1b278ef911d Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 14 Feb 2020 15:50:29 -0800 Subject: [PATCH 04/16] added some negative tests --- .../format/DateFieldFormatterTest.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index ab31cacabd..0c2ec1cd7a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -376,8 +376,6 @@ public void testApplyJDBCDateFormat_hourMinuteSecondMillis() verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } - - @Test public void testApplyJDBCDateFormat_ordinalDate() { @@ -488,13 +486,6 @@ public void testApplyJDBCDateFormat_weekDateTimeNoMillis() verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } -// "field_weekyear" : "1993", -// "field_weekyear_week" : "1993-W04", -// "field_weekyear_week_day" : "1993-W04-2", -// "field_year" : "1993", -// "field_year_month" : "1993-01", -// "field_year_month_day" : "1993-01-19" - @Test public void testApplyJDBCDateFormat_weekyear() { @@ -561,6 +552,30 @@ public void testApplyJDBCDateFormat_yearMonthDay() verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } + @Test + public void testApplyJDBCDateFormat_incorrectFormat() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = "1581724085"; + // Invalid format for date value; should return original value + String expectedDateValue = "1581724085"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testApplyJDBCDateFormat_nullDateData() + { + String columnName = "date_field"; + DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; + String originalDateValue = null; + // Nulls should be preserved + String expectedDateValue = null; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + private void verifyFormatting(String columnName, DateFormat dateFormat, String originalDateValue, String expectedDateValue) { List columns = buildColumnList(columnName); From b220ce4dd17c06ec1841924fb91e1d6b01fe91bc Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 14 Feb 2020 16:18:58 -0800 Subject: [PATCH 05/16] style fixes --- .../semantic/visitor/ESMappingLoader.java | 5 +- .../sql/esdomain/LocalClusterState.java | 6 +- .../executor/format/DateFieldFormatter.java | 131 ++++++++++-------- .../sql/executor/format/DateFormat.java | 24 +++- .../sql/query/QueryAction.java | 6 +- .../format/DateFieldFormatterTest.java | 15 ++ 6 files changed, 110 insertions(+), 77 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java index c899b671ec..40da373b42 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java @@ -28,8 +28,6 @@ import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.Map; @@ -111,8 +109,7 @@ private void loadAllFieldsWithType(String indexName) { mappings.flat(this::defineFieldName); } - private void saveDateFormats(FieldMappings mappings) - { + private void saveDateFormats(FieldMappings mappings) { for (Map.Entry> data : mappings.data().entrySet()) { String fieldName = data.getKey(); Object type = data.getValue().get("type"); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index 68d760ea55..dd97944510 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -244,13 +244,11 @@ private List sortToList(T[] array) { return Arrays.asList(array); } - public void pushDateFieldFormat(String fieldName, String dateFormat) - { + public void pushDateFieldFormat(String fieldName, String dateFormat) { dateFieldFormatMap.put(fieldName, dateFormat); } - public Map getDateFieldFormatMap() - { + public Map getDateFieldFormatMap() { return dateFieldFormatMap; } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index b624c06ec1..0a7c050e71 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.sql.executor.format; import org.apache.commons.lang3.time.DateUtils; @@ -9,75 +24,73 @@ import java.util.List; import java.util.Map; -public class DateFieldFormatter -{ - private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); - private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; +public class DateFieldFormatter { + private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); + private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; - private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; - private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; - private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; - private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; - private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; + private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; + private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; - private final Map dateFieldFormatMap; - private List columns; + private final Map dateFieldFormatMap; + private List columns; - public DateFieldFormatter(Map dateFieldFormatMap, List columns) - { - this.dateFieldFormatMap = dateFieldFormatMap; - this.columns = columns; - } + public DateFieldFormatter(Map dateFieldFormatMap, List columns) { + this.dateFieldFormatMap = dateFieldFormatMap; + this.columns = columns; + } - public void applyJDBCDateFormat(Map rowSource) { - for (Schema.Column column : columns) { - String columnType = column.getType(); - String columnName = column.getName(); + public void applyJDBCDateFormat(Map rowSource) { + for (Schema.Column column : columns) { + String columnType = column.getType(); + String columnName = column.getName(); - if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { - String columnFormat = dateFieldFormatMap.get(columnName); - DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); + if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { + String columnFormat = dateFieldFormatMap.get(columnName); + DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - Object columnOriginalDate = rowSource.get(columnName); - if (columnOriginalDate == null) { - // Don't try to parse null date values - continue; - } + Object columnOriginalDate = rowSource.get(columnName); + if (columnOriginalDate == null) { + // Don't try to parse null date values + continue; + } - Date date = parseDateString(format, columnOriginalDate.toString()); - if (date != null) { - rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); - } else { - LOG.warn("Could not parse date value; returning original value"); + Date date = parseDateString(format, columnOriginalDate.toString()); + if (date != null) { + rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); + } else { + LOG.warn("Could not parse date value; returning original value"); + } + } } - } } - } - private Date parseDateString(DateFormat format, String columnOriginalDate) - { - try { - switch (format) { - case DATE_OPTIONAL_TIME: - return DateUtils.parseDate( - columnOriginalDate, - FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, - FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, - FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, - FORMAT_DOT_DATE_AND_TIME, - FORMAT_DOT_DATE); - case EPOCH_MILLIS: - return new Date(Long.parseLong(columnOriginalDate)); - case EPOCH_SECOND: - return new Date(Long.parseLong(columnOriginalDate) * 1000); - default: - return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); - } - } - catch (ParseException e) { - LOG.error(String.format("Error parsing date string %s as %s", columnOriginalDate, format.nameLowerCase()), e); + private Date parseDateString(DateFormat format, String columnOriginalDate) { + try { + switch (format) { + case DATE_OPTIONAL_TIME: + return DateUtils.parseDate( + columnOriginalDate, + FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); + case EPOCH_MILLIS: + return new Date(Long.parseLong(columnOriginalDate)); + case EPOCH_SECOND: + return new Date(Long.parseLong(columnOriginalDate) * 1000); + default: + return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); + } + } catch (ParseException e) { + LOG.error( + String.format("Error parsing date string %s as %s", columnOriginalDate, format.nameLowerCase()), + e); + } + return null; } - return null; - } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java index 78202d5394..730c272993 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -1,9 +1,23 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.sql.executor.format; import java.text.SimpleDateFormat; -public enum DateFormat -{ +public enum DateFormat { // Special cases that are parsed separately DATE_OPTIONAL_TIME(""), EPOCH_MILLIS(""), @@ -96,8 +110,7 @@ private static class Time { private String formatString; - DateFormat(String formatString) - { + DateFormat(String formatString) { this.formatString = formatString; } @@ -105,8 +118,7 @@ public String getFormatString() { return formatString; } - public String nameLowerCase() - { + public String nameLowerCase() { return name().toLowerCase(); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java index 5050b63876..91c9d27a96 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java @@ -220,13 +220,11 @@ private char[] fromArrayListToCharArray(ArrayList arrayList) { */ public abstract SqlElasticRequestBuilder explain() throws SqlParseException; - public Map getDateFieldFormatMap() - { + public Map getDateFieldFormatMap() { return dateFieldFormatMap; } - public void setDateFieldFormatMap(Map dateFieldFormatMap) - { + public void setDateFieldFormatMap(Map dateFieldFormatMap) { this.dateFieldFormatMap = dateFieldFormatMap; } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index 0c2ec1cd7a..fb0fa20f5f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.sql.executor.format; import com.google.common.collect.ImmutableList; From 2ae4195a41419305c44483944bae841795d59393 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Tue, 18 Feb 2020 11:35:20 -0800 Subject: [PATCH 06/16] testing locale fixes --- .../sql/executor/format/DateFormat.java | 18 ++++++- .../format/DateFieldFormatterTest.java | 48 +++++++++---------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java index 730c272993..2f9689c252 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -15,7 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.executor.format; -import java.text.SimpleDateFormat; +// import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +// import java.util.TimeZone; public enum DateFormat { // Special cases that are parsed separately @@ -123,6 +128,15 @@ public String nameLowerCase() { } public static String getFormattedDate(java.util.Date date, String dateFormat) { - return new SimpleDateFormat(dateFormat).format(date); + Instant instant = date.toInstant(); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("Etc/UTC")); + // return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); + return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); + // return DateTimeFormatter.ofLocalizedDateTime().format(date.toInstant()); + // SimpleDateFormat formatter = new SimpleDateFormat(dateFormat); + // formatter.applyLocalizedPattern(dateFormat); + // formatter.setTimeZone(TimeZone.getTimeZone("UTC")); + // return formatter.format(date); + // formatter. } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index fb0fa20f5f..d64c4cd2de 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -34,7 +34,7 @@ public void testApplyJDBCDateFormat_kibana_sample_data_ecommerce_order_date() String columnName = "order_date"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; String originalDateValue = "2020-02-24T09:28:48+00:00"; - String expectedDateValue = "2020-02-24 01:28:48.000"; + String expectedDateValue = "2020-02-24 09:28:48.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -67,7 +67,7 @@ public void testApplyJDBCDateFormat_epochMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.EPOCH_MILLIS; String originalDateValue = "727430805000"; - String expectedDateValue = "1993-01-19 00:06:45.000"; + String expectedDateValue = "1993-01-19 08:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -78,7 +78,7 @@ public void testApplyJDBCDateFormat_epochSecond() String columnName = "date_field"; DateFormat dateFormat = DateFormat.EPOCH_SECOND; String originalDateValue = "727430805"; - String expectedDateValue = "1993-01-19 00:06:45.000"; + String expectedDateValue = "1993-01-19 08:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -100,7 +100,7 @@ public void testApplyJDBCDateFormat_dateOptionalTime_dateAndTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; String originalDateValue = "1993-01-19T00:06:45.123-0800"; - String expectedDateValue = "1993-01-19 00:06:45.123"; + String expectedDateValue = "1993-01-19 08:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -122,7 +122,7 @@ public void testApplyJDBCDateFormat_basicDateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_DATE_TIME; String originalDateValue = "19930119T120645.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -133,7 +133,7 @@ public void testApplyJDBCDateFormat_basicDateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_DATE_TIME_NO_MILLIS; String originalDateValue = "19930119T120645-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -155,7 +155,7 @@ public void testApplyJDBCDateFormat_basicOrdinalDateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME; String originalDateValue = "1993019T120645.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -166,7 +166,7 @@ public void testApplyJDBCDateFormat_basicOrdinalDateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME_NO_MILLIS; String originalDateValue = "1993019T120645-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -177,7 +177,7 @@ public void testApplyJDBCDateFormat_basicTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_TIME; String originalDateValue = "120645.123-0800"; - String expectedDateValue = "1970-01-01 12:06:45.123"; + String expectedDateValue = "1970-01-01 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -188,7 +188,7 @@ public void testApplyJDBCDateFormat_basicTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_TIME_NO_MILLIS; String originalDateValue = "120645-0800"; - String expectedDateValue = "1970-01-01 12:06:45.000"; + String expectedDateValue = "1970-01-01 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -199,7 +199,7 @@ public void testApplyJDBCDateFormat_basicTTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_T_TIME; String originalDateValue = "T120645.123-0800"; - String expectedDateValue = "1970-01-01 12:06:45.123"; + String expectedDateValue = "1970-01-01 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -210,7 +210,7 @@ public void testApplyJDBCDateFormat_basicTTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_T_TIME_NO_MILLIS; String originalDateValue = "T120645-0800"; - String expectedDateValue = "1970-01-01 12:06:45.000"; + String expectedDateValue = "1970-01-01 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -232,7 +232,7 @@ public void testApplyJDBCDateFormat_basicWeekDateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME; String originalDateValue = "1993W042T120645.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -243,7 +243,7 @@ public void testApplyJDBCDateFormat_basicWeekDateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME_NO_MILLIS; String originalDateValue = "1993W042T120645-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -320,7 +320,7 @@ public void testApplyJDBCDateFormat_dateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_TIME; String originalDateValue = "1993-01-19T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -331,7 +331,7 @@ public void testApplyJDBCDateFormat_dateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_TIME_NO_MILLIS; String originalDateValue = "1993-01-19T12:06:45-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -408,7 +408,7 @@ public void testApplyJDBCDateFormat_ordinalDateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME; String originalDateValue = "1993-019T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -419,7 +419,7 @@ public void testApplyJDBCDateFormat_ordinalDateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME_NO_MILLIS; String originalDateValue = "1993-019T12:06:45-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -430,7 +430,7 @@ public void testApplyJDBCDateFormat_time() String columnName = "date_field"; DateFormat dateFormat = DateFormat.TIME; String originalDateValue = "12:06:45.123-0800"; - String expectedDateValue = "1970-01-01 12:06:45.123"; + String expectedDateValue = "1970-01-01 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -441,7 +441,7 @@ public void testApplyJDBCDateFormat_timeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.TIME_NO_MILLIS; String originalDateValue = "12:06:45-0800"; - String expectedDateValue = "1970-01-01 12:06:45.000"; + String expectedDateValue = "1970-01-01 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -452,7 +452,7 @@ public void testApplyJDBCDateFormat_tTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.T_TIME; String originalDateValue = "T12:06:45.123-0800"; - String expectedDateValue = "1970-01-01 12:06:45.123"; + String expectedDateValue = "1970-01-01 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -463,7 +463,7 @@ public void testApplyJDBCDateFormat_tTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.T_TIME_NO_MILLIS; String originalDateValue = "T12:06:45-0800"; - String expectedDateValue = "1970-01-01 12:06:45.000"; + String expectedDateValue = "1970-01-01 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -485,7 +485,7 @@ public void testApplyJDBCDateFormat_weekDateTime() String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_DATE_TIME; String originalDateValue = "1993-W04-2T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 12:06:45.123"; + String expectedDateValue = "1993-01-19 20:06:45.123"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -496,7 +496,7 @@ public void testApplyJDBCDateFormat_weekDateTimeNoMillis() String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_DATE_TIME_NO_MILLIS; String originalDateValue = "1993-W04-2T12:06:45-0800"; - String expectedDateValue = "1993-01-19 12:06:45.000"; + String expectedDateValue = "1993-01-19 20:06:45.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } From 428f18970e40190eed673188b99f41a613730bc8 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 21 Feb 2020 14:13:19 -0800 Subject: [PATCH 07/16] addressed code review comments and build failures --- .../semantic/visitor/ESMappingLoader.java | 16 --- .../sql/esdomain/LocalClusterState.java | 17 +-- .../executor/format/DateFieldFormatter.java | 103 ++++++++++++++--- .../sql/executor/format/DateFormat.java | 9 -- .../sql/executor/format/Protocol.java | 6 +- .../sql/executor/format/SelectResultSet.java | 18 ++- .../sql/parser/FieldMaker.java | 2 + .../sql/plugin/RestSqlAction.java | 5 - .../sql/query/QueryAction.java | 10 -- .../sql/esintgtest/SQLFunctionsIT.java | 12 +- .../format/DateFieldFormatterTest.java | 106 +++++++++--------- 11 files changed, 169 insertions(+), 135 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java index 40da373b42..34c1ebc801 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java @@ -105,25 +105,9 @@ private void defineIndexType(String indexName) { private void loadAllFieldsWithType(String indexName) { FieldMappings mappings = getFieldMappings(indexName); - saveDateFormats(mappings); mappings.flat(this::defineFieldName); } - private void saveDateFormats(FieldMappings mappings) { - for (Map.Entry> data : mappings.data().entrySet()) { - String fieldName = data.getKey(); - Object type = data.getValue().get("type"); - if ("date".equals(type)) { - Object fieldDateFormat = data.getValue().get("format"); - if (fieldDateFormat == null) { - // Default format when not specified is "date_optional_time" - fieldDateFormat = "date_optional_time"; - } - clusterState.pushDateFieldFormat(fieldName, fieldDateFormat.toString()); - } - } - } - /* * 3.1 Define with alias if given: ex."SELECT * FROM accounts a". * 'accounts' -> INDEX diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index dd97944510..acf5a5d799 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -92,7 +92,15 @@ public class LocalClusterState { */ private final Map latestSettings = new ConcurrentHashMap<>(); - private Map dateFieldFormatMap = new HashMap<>(); + private Map possibleAliasMap = new HashMap<>(); + + public void putAliasInMap(String field, String alias) { + possibleAliasMap.put(alias, field); + } + + public Map getAliasMap() { + return possibleAliasMap; + } public static synchronized LocalClusterState state() { if (INSTANCE == null) { @@ -244,11 +252,4 @@ private List sortToList(T[] array) { return Arrays.asList(array); } - public void pushDateFieldFormat(String fieldName, String dateFormat) { - dateFieldFormatMap.put(fieldName, dateFormat); - } - - public Map getDateFieldFormatMap() { - return dateFieldFormatMap; - } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 0a7c050e71..18a84c69f1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -15,15 +15,25 @@ package com.amazon.opendistroforelasticsearch.sql.executor.format; +import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.TypeMappings; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.time.DateUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.text.ParseException; +import java.util.Collection; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +/** + * Formatter to transform date fields into a consistent format for consumption by clients. + */ public class DateFieldFormatter { private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; @@ -35,36 +45,94 @@ public class DateFieldFormatter { private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; private final Map dateFieldFormatMap; - private List columns; + private List dateColumns; + + public DateFieldFormatter(String indexName, List columns) { + this.dateFieldFormatMap = getDateFieldFormatMap(indexName); + this.dateColumns = getDateColumns(columns); + } - public DateFieldFormatter(Map dateFieldFormatMap, List columns) { + @VisibleForTesting + protected DateFieldFormatter(Map dateFieldFormatMap, List columns) { this.dateFieldFormatMap = dateFieldFormatMap; - this.columns = columns; + this.dateColumns = getDateColumns(columns); } + /** + * Apply the JDBC date format ({@code yyyy-MM-dd HH:mm:ss.SSS}) to date values in the current row. + * + * @param rowSource The row in which to format the date values. + */ public void applyJDBCDateFormat(Map rowSource) { - for (Schema.Column column : columns) { - String columnType = column.getType(); + for (Schema.Column column : dateColumns) { String columnName = column.getName(); - if (columnType.equals(Schema.Type.DATE.nameLowerCase())) { - String columnFormat = dateFieldFormatMap.get(columnName); - DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); + String columnFormat; + columnFormat = getFormatForColumn(columnName); + if (columnFormat == null) { + LOG.warn("Could not determine date format for column {}; returning original value", columnName); + continue; + } + DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - Object columnOriginalDate = rowSource.get(columnName); - if (columnOriginalDate == null) { - // Don't try to parse null date values - continue; - } + Object columnOriginalDate = rowSource.get(columnName); + if (columnOriginalDate == null) { + // Don't try to parse null date values + continue; + } + + Date date = parseDateString(format, columnOriginalDate.toString()); + if (date != null) { + rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); + } else { + LOG.warn("Could not parse date value; returning original value"); + } + } + } - Date date = parseDateString(format, columnOriginalDate.toString()); - if (date != null) { - rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); + private String getFormatForColumn(String columnName) { + // Handle special cases for column names + if (columnName.startsWith("cast_")) { + // Column was cast to a date type, and prefixed with "cast_" + columnName = columnName.replaceFirst("cast_", ""); + } else if (columnName.split("\\.").length == 2) { + // Column is part of a join, and is qualified by the table alias + columnName = columnName.split("\\.")[1]; + } + return dateFieldFormatMap.get(columnName); + } + + private List getDateColumns(List columns) { + return columns.stream() + .filter(column -> column.getType().equals(Schema.Type.DATE.nameLowerCase())) + .collect(Collectors.toList()); + } + + private Map getDateFieldFormatMap(String indexName) { + LocalClusterState state = LocalClusterState.state(); + Map formatMap = new HashMap<>(); + + String[] indices = indexName.split("\\|"); + Collection typeProperties = state.getFieldMappings(indices) + .allMappings(); + + for (TypeMappings mappings: typeProperties) { + FieldMappings fieldMappings = mappings.firstMapping(); + for (Map.Entry> field : fieldMappings.data().entrySet()) { + String fieldName = field.getKey(); + Map properties = field.getValue(); + + if (properties.containsKey("format")) { + formatMap.put(fieldName, properties.get("format").toString()); } else { - LOG.warn("Could not parse date value; returning original value"); + // Give all field types a format, since operations such as casts + // can change the output type for a field to `date`. + formatMap.put(fieldName, "date_optional_time"); } } } + + return formatMap; } private Date parseDateString(DateFormat format, String columnOriginalDate) { @@ -92,5 +160,4 @@ private Date parseDateString(DateFormat format, String columnOriginalDate) { } return null; } - } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java index 2f9689c252..c605d8cde5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -15,12 +15,10 @@ package com.amazon.opendistroforelasticsearch.sql.executor.format; -// import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -// import java.util.TimeZone; public enum DateFormat { // Special cases that are parsed separately @@ -130,13 +128,6 @@ public String nameLowerCase() { public static String getFormattedDate(java.util.Date date, String dateFormat) { Instant instant = date.toInstant(); ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("Etc/UTC")); - // return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); - // return DateTimeFormatter.ofLocalizedDateTime().format(date.toInstant()); - // SimpleDateFormat formatter = new SimpleDateFormat(dateFormat); - // formatter.applyLocalizedPattern(dateFormat); - // formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - // return formatter.format(date); - // formatter. } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java index 0aadafe619..c2238e2d9d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java @@ -30,9 +30,7 @@ import org.json.JSONArray; import org.json.JSONObject; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -44,7 +42,6 @@ public class Protocol { static final int ERROR_STATUS = 500; private final String formatType; - private Map dateFieldFormatMap = new HashMap<>(); private int status; private long size; private long total; @@ -59,7 +56,6 @@ public Protocol(Client client, QueryAction queryAction, Object queryResult, Stri } this.formatType = formatType; QueryStatement query = queryAction.getQueryStatement(); - this.dateFieldFormatMap = queryAction.getDateFieldFormatMap(); this.status = OK_STATUS; this.resultSet = loadResultSet(client, query, queryResult); this.size = resultSet.getDataRows().getSize(); @@ -79,7 +75,7 @@ private ResultSet loadResultSet(Client client, QueryStatement queryStatement, Ob if (queryStatement instanceof Delete) { return new DeleteResultSet(client, (Delete) queryStatement, queryResult); } else if (queryStatement instanceof Query) { - return new SelectResultSet(client, (Query) queryStatement, queryResult, dateFieldFormatMap); + return new SelectResultSet(client, (Query) queryStatement, queryResult, formatType); } else if (queryStatement instanceof IndexStatement) { IndexStatement statement = (IndexStatement) queryStatement; StatementType statementType = statement.getStatementType(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index a0e4e66257..85952f4488 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -24,6 +24,7 @@ import com.amazon.opendistroforelasticsearch.sql.domain.TableOnJoinSelect; import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMapping; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.executor.Format; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; @@ -57,6 +58,7 @@ public class SelectResultSet extends ResultSet { public static final String SCORE = "_score"; + private final String formatType; private Query query; private Object queryResult; @@ -73,11 +75,15 @@ public class SelectResultSet extends ResultSet { private DateFieldFormatter dateFieldFormatter; - public SelectResultSet(Client client, Query query, Object queryResult, Map dateFieldFormatMap) { + public SelectResultSet(Client client, + Query query, + Object queryResult, + String formatType) { this.client = client; this.query = query; this.queryResult = queryResult; this.selectAll = false; + this.formatType = formatType; if (isJoinQuery()) { JoinSelect joinQuery = (JoinSelect) query; @@ -88,7 +94,7 @@ public SelectResultSet(Client client, Query query, Object queryResult, Map populateRows(SearchHits searchHits) { Set newKeys = new HashSet<>(head); for (SearchHit hit : searchHits) { Map rowSource = hit.getSourceAsMap(); - dateFieldFormatter.applyJDBCDateFormat(rowSource); - List result; if (!isJoinQuery()) { @@ -530,8 +534,14 @@ private List populateRows(SearchHits searchHits) { for (Map.Entry field : hit.getFields().entrySet()) { rowSource.put(field.getKey(), field.getValue().getValue()); } + if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { + dateFieldFormatter.applyJDBCDateFormat(rowSource); + } result = flatNestedField(newKeys, rowSource, hit.getInnerHits()); } else { + if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { + dateFieldFormatter.applyJDBCDateFormat(rowSource); + } result = new ArrayList<>(); result.add(new DataRows.Row(rowSource)); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java index de2020daa6..298b62d18c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java @@ -39,6 +39,7 @@ import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Where; +// import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; @@ -118,6 +119,7 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw if (alias == null) { alias = "cast_" + castExpr.getExpr().toString(); } + // LocalClusterState.state().putAliasInMap(castExpr.getExpr().toString(), alias); ArrayList methodParameters = new ArrayList<>(); methodParameters.add(((SQLCastExpr) expr).getExpr()); return makeMethodField("CAST", methodParameters, null, alias, tableAlias, true); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java index c1fb884e73..8c175c943d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java @@ -72,7 +72,6 @@ public class RestSqlAction extends BaseRestHandler { private static final Logger LOG = LogManager.getLogger(RestSqlAction.class); - private static Map dateFieldFormatMap = new HashMap<>(); private final boolean allowExplicitIndex; @@ -151,7 +150,6 @@ private static QueryAction explainRequest(final NodeClient client, final SqlRequ final QueryAction queryAction = new SearchDao(client) .explain(new QueryActionRequest(sqlRequest.getSql(), typeProvider, format)); queryAction.setSqlRequest(sqlRequest); - queryAction.setDateFieldFormatMap(dateFieldFormatMap); return queryAction; } @@ -221,9 +219,6 @@ private static ColumnTypeProvider performAnalysis(String sql) { OpenDistroSqlAnalyzer analyzer = new OpenDistroSqlAnalyzer(config); Optional outputColumnType = analyzer.analyze(sql, clusterState); - - dateFieldFormatMap = clusterState.getDateFieldFormatMap(); - if (outputColumnType.isPresent()) { return new ColumnTypeProvider(outputColumnType.get()); } else { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java index 91c9d27a96..d83031eb25 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -49,7 +48,6 @@ public abstract class QueryAction { protected Query query; protected Client client; protected SqlRequest sqlRequest = SqlRequest.NULL; - private Map dateFieldFormatMap = new HashMap<>(); public QueryAction(Client client, Query query) { this.client = client; @@ -219,12 +217,4 @@ private char[] fromArrayListToCharArray(ArrayList arrayList) { * @throws SqlParseException */ public abstract SqlElasticRequestBuilder explain() throws SqlParseException; - - public Map getDateFieldFormatMap() { - return dateFieldFormatMap; - } - - public void setDateFieldFormatMap(Map dateFieldFormatMap) { - this.dateFieldFormatMap = dateFieldFormatMap; - } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java index 80e61cb381..d639385527 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java @@ -331,8 +331,8 @@ public void castKeywordFieldToDatetimeWithoutAliasJdbcFormatTest() { verifySchema(response, schema("cast_date_keyword", null, "date")); verifyDataRows(response, - rows("2014-08-19T07:09:13.434Z"), - rows("2019-09-25T02:04:13.469Z")); + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); } @Test @@ -343,8 +343,8 @@ public void castKeywordFieldToDatetimeWithAliasJdbcFormatTest() { verifySchema(response, schema("test_alias", null, "date")); verifyDataRows(response, - rows("2014-08-19T07:09:13.434Z"), - rows("2019-09-25T02:04:13.469Z")); + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); } @Test @@ -355,8 +355,8 @@ public void castFieldToDatetimeWithWhereClauseJdbcFormatTest() { verifySchema(response, schema("cast_date_keyword", null, "date")); verifyDataRows(response, - rows("2014-08-19T07:09:13.434Z"), - rows("2019-09-25T02:04:13.469Z")); + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index d64c4cd2de..43d9f4219b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -25,11 +25,10 @@ import static org.junit.Assert.*; -public class DateFieldFormatterTest -{ +public class DateFieldFormatterTest { @Test - public void testApplyJDBCDateFormat_kibana_sample_data_ecommerce_order_date() + public void testKibanaSampleDataEcommerceOrderDateField() { String columnName = "order_date"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -40,7 +39,7 @@ public void testApplyJDBCDateFormat_kibana_sample_data_ecommerce_order_date() } @Test - public void testApplyJDBCDateFormat_kibana_sample_data_flights_timestamp() + public void testKibanaSampleDataFlightsTimestampField() { String columnName = "timestamp"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -51,7 +50,7 @@ public void testApplyJDBCDateFormat_kibana_sample_data_flights_timestamp() } @Test - public void testApplyJDBCDateFormat_kibana_sample_data_logs_utc_date() + public void testKibanaSampleDataLogsUtcDateField() { String columnName = "utc_date"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -62,7 +61,7 @@ public void testApplyJDBCDateFormat_kibana_sample_data_logs_utc_date() } @Test - public void testApplyJDBCDateFormat_epochMillis() + public void testEpochMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.EPOCH_MILLIS; @@ -73,7 +72,7 @@ public void testApplyJDBCDateFormat_epochMillis() } @Test - public void testApplyJDBCDateFormat_epochSecond() + public void testEpochSecond() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.EPOCH_SECOND; @@ -84,7 +83,7 @@ public void testApplyJDBCDateFormat_epochSecond() } @Test - public void testApplyJDBCDateFormat_dateOptionalTime_date() + public void testDateOptionalTimeDateOnly() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -95,7 +94,7 @@ public void testApplyJDBCDateFormat_dateOptionalTime_date() } @Test - public void testApplyJDBCDateFormat_dateOptionalTime_dateAndTime() + public void testDateOptionalTimeDateAndTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -106,7 +105,7 @@ public void testApplyJDBCDateFormat_dateOptionalTime_dateAndTime() } @Test - public void testApplyJDBCDateFormat_basicDate() + public void testBasicDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_DATE; @@ -117,7 +116,7 @@ public void testApplyJDBCDateFormat_basicDate() } @Test - public void testApplyJDBCDateFormat_basicDateTime() + public void testBasicDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_DATE_TIME; @@ -128,7 +127,7 @@ public void testApplyJDBCDateFormat_basicDateTime() } @Test - public void testApplyJDBCDateFormat_basicDateTimeNoMillis() + public void testBasicDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_DATE_TIME_NO_MILLIS; @@ -139,7 +138,7 @@ public void testApplyJDBCDateFormat_basicDateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_basicOrdinalDate() + public void testBasicOrdinalDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE; @@ -150,7 +149,7 @@ public void testApplyJDBCDateFormat_basicOrdinalDate() } @Test - public void testApplyJDBCDateFormat_basicOrdinalDateTime() + public void testBasicOrdinalDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME; @@ -161,7 +160,7 @@ public void testApplyJDBCDateFormat_basicOrdinalDateTime() } @Test - public void testApplyJDBCDateFormat_basicOrdinalDateTimeNoMillis() + public void testBasicOrdinalDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME_NO_MILLIS; @@ -172,7 +171,7 @@ public void testApplyJDBCDateFormat_basicOrdinalDateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_basicTime() + public void testBasicTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_TIME; @@ -183,7 +182,7 @@ public void testApplyJDBCDateFormat_basicTime() } @Test - public void testApplyJDBCDateFormat_basicTimeNoMillis() + public void testBasicTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_TIME_NO_MILLIS; @@ -194,7 +193,7 @@ public void testApplyJDBCDateFormat_basicTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_basicTTime() + public void testBasicTTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_T_TIME; @@ -205,7 +204,7 @@ public void testApplyJDBCDateFormat_basicTTime() } @Test - public void testApplyJDBCDateFormat_basicTTimeNoMillis() + public void testBasicTTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_T_TIME_NO_MILLIS; @@ -216,7 +215,7 @@ public void testApplyJDBCDateFormat_basicTTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_basicWeekDate() + public void testBasicWeekDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE; @@ -227,7 +226,7 @@ public void testApplyJDBCDateFormat_basicWeekDate() } @Test - public void testApplyJDBCDateFormat_basicWeekDateTime() + public void testBasicWeekDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME; @@ -238,7 +237,7 @@ public void testApplyJDBCDateFormat_basicWeekDateTime() } @Test - public void testApplyJDBCDateFormat_basicWeekDateTimeNoMillis() + public void testBasicWeekDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME_NO_MILLIS; @@ -249,7 +248,7 @@ public void testApplyJDBCDateFormat_basicWeekDateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_date() + public void testDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE; @@ -260,7 +259,7 @@ public void testApplyJDBCDateFormat_date() } @Test - public void testApplyJDBCDateFormat_dateHour() + public void testDateHour() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_HOUR; @@ -271,7 +270,7 @@ public void testApplyJDBCDateFormat_dateHour() } @Test - public void testApplyJDBCDateFormat_dateHourMinute() + public void testDateHourMinute() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE; @@ -282,7 +281,7 @@ public void testApplyJDBCDateFormat_dateHourMinute() } @Test - public void testApplyJDBCDateFormat_dateHourMinuteSecond() + public void testDateHourMinuteSecond() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND; @@ -293,7 +292,7 @@ public void testApplyJDBCDateFormat_dateHourMinuteSecond() } @Test - public void testApplyJDBCDateFormat_dateHourMinuteSecondFraction() + public void testDateHourMinuteSecondFraction() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_FRACTION; @@ -304,7 +303,7 @@ public void testApplyJDBCDateFormat_dateHourMinuteSecondFraction() } @Test - public void testApplyJDBCDateFormat_dateHourMinuteSecondMillis() + public void testDateHourMinuteSecondMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_MILLIS; @@ -315,7 +314,7 @@ public void testApplyJDBCDateFormat_dateHourMinuteSecondMillis() } @Test - public void testApplyJDBCDateFormat_dateTime() + public void testDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_TIME; @@ -326,7 +325,7 @@ public void testApplyJDBCDateFormat_dateTime() } @Test - public void testApplyJDBCDateFormat_dateTimeNoMillis() + public void testDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_TIME_NO_MILLIS; @@ -337,7 +336,7 @@ public void testApplyJDBCDateFormat_dateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_hour() + public void testHour() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.HOUR; @@ -348,7 +347,7 @@ public void testApplyJDBCDateFormat_hour() } @Test - public void testApplyJDBCDateFormat_hourMinute() + public void testHourMinute() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.HOUR_MINUTE; @@ -359,7 +358,7 @@ public void testApplyJDBCDateFormat_hourMinute() } @Test - public void testApplyJDBCDateFormat_hourMinuteSecond() + public void testHourMinuteSecond() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND; @@ -370,7 +369,7 @@ public void testApplyJDBCDateFormat_hourMinuteSecond() } @Test - public void testApplyJDBCDateFormat_hourMinuteSecondFraction() + public void testHourMinuteSecondFraction() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_FRACTION; @@ -381,7 +380,7 @@ public void testApplyJDBCDateFormat_hourMinuteSecondFraction() } @Test - public void testApplyJDBCDateFormat_hourMinuteSecondMillis() + public void testHourMinuteSecondMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_MILLIS; @@ -392,7 +391,7 @@ public void testApplyJDBCDateFormat_hourMinuteSecondMillis() } @Test - public void testApplyJDBCDateFormat_ordinalDate() + public void testOrdinalDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.ORDINAL_DATE; @@ -403,7 +402,7 @@ public void testApplyJDBCDateFormat_ordinalDate() } @Test - public void testApplyJDBCDateFormat_ordinalDateTime() + public void testOrdinalDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME; @@ -414,7 +413,7 @@ public void testApplyJDBCDateFormat_ordinalDateTime() } @Test - public void testApplyJDBCDateFormat_ordinalDateTimeNoMillis() + public void testOrdinalDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME_NO_MILLIS; @@ -425,7 +424,7 @@ public void testApplyJDBCDateFormat_ordinalDateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_time() + public void testTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.TIME; @@ -436,7 +435,7 @@ public void testApplyJDBCDateFormat_time() } @Test - public void testApplyJDBCDateFormat_timeNoMillis() + public void testTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.TIME_NO_MILLIS; @@ -447,7 +446,7 @@ public void testApplyJDBCDateFormat_timeNoMillis() } @Test - public void testApplyJDBCDateFormat_tTime() + public void testTTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.T_TIME; @@ -458,7 +457,7 @@ public void testApplyJDBCDateFormat_tTime() } @Test - public void testApplyJDBCDateFormat_tTimeNoMillis() + public void testTTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.T_TIME_NO_MILLIS; @@ -469,7 +468,7 @@ public void testApplyJDBCDateFormat_tTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_weekDate() + public void testWeekDate() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_DATE; @@ -480,7 +479,7 @@ public void testApplyJDBCDateFormat_weekDate() } @Test - public void testApplyJDBCDateFormat_weekDateTime() + public void testWeekDateTime() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_DATE_TIME; @@ -491,7 +490,7 @@ public void testApplyJDBCDateFormat_weekDateTime() } @Test - public void testApplyJDBCDateFormat_weekDateTimeNoMillis() + public void testWeekDateTimeNoMillis() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_DATE_TIME_NO_MILLIS; @@ -502,7 +501,7 @@ public void testApplyJDBCDateFormat_weekDateTimeNoMillis() } @Test - public void testApplyJDBCDateFormat_weekyear() + public void testWeekyear() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEK_YEAR; @@ -513,7 +512,7 @@ public void testApplyJDBCDateFormat_weekyear() } @Test - public void testApplyJDBCDateFormat_weekyearWeek() + public void testWeekyearWeek() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK; @@ -524,7 +523,7 @@ public void testApplyJDBCDateFormat_weekyearWeek() } @Test - public void testApplyJDBCDateFormat_weekyearWeekDay() + public void testWeekyearWeekDay() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK_DAY; @@ -535,7 +534,7 @@ public void testApplyJDBCDateFormat_weekyearWeekDay() } @Test - public void testApplyJDBCDateFormat_year() + public void testYear() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.YEAR; @@ -546,7 +545,7 @@ public void testApplyJDBCDateFormat_year() } @Test - public void testApplyJDBCDateFormat_yearMonth() + public void testYearMonth() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.YEAR_MONTH; @@ -557,7 +556,7 @@ public void testApplyJDBCDateFormat_yearMonth() } @Test - public void testApplyJDBCDateFormat_yearMonthDay() + public void testYearMonthDay() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.YEAR_MONTH_DAY; @@ -568,7 +567,7 @@ public void testApplyJDBCDateFormat_yearMonthDay() } @Test - public void testApplyJDBCDateFormat_incorrectFormat() + public void testIncorrectFormat() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -580,7 +579,7 @@ public void testApplyJDBCDateFormat_incorrectFormat() } @Test - public void testApplyJDBCDateFormat_nullDateData() + public void testNullDateData() { String columnName = "date_field"; DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; @@ -623,5 +622,4 @@ private Map buildDateFieldFormatMap(String columnName, DateForma .put(columnName, dateFormat.nameLowerCase()) .build(); } - } \ No newline at end of file From 0e430222dba5fd1687bacb6a1febaef9cdad33dd Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 21 Feb 2020 14:25:12 -0800 Subject: [PATCH 08/16] post-merge fix --- .../sql/executor/format/SelectResultSet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index 89f96c3c51..c6a4d38ef2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -80,8 +80,8 @@ public class SelectResultSet extends ResultSet { public SelectResultSet(Client client, Query query, Object queryResult, - String formatType, - ColumnTypeProvider outputColumnType) { + ColumnTypeProvider outputColumnType, + String formatType) { this.client = client; this.query = query; this.queryResult = queryResult; From a1a87102569779beb1f882847226693af25878b8 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Fri, 21 Feb 2020 15:30:10 -0800 Subject: [PATCH 09/16] additional fixes --- .../sql/esdomain/LocalClusterState.java | 31 +++++++++++++------ .../executor/format/DateFieldFormatter.java | 6 ++-- .../sql/parser/FieldMaker.java | 4 +-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index acf5a5d799..6c52d18fcc 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -92,15 +92,7 @@ public class LocalClusterState { */ private final Map latestSettings = new ConcurrentHashMap<>(); - private Map possibleAliasMap = new HashMap<>(); - - public void putAliasInMap(String field, String alias) { - possibleAliasMap.put(alias, field); - } - - public Map getAliasMap() { - return possibleAliasMap; - } + private Map fieldAliasMap = new HashMap<>(); public static synchronized LocalClusterState state() { if (INSTANCE == null) { @@ -219,6 +211,27 @@ public IndexMappings getFieldMappings(String[] indices, String[] types, } } + /** + * Save an alias for the provided field, which may be used in place of the field name in certain circumstances. + * + * @param alias The name of the alias for the field. + * @param field The base field for the alias. + */ + public void putAliasInMap(String alias, String field) { + fieldAliasMap.put(alias, field); + } + + /** + * Get the base field for a given alias. + * + * @param alias The alias for which to find the base field. + * @return The base field for the alias. + */ + public String getFieldForAlias(String alias) { + return fieldAliasMap.get(alias); + } + + private String[] resolveIndexExpression(ClusterState state, String[] indices) { String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.strictExpandOpen(), indices); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 18a84c69f1..2e9311789e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -92,9 +92,9 @@ public void applyJDBCDateFormat(Map rowSource) { private String getFormatForColumn(String columnName) { // Handle special cases for column names - if (columnName.startsWith("cast_")) { - // Column was cast to a date type, and prefixed with "cast_" - columnName = columnName.replaceFirst("cast_", ""); + if (LocalClusterState.state().getFieldForAlias(columnName) != null) { + // Column was aliased, and we need to find the base name for the column + columnName = LocalClusterState.state().getFieldForAlias(columnName); } else if (columnName.split("\\.").length == 2) { // Column is part of a join, and is qualified by the table alias columnName = columnName.split("\\.")[1]; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java index 298b62d18c..927c7145d5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java @@ -39,7 +39,7 @@ import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Where; -// import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; @@ -119,7 +119,7 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw if (alias == null) { alias = "cast_" + castExpr.getExpr().toString(); } - // LocalClusterState.state().putAliasInMap(castExpr.getExpr().toString(), alias); + LocalClusterState.state().putAliasInMap(alias, castExpr.getExpr().toString()); ArrayList methodParameters = new ArrayList<>(); methodParameters.add(((SQLCastExpr) expr).getExpr()); return makeMethodField("CAST", methodParameters, null, alias, tableAlias, true); From f92185c4a1ad4b3767e57d6b183f56398fcb787a Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Mon, 24 Feb 2020 12:02:58 -0800 Subject: [PATCH 10/16] get CAST alias info from result set class, rather than cluster state --- .../sql/esdomain/LocalClusterState.java | 23 ----------- .../executor/format/DateFieldFormatter.java | 38 ++++++++++--------- .../sql/executor/format/SelectResultSet.java | 16 +++++++- .../sql/parser/FieldMaker.java | 1 - .../format/DateFieldFormatterTest.java | 2 +- 5 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index 6c52d18fcc..79e7c0a961 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -92,8 +92,6 @@ public class LocalClusterState { */ private final Map latestSettings = new ConcurrentHashMap<>(); - private Map fieldAliasMap = new HashMap<>(); - public static synchronized LocalClusterState state() { if (INSTANCE == null) { INSTANCE = new LocalClusterState(); @@ -211,27 +209,6 @@ public IndexMappings getFieldMappings(String[] indices, String[] types, } } - /** - * Save an alias for the provided field, which may be used in place of the field name in certain circumstances. - * - * @param alias The name of the alias for the field. - * @param field The base field for the alias. - */ - public void putAliasInMap(String alias, String field) { - fieldAliasMap.put(alias, field); - } - - /** - * Get the base field for a given alias. - * - * @param alias The alias for which to find the base field. - * @return The base field for the alias. - */ - public String getFieldForAlias(String alias) { - return fieldAliasMap.get(alias); - } - - private String[] resolveIndexExpression(ClusterState state, String[] indices) { String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.strictExpandOpen(), indices); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 2e9311789e..2c16c4724a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -45,17 +46,22 @@ public class DateFieldFormatter { private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; private final Map dateFieldFormatMap; - private List dateColumns; + private final Map fieldAliasMap; + private Set dateColumns; - public DateFieldFormatter(String indexName, List columns) { + public DateFieldFormatter(String indexName, List columns, Map fieldAliasMap) { this.dateFieldFormatMap = getDateFieldFormatMap(indexName); this.dateColumns = getDateColumns(columns); + this.fieldAliasMap = fieldAliasMap; } @VisibleForTesting - protected DateFieldFormatter(Map dateFieldFormatMap, List columns) { + protected DateFieldFormatter(Map dateFieldFormatMap, + List columns, + Map fieldAliasMap) { this.dateFieldFormatMap = dateFieldFormatMap; this.dateColumns = getDateColumns(columns); + this.fieldAliasMap = fieldAliasMap; } /** @@ -64,23 +70,20 @@ protected DateFieldFormatter(Map dateFieldFormatMap, List rowSource) { - for (Schema.Column column : dateColumns) { - String columnName = column.getName(); + for (String columnName : dateColumns) { + Object columnOriginalDate = rowSource.get(columnName); + if (columnOriginalDate == null) { + // Don't try to parse null date values + continue; + } - String columnFormat; - columnFormat = getFormatForColumn(columnName); + String columnFormat = getFormatForColumn(columnName); if (columnFormat == null) { LOG.warn("Could not determine date format for column {}; returning original value", columnName); continue; } DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - Object columnOriginalDate = rowSource.get(columnName); - if (columnOriginalDate == null) { - // Don't try to parse null date values - continue; - } - Date date = parseDateString(format, columnOriginalDate.toString()); if (date != null) { rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); @@ -92,9 +95,9 @@ public void applyJDBCDateFormat(Map rowSource) { private String getFormatForColumn(String columnName) { // Handle special cases for column names - if (LocalClusterState.state().getFieldForAlias(columnName) != null) { + if (fieldAliasMap.get(columnName) != null) { // Column was aliased, and we need to find the base name for the column - columnName = LocalClusterState.state().getFieldForAlias(columnName); + columnName = fieldAliasMap.get(columnName); } else if (columnName.split("\\.").length == 2) { // Column is part of a join, and is qualified by the table alias columnName = columnName.split("\\.")[1]; @@ -102,10 +105,11 @@ private String getFormatForColumn(String columnName) { return dateFieldFormatMap.get(columnName); } - private List getDateColumns(List columns) { + private Set getDateColumns(List columns) { return columns.stream() .filter(column -> column.getType().equals(Schema.Type.DATE.nameLowerCase())) - .collect(Collectors.toList()); + .map(Schema.Column::getName) + .collect(Collectors.toSet()); } private Map getDateFieldFormatMap(String indexName) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index c6a4d38ef2..a94aed514c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -15,7 +15,10 @@ package com.amazon.opendistroforelasticsearch.sql.executor.format; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; +import com.alibaba.druid.sql.ast.expr.SQLCastExpr; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; import com.amazon.opendistroforelasticsearch.sql.domain.Field; import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; @@ -76,6 +79,8 @@ public class SelectResultSet extends ResultSet { private List rows; private DateFieldFormatter dateFieldFormatter; + // alias -> base field name + private Map fieldAliasMap = new HashMap<>(); public SelectResultSet(Client client, Query query, @@ -98,7 +103,7 @@ public SelectResultSet(Client client, } this.schema = new Schema(indexName, typeName, columns); this.head = schema.getHeaders(); - this.dateFieldFormatter = new DateFieldFormatter(indexName, columns); + this.dateFieldFormatter = new DateFieldFormatter(indexName, columns, fieldAliasMap); extractData(); this.dataRows = new DataRows(size, totalHits, rows); @@ -390,6 +395,15 @@ private List populateColumns(Query query, String[] fieldNames, Ma if (fieldMap.get(fieldName) instanceof MethodField) { MethodField methodField = (MethodField) fieldMap.get(fieldName); int fieldIndex = fieldNameList.indexOf(fieldName); + + SQLExpr expr = methodField.getExpression(); + if (expr instanceof SQLCastExpr) { + // Since CAST expressions create an alias for a field, we need to save the original field name + // for this alias for formatting data later. + SQLIdentifierExpr castFieldIdentifier = (SQLIdentifierExpr) ((SQLCastExpr) expr).getExpr(); + fieldAliasMap.put(methodField.getAlias(), castFieldIdentifier.getName()); + } + columns.add( new Schema.Column( methodField.getAlias(), diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java index 927c7145d5..eb7e62f788 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java @@ -119,7 +119,6 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw if (alias == null) { alias = "cast_" + castExpr.getExpr().toString(); } - LocalClusterState.state().putAliasInMap(alias, castExpr.getExpr().toString()); ArrayList methodParameters = new ArrayList<>(); methodParameters.add(((SQLCastExpr) expr).getExpr()); return makeMethodField("CAST", methodParameters, null, alias, tableAlias, true); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index 43d9f4219b..76465f53be 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -598,7 +598,7 @@ private void verifyFormatting(String columnName, DateFormat dateFormat, String o Map rowSource = new HashMap<>(); rowSource.put(columnName, originalDateValue); - DateFieldFormatter dateFieldFormatter = new DateFieldFormatter(dateFieldFormatMap, columns); + DateFieldFormatter dateFieldFormatter = new DateFieldFormatter(dateFieldFormatMap, columns, new HashMap<>()); executeFormattingAndCompare(dateFieldFormatter, rowSource, columnName, expectedDateValue); } From 6f31ca79ba9018701d291bf845198468543bef7b Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Mon, 24 Feb 2020 12:06:05 -0800 Subject: [PATCH 11/16] remove unused import --- .../sql/esdomain/LocalClusterState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index 79e7c0a961..2d33ea53e2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; From e0d406ae9c19d6bcf78971b690a2a0bdaf0a8dd4 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Mon, 24 Feb 2020 12:09:39 -0800 Subject: [PATCH 12/16] remove unused import --- .../amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java index eb7e62f788..de2020daa6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java @@ -39,7 +39,6 @@ import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; From 2583eedd5f1d05cc29132477ead47af06408ce35 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Tue, 25 Feb 2020 15:25:23 -0800 Subject: [PATCH 13/16] reduce duplication & reference enum value --- .../sql/executor/format/DateFieldFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 2c16c4724a..e82a085334 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -43,7 +43,7 @@ public class DateFieldFormatter { private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; - private static final String FORMAT_DOT_DATE = "yyyy-MM-dd"; + private static final String FORMAT_DOT_DATE = DateFormat.DATE.getFormatString(); private final Map dateFieldFormatMap; private final Map fieldAliasMap; From b5b2eac464d872595be216b5a2d356a25633ac63 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Tue, 25 Feb 2020 16:53:21 -0800 Subject: [PATCH 14/16] setting default timezone to UTC while parsing date values --- .../executor/format/DateFieldFormatter.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index e82a085334..04161c2320 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.stream.Collectors; /** @@ -140,28 +141,41 @@ private Map getDateFieldFormatMap(String indexName) { } private Date parseDateString(DateFormat format, String columnOriginalDate) { + TimeZone originalDefaultTimeZone = TimeZone.getDefault(); + Date parsedDate = null; + try { + // Apache Commons DateUtils uses the default TimeZone for the JVM when parsing. + // However, since all dates on Elasticsearch are stored as UTC, we need to + // parse these values using the UTC timezone. + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); switch (format) { case DATE_OPTIONAL_TIME: - return DateUtils.parseDate( + parsedDate = DateUtils.parseDate( columnOriginalDate, FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, FORMAT_DOT_DATE_AND_TIME, FORMAT_DOT_DATE); + break; case EPOCH_MILLIS: - return new Date(Long.parseLong(columnOriginalDate)); + parsedDate = new Date(Long.parseLong(columnOriginalDate)); + break; case EPOCH_SECOND: - return new Date(Long.parseLong(columnOriginalDate) * 1000); + parsedDate = new Date(Long.parseLong(columnOriginalDate) * 1000); + break; default: - return DateUtils.parseDate(columnOriginalDate, format.getFormatString()); + parsedDate = DateUtils.parseDate(columnOriginalDate, format.getFormatString()); } } catch (ParseException e) { LOG.error( String.format("Error parsing date string %s as %s", columnOriginalDate, format.nameLowerCase()), e); + } finally { + // Reset default timezone after parsing + TimeZone.setDefault(originalDefaultTimeZone); } - return null; + return parsedDate; } } From e5649f001d5d7635d64df5adf2529ad9a80b6332 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Wed, 26 Feb 2020 12:19:28 -0800 Subject: [PATCH 15/16] add support for custom & multiple formats --- .../executor/format/DateFieldFormatter.java | 100 +- .../sql/executor/format/DateFormat.java | 223 +-- .../format/DateFieldFormatterTest.java | 1247 +++++++++-------- 3 files changed, 820 insertions(+), 750 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 04161c2320..6f32fe0639 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.Logger; import java.text.ParseException; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashMap; @@ -39,14 +40,15 @@ public class DateFieldFormatter { private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; + private static final String FORMAT_DELIMITER = "\\|\\|"; private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; - private static final String FORMAT_DOT_DATE = DateFormat.DATE.getFormatString(); + private static final String FORMAT_DOT_DATE = DateFormat.getFormatString("date"); - private final Map dateFieldFormatMap; + private final Map> dateFieldFormatMap; private final Map fieldAliasMap; private Set dateColumns; @@ -57,7 +59,7 @@ public DateFieldFormatter(String indexName, List columns, Map dateFieldFormatMap, + protected DateFieldFormatter(Map> dateFieldFormatMap, List columns, Map fieldAliasMap) { this.dateFieldFormatMap = dateFieldFormatMap; @@ -78,23 +80,23 @@ public void applyJDBCDateFormat(Map rowSource) { continue; } - String columnFormat = getFormatForColumn(columnName); - if (columnFormat == null) { - LOG.warn("Could not determine date format for column {}; returning original value", columnName); + List formats = getFormatsForColumn(columnName); + if (formats == null) { + LOG.warn("Could not determine date formats for column {}; returning original value", columnName); continue; } - DateFormat format = DateFormat.valueOf(columnFormat.toUpperCase()); - Date date = parseDateString(format, columnOriginalDate.toString()); + Date date = parseDateString(formats, columnOriginalDate.toString()); if (date != null) { rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); + break; } else { LOG.warn("Could not parse date value; returning original value"); } } } - private String getFormatForColumn(String columnName) { + private List getFormatsForColumn(String columnName) { // Handle special cases for column names if (fieldAliasMap.get(columnName) != null) { // Column was aliased, and we need to find the base name for the column @@ -113,9 +115,9 @@ private Set getDateColumns(List columns) { .collect(Collectors.toSet()); } - private Map getDateFieldFormatMap(String indexName) { + private Map> getDateFieldFormatMap(String indexName) { LocalClusterState state = LocalClusterState.state(); - Map formatMap = new HashMap<>(); + Map> formatMap = new HashMap<>(); String[] indices = indexName.split("\\|"); Collection typeProperties = state.getFieldMappings(indices) @@ -128,11 +130,11 @@ private Map getDateFieldFormatMap(String indexName) { Map properties = field.getValue(); if (properties.containsKey("format")) { - formatMap.put(fieldName, properties.get("format").toString()); + formatMap.put(fieldName, getFormatsFromProperties(properties.get("format").toString())); } else { // Give all field types a format, since operations such as casts // can change the output type for a field to `date`. - formatMap.put(fieldName, "date_optional_time"); + formatMap.put(fieldName, getFormatsFromProperties("date_optional_time")); } } } @@ -140,42 +142,52 @@ private Map getDateFieldFormatMap(String indexName) { return formatMap; } - private Date parseDateString(DateFormat format, String columnOriginalDate) { + private List getFormatsFromProperties(String formatProperty) { + String[] formats = formatProperty.split(FORMAT_DELIMITER); + return Arrays.asList(formats); + } + + private Date parseDateString(List formats, String columnOriginalDate) { TimeZone originalDefaultTimeZone = TimeZone.getDefault(); Date parsedDate = null; - try { - // Apache Commons DateUtils uses the default TimeZone for the JVM when parsing. - // However, since all dates on Elasticsearch are stored as UTC, we need to - // parse these values using the UTC timezone. - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - switch (format) { - case DATE_OPTIONAL_TIME: - parsedDate = DateUtils.parseDate( - columnOriginalDate, - FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, - FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, - FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, - FORMAT_DOT_DATE_AND_TIME, - FORMAT_DOT_DATE); - break; - case EPOCH_MILLIS: - parsedDate = new Date(Long.parseLong(columnOriginalDate)); - break; - case EPOCH_SECOND: - parsedDate = new Date(Long.parseLong(columnOriginalDate) * 1000); - break; - default: - parsedDate = DateUtils.parseDate(columnOriginalDate, format.getFormatString()); + // Apache Commons DateUtils uses the default TimeZone for the JVM when parsing. + // However, since all dates on Elasticsearch are stored as UTC, we need to + // parse these values using the UTC timezone. + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + for (String columnFormat : formats) { + try { + switch (columnFormat) { + case "date_optional_time": + parsedDate = DateUtils.parseDate( + columnOriginalDate, + FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); + break; + case "epoch_millis": + parsedDate = new Date(Long.parseLong(columnOriginalDate)); + break; + case "epoch_second": + parsedDate = new Date(Long.parseLong(columnOriginalDate) * 1000); + break; + default: + String formatString = DateFormat.getFormatString(columnFormat); + if (formatString == null) { + // Custom format; take as-is + formatString = columnFormat; + } + parsedDate = DateUtils.parseDate(columnOriginalDate, formatString); + } + } catch (ParseException | NumberFormatException e) { + LOG.warn(String.format("Could not parse date string %s as %s", columnOriginalDate, columnFormat)); } - } catch (ParseException e) { - LOG.error( - String.format("Error parsing date string %s as %s", columnOriginalDate, format.nameLowerCase()), - e); - } finally { - // Reset default timezone after parsing - TimeZone.setDefault(originalDefaultTimeZone); } + // Reset default timezone after parsing + TimeZone.setDefault(originalDefaultTimeZone); + return parsedDate; } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java index c605d8cde5..c3baa0d912 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java @@ -19,115 +19,116 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; - -public enum DateFormat { - // Special cases that are parsed separately - DATE_OPTIONAL_TIME(""), - EPOCH_MILLIS(""), - EPOCH_SECOND(""), - - BASIC_DATE(Date.BASIC_DATE), - BASIC_DATE_TIME(Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), - BASIC_DATE_TIME_NO_MILLIS(Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.TZ), - - BASIC_ORDINAL_DATE(Date.BASIC_ORDINAL_DATE), - BASIC_ORDINAL_DATE_TIME(Date.BASIC_ORDINAL_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), - BASIC_ORDINAL_DATE_TIME_NO_MILLIS(Date.BASIC_ORDINAL_DATE+ Time.T + Time.BASIC_TIME + Time.TZ), - - BASIC_TIME(Time.BASIC_TIME + Time.MILLIS + Time.TZ), - BASIC_TIME_NO_MILLIS(Time.BASIC_TIME + Time.TZ), - - BASIC_T_TIME(Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), - BASIC_T_TIME_NO_MILLIS(Time.T + Time.BASIC_TIME + Time.TZ), - - BASIC_WEEK_DATE(Date.BASIC_WEEK_DATE), - BASIC_WEEK_DATE_TIME(Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ), - BASIC_WEEK_DATE_TIME_NO_MILLIS(Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.TZ), - - DATE(Date.DATE), - DATE_HOUR(Date.DATE + Time.T + Time.HOUR), - DATE_HOUR_MINUTE(Date.DATE + Time.T + Time.HOUR_MINUTE), - DATE_HOUR_MINUTE_SECOND(Date.DATE + Time.T + Time.TIME), - DATE_HOUR_MINUTE_SECOND_FRACTION(Date.DATE + Time.T + Time.TIME + Time.MILLIS), - DATE_HOUR_MINUTE_SECOND_MILLIS(Date.DATE + Time.T + Time.TIME + Time.MILLIS), - DATE_TIME(Date.DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), - DATE_TIME_NO_MILLIS(Date.DATE + Time.T + Time.TIME + Time.TZZ), - - HOUR(Time.HOUR), - HOUR_MINUTE(Time.HOUR_MINUTE), - HOUR_MINUTE_SECOND(Time.TIME), - HOUR_MINUTE_SECOND_FRACTION(Time.TIME + Time.MILLIS), - HOUR_MINUTE_SECOND_MILLIS(Time.TIME + Time.MILLIS), - - ORDINAL_DATE(Date.ORDINAL_DATE), - ORDINAL_DATE_TIME(Date.ORDINAL_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), - ORDINAL_DATE_TIME_NO_MILLIS(Date.ORDINAL_DATE + Time.T + Time.TIME + Time.TZZ), - - TIME(Time.TIME + Time.MILLIS + Time.TZZ), - TIME_NO_MILLIS(Time.TIME + Time.TZZ), - - T_TIME(Time.T + Time.TIME + Time.MILLIS + Time.TZZ), - T_TIME_NO_MILLIS(Time.T + Time.TIME + Time.TZZ), - - WEEK_DATE(Date.WEEK_DATE), - WEEK_DATE_TIME(Date.WEEK_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ), - WEEK_DATE_TIME_NO_MILLIS(Date.WEEK_DATE + Time.T + Time.TIME + Time.TZZ), - - // Note: input mapping is "weekyear", but output value is "week_year" - WEEK_YEAR(Date.WEEKYEAR), - WEEKYEAR_WEEK(Date.WEEKYEAR_WEEK), - WEEKYEAR_WEEK_DAY(Date.WEEK_DATE), - - YEAR(Date.YEAR), - YEAR_MONTH(Date.YEAR_MONTH), - YEAR_MONTH_DAY(Date.DATE); - - private static class Date { - static String BASIC_DATE = "yyyyMMdd"; - static String BASIC_ORDINAL_DATE = "yyyyDDD"; - static String BASIC_WEEK_DATE = "YYYY'W'wwu"; - - static String DATE = "yyyy-MM-dd"; - static String ORDINAL_DATE = "yyyy-DDD"; - - static String YEAR = "yyyy"; - static String YEAR_MONTH = "yyyy-MM"; - - static String WEEK_DATE = "YYYY-'W'ww-u"; - static String WEEKYEAR = "YYYY"; - static String WEEKYEAR_WEEK = "YYYY-'W'ww"; - } - - private static class Time { - static String T = "'T'"; - static String BASIC_TIME = "HHmmss"; - static String TIME = "HH:mm:ss"; - - static String HOUR = "HH"; - static String HOUR_MINUTE = "HH:mm"; - - static String MILLIS = ".SSS"; - static String TZ = "Z"; - static String TZZ = "XX"; - } - - private String formatString; - - DateFormat(String formatString) { - this.formatString = formatString; - } - - public String getFormatString() { - return formatString; - } - - public String nameLowerCase() { - return name().toLowerCase(); - } - - public static String getFormattedDate(java.util.Date date, String dateFormat) { - Instant instant = date.toInstant(); - ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("Etc/UTC")); - return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); - } +import java.util.HashMap; +import java.util.Map; + +public class DateFormat { + + private static Map formatMap = new HashMap<>(); + + static { + // Special cases that are parsed separately + formatMap.put("date_optional_time", ""); + formatMap.put("epoch_millis", ""); + formatMap.put("epoch_second", ""); + + formatMap.put("basic_date", Date.BASIC_DATE); + formatMap.put("basic_date_time", Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ); + formatMap.put("basic_date_time_no_millis", Date.BASIC_DATE + Time.T + Time.BASIC_TIME + Time.TZ); + + formatMap.put("basic_ordinal_date", Date.BASIC_ORDINAL_DATE); + formatMap.put("basic_ordinal_date_time", + Date.BASIC_ORDINAL_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ); + formatMap.put("basic_ordinal_date_time_no_millis", Date.BASIC_ORDINAL_DATE+ Time.T + Time.BASIC_TIME + Time.TZ); + + formatMap.put("basic_time", Time.BASIC_TIME + Time.MILLIS + Time.TZ); + formatMap.put("basic_time_no_millis", Time.BASIC_TIME + Time.TZ); + + formatMap.put("basic_t_time", Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ); + formatMap.put("basic_t_time_no_millis", Time.T + Time.BASIC_TIME + Time.TZ); + + formatMap.put("basic_week_date", Date.BASIC_WEEK_DATE); + formatMap.put("basic_week_date_time", Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.MILLIS + Time.TZ); + formatMap.put("basic_week_date_time_no_millis", Date.BASIC_WEEK_DATE + Time.T + Time.BASIC_TIME + Time.TZ); + + formatMap.put("date", Date.DATE); + formatMap.put("date_hour", Date.DATE + Time.T + Time.HOUR); + formatMap.put("date_hour_minute", Date.DATE + Time.T + Time.HOUR_MINUTE); + formatMap.put("date_hour_minute_second", Date.DATE + Time.T + Time.TIME); + formatMap.put("date_hour_minute_second_fraction", Date.DATE + Time.T + Time.TIME + Time.MILLIS); + formatMap.put("date_hour_minute_second_millis", Date.DATE + Time.T + Time.TIME + Time.MILLIS); + formatMap.put("date_time", Date.DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ); + formatMap.put("date_time_no_millis", Date.DATE + Time.T + Time.TIME + Time.TZZ); + + formatMap.put("hour", Time.HOUR); + formatMap.put("hour_minute", Time.HOUR_MINUTE); + formatMap.put("hour_minute_second", Time.TIME); + formatMap.put("hour_minute_second_fraction", Time.TIME + Time.MILLIS); + formatMap.put("hour_minute_second_millis", Time.TIME + Time.MILLIS); + + formatMap.put("ordinal_date", Date.ORDINAL_DATE); + formatMap.put("ordinal_date_time", Date.ORDINAL_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ); + formatMap.put("ordinal_date_time_no_millis", Date.ORDINAL_DATE + Time.T + Time.TIME + Time.TZZ); + + formatMap.put("time", Time.TIME + Time.MILLIS + Time.TZZ); + formatMap.put("time_no_millis", Time.TIME + Time.TZZ); + + formatMap.put("t_time", Time.T + Time.TIME + Time.MILLIS + Time.TZZ); + formatMap.put("t_time_no_millis", Time.T + Time.TIME + Time.TZZ); + + formatMap.put("week_date", Date.WEEK_DATE); + formatMap.put("week_date_time", Date.WEEK_DATE + Time.T + Time.TIME + Time.MILLIS + Time.TZZ); + formatMap.put("week_date_time_no_millis", Date.WEEK_DATE + Time.T + Time.TIME + Time.TZZ); + + // Note: input mapping is "weekyear", but output value is "week_year" + formatMap.put("week_year", Date.WEEKYEAR); + formatMap.put("weekyear_week", Date.WEEKYEAR_WEEK); + formatMap.put("weekyear_week_day", Date.WEEK_DATE); + + formatMap.put("year", Date.YEAR); + formatMap.put("year_month", Date.YEAR_MONTH); + formatMap.put("year_month_day", Date.DATE); + } + + private DateFormat() { + } + + public static String getFormatString(String formatName) { + return formatMap.get(formatName); + } + + public static String getFormattedDate(java.util.Date date, String dateFormat) { + Instant instant = date.toInstant(); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("Etc/UTC")); + return zdt.format(DateTimeFormatter.ofPattern(dateFormat)); + } + + private static class Date { + static String BASIC_DATE = "yyyyMMdd"; + static String BASIC_ORDINAL_DATE = "yyyyDDD"; + static String BASIC_WEEK_DATE = "YYYY'W'wwu"; + + static String DATE = "yyyy-MM-dd"; + static String ORDINAL_DATE = "yyyy-DDD"; + + static String YEAR = "yyyy"; + static String YEAR_MONTH = "yyyy-MM"; + + static String WEEK_DATE = "YYYY-'W'ww-u"; + static String WEEKYEAR = "YYYY"; + static String WEEKYEAR_WEEK = "YYYY-'W'ww"; + } + + private static class Time { + static String T = "'T'"; + static String BASIC_TIME = "HHmmss"; + static String TIME = "HH:mm:ss"; + + static String HOUR = "HH"; + static String HOUR_MINUTE = "HH:mm"; + + static String MILLIS = ".SSS"; + static String TZ = "Z"; + static String TZZ = "XX"; + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index 76465f53be..c239e77d0e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,599 +28,655 @@ public class DateFieldFormatterTest { - @Test - public void testKibanaSampleDataEcommerceOrderDateField() - { - String columnName = "order_date"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "2020-02-24T09:28:48+00:00"; - String expectedDateValue = "2020-02-24 09:28:48.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testKibanaSampleDataFlightsTimestampField() - { - String columnName = "timestamp"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "2020-02-03T00:00:00"; - String expectedDateValue = "2020-02-03 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testKibanaSampleDataLogsUtcDateField() - { - String columnName = "utc_date"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "2020-02-02T00:39:02.912Z"; - String expectedDateValue = "2020-02-02 00:39:02.912"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testEpochMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.EPOCH_MILLIS; - String originalDateValue = "727430805000"; - String expectedDateValue = "1993-01-19 08:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testEpochSecond() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.EPOCH_SECOND; - String originalDateValue = "727430805"; - String expectedDateValue = "1993-01-19 08:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateOptionalTimeDateOnly() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "1993-01-19"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateOptionalTimeDateAndTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "1993-01-19T00:06:45.123-0800"; - String expectedDateValue = "1993-01-19 08:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_DATE; - String originalDateValue = "19930119"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_DATE_TIME; - String originalDateValue = "19930119T120645.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_DATE_TIME_NO_MILLIS; - String originalDateValue = "19930119T120645-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicOrdinalDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE; - String originalDateValue = "1993019"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicOrdinalDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME; - String originalDateValue = "1993019T120645.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicOrdinalDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_ORDINAL_DATE_TIME_NO_MILLIS; - String originalDateValue = "1993019T120645-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_TIME; - String originalDateValue = "120645.123-0800"; - String expectedDateValue = "1970-01-01 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_TIME_NO_MILLIS; - String originalDateValue = "120645-0800"; - String expectedDateValue = "1970-01-01 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicTTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_T_TIME; - String originalDateValue = "T120645.123-0800"; - String expectedDateValue = "1970-01-01 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicTTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_T_TIME_NO_MILLIS; - String originalDateValue = "T120645-0800"; - String expectedDateValue = "1970-01-01 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicWeekDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE; - String originalDateValue = "1993W042"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicWeekDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME; - String originalDateValue = "1993W042T120645.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testBasicWeekDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.BASIC_WEEK_DATE_TIME_NO_MILLIS; - String originalDateValue = "1993W042T120645-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE; - String originalDateValue = "1993-01-19"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateHour() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_HOUR; - String originalDateValue = "1993-01-19T12"; - String expectedDateValue = "1993-01-19 12:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateHourMinute() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE; - String originalDateValue = "1993-01-19T12:06"; - String expectedDateValue = "1993-01-19 12:06:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateHourMinuteSecond() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND; - String originalDateValue = "1993-01-19T12:06:45"; - String expectedDateValue = "1993-01-19 12:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateHourMinuteSecondFraction() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_FRACTION; - String originalDateValue = "1993-01-19T12:06:45.123"; - String expectedDateValue = "1993-01-19 12:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateHourMinuteSecondMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_HOUR_MINUTE_SECOND_MILLIS; - String originalDateValue = "1993-01-19T12:06:45.123"; - String expectedDateValue = "1993-01-19 12:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_TIME; - String originalDateValue = "1993-01-19T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_TIME_NO_MILLIS; - String originalDateValue = "1993-01-19T12:06:45-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testHour() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.HOUR; - String originalDateValue = "12"; - String expectedDateValue = "1970-01-01 12:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testHourMinute() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.HOUR_MINUTE; - String originalDateValue = "12:06"; - String expectedDateValue = "1970-01-01 12:06:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testHourMinuteSecond() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND; - String originalDateValue = "12:06:45"; - String expectedDateValue = "1970-01-01 12:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testHourMinuteSecondFraction() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_FRACTION; - String originalDateValue = "12:06:45.123"; - String expectedDateValue = "1970-01-01 12:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testHourMinuteSecondMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.HOUR_MINUTE_SECOND_MILLIS; - String originalDateValue = "12:06:45.123"; - String expectedDateValue = "1970-01-01 12:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testOrdinalDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.ORDINAL_DATE; - String originalDateValue = "1993-019"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testOrdinalDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME; - String originalDateValue = "1993-019T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testOrdinalDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.ORDINAL_DATE_TIME_NO_MILLIS; - String originalDateValue = "1993-019T12:06:45-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.TIME; - String originalDateValue = "12:06:45.123-0800"; - String expectedDateValue = "1970-01-01 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.TIME_NO_MILLIS; - String originalDateValue = "12:06:45-0800"; - String expectedDateValue = "1970-01-01 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testTTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.T_TIME; - String originalDateValue = "T12:06:45.123-0800"; - String expectedDateValue = "1970-01-01 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testTTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.T_TIME_NO_MILLIS; - String originalDateValue = "T12:06:45-0800"; - String expectedDateValue = "1970-01-01 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekDate() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEK_DATE; - String originalDateValue = "1993-W04-2"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekDateTime() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEK_DATE_TIME; - String originalDateValue = "1993-W04-2T12:06:45.123-0800"; - String expectedDateValue = "1993-01-19 20:06:45.123"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekDateTimeNoMillis() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEK_DATE_TIME_NO_MILLIS; - String originalDateValue = "1993-W04-2T12:06:45-0800"; - String expectedDateValue = "1993-01-19 20:06:45.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekyear() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEK_YEAR; - String originalDateValue = "1993"; - String expectedDateValue = "1993-01-01 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekyearWeek() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK; - String originalDateValue = "1993-W04"; - String expectedDateValue = "1993-01-17 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testWeekyearWeekDay() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.WEEKYEAR_WEEK_DAY; - String originalDateValue = "1993-W04-2"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testYear() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.YEAR; - String originalDateValue = "1993"; - String expectedDateValue = "1993-01-01 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testYearMonth() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.YEAR_MONTH; - String originalDateValue = "1993-01"; - String expectedDateValue = "1993-01-01 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testYearMonthDay() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.YEAR_MONTH_DAY; - String originalDateValue = "1993-01-19"; - String expectedDateValue = "1993-01-19 00:00:00.000"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testIncorrectFormat() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = "1581724085"; - // Invalid format for date value; should return original value - String expectedDateValue = "1581724085"; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - @Test - public void testNullDateData() - { - String columnName = "date_field"; - DateFormat dateFormat = DateFormat.DATE_OPTIONAL_TIME; - String originalDateValue = null; - // Nulls should be preserved - String expectedDateValue = null; - - verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); - } - - private void verifyFormatting(String columnName, DateFormat dateFormat, String originalDateValue, String expectedDateValue) - { - List columns = buildColumnList(columnName); - Map dateFieldFormatMap = buildDateFieldFormatMap(columnName, dateFormat); - - Map rowSource = new HashMap<>(); - rowSource.put(columnName, originalDateValue); - - DateFieldFormatter dateFieldFormatter = new DateFieldFormatter(dateFieldFormatMap, columns, new HashMap<>()); - executeFormattingAndCompare(dateFieldFormatter, rowSource, columnName, expectedDateValue); - } - - private void executeFormattingAndCompare( - DateFieldFormatter formatter, - Map rowSource, - String columnToCheck, - String expectedDateValue) { - formatter.applyJDBCDateFormat(rowSource); - assertEquals(expectedDateValue, rowSource.get(columnToCheck)); - } - - private List buildColumnList(String columnName) { - return ImmutableList.builder() - .add(new Schema.Column(columnName, null, Schema.Type.DATE)) - .build(); - } - - private Map buildDateFieldFormatMap(String columnName, DateFormat dateFormat) { - return ImmutableMap.builder() - .put(columnName, dateFormat.nameLowerCase()) - .build(); - } + @Test + public void testKibanaSampleDataEcommerceOrderDateField() + { + String columnName = "order_date"; + String dateFormat = "date_optional_time"; + String originalDateValue = "2020-02-24T09:28:48+00:00"; + String expectedDateValue = "2020-02-24 09:28:48.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testKibanaSampleDataFlightsTimestampField() + { + String columnName = "timestamp"; + String dateFormat = "date_optional_time"; + String originalDateValue = "2020-02-03T00:00:00"; + String expectedDateValue = "2020-02-03 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testKibanaSampleDataLogsUtcDateField() + { + String columnName = "utc_date"; + String dateFormat = "date_optional_time"; + String originalDateValue = "2020-02-02T00:39:02.912Z"; + String expectedDateValue = "2020-02-02 00:39:02.912"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testEpochMillis() + { + String columnName = "date_field"; + String dateFormat = "epoch_millis"; + String originalDateValue = "727430805000"; + String expectedDateValue = "1993-01-19 08:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testEpochSecond() + { + String columnName = "date_field"; + String dateFormat = "epoch_second"; + String originalDateValue = "727430805"; + String expectedDateValue = "1993-01-19 08:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateOptionalTimeDateOnly() + { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateOptionalTimeDateAndTime() + { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1993-01-19T00:06:45.123-0800"; + String expectedDateValue = "1993-01-19 08:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicDate() + { + String columnName = "date_field"; + String dateFormat = "basic_date"; + String originalDateValue = "19930119"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicDateTime() + { + String columnName = "date_field"; + String dateFormat = "basic_date_time"; + String originalDateValue = "19930119T120645.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "basic_date_time_no_millis"; + String originalDateValue = "19930119T120645-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicOrdinalDate() + { + String columnName = "date_field"; + String dateFormat = "basic_ordinal_date"; + String originalDateValue = "1993019"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicOrdinalDateTime() + { + String columnName = "date_field"; + String dateFormat = "basic_ordinal_date_time"; + String originalDateValue = "1993019T120645.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicOrdinalDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "basic_ordinal_date_time_no_millis"; + String originalDateValue = "1993019T120645-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicTime() + { + String columnName = "date_field"; + String dateFormat = "basic_time"; + String originalDateValue = "120645.123-0800"; + String expectedDateValue = "1970-01-01 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "basic_time_no_millis"; + String originalDateValue = "120645-0800"; + String expectedDateValue = "1970-01-01 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicTTime() + { + String columnName = "date_field"; + String dateFormat = "basic_t_time"; + String originalDateValue = "T120645.123-0800"; + String expectedDateValue = "1970-01-01 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicTTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "basic_t_time_no_millis"; + String originalDateValue = "T120645-0800"; + String expectedDateValue = "1970-01-01 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicWeekDate() + { + String columnName = "date_field"; + String dateFormat = "basic_week_date"; + String originalDateValue = "1993W042"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicWeekDateTime() + { + String columnName = "date_field"; + String dateFormat = "basic_week_date_time"; + String originalDateValue = "1993W042T120645.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testBasicWeekDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "basic_week_date_time_no_millis"; + String originalDateValue = "1993W042T120645-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDate() + { + String columnName = "date_field"; + String dateFormat = "date"; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateHour() + { + String columnName = "date_field"; + String dateFormat = "date_hour"; + String originalDateValue = "1993-01-19T12"; + String expectedDateValue = "1993-01-19 12:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateHourMinute() + { + String columnName = "date_field"; + String dateFormat = "date_hour_minute"; + String originalDateValue = "1993-01-19T12:06"; + String expectedDateValue = "1993-01-19 12:06:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateHourMinuteSecond() + { + String columnName = "date_field"; + String dateFormat = "date_hour_minute_second"; + String originalDateValue = "1993-01-19T12:06:45"; + String expectedDateValue = "1993-01-19 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateHourMinuteSecondFraction() + { + String columnName = "date_field"; + String dateFormat = "date_hour_minute_second_fraction"; + String originalDateValue = "1993-01-19T12:06:45.123"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateHourMinuteSecondMillis() + { + String columnName = "date_field"; + String dateFormat = "date_hour_minute_second_millis"; + String originalDateValue = "1993-01-19T12:06:45.123"; + String expectedDateValue = "1993-01-19 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateTime() + { + String columnName = "date_field"; + String dateFormat = "date_time"; + String originalDateValue = "1993-01-19T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "date_time_no_millis"; + String originalDateValue = "1993-01-19T12:06:45-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testHour() + { + String columnName = "date_field"; + String dateFormat = "hour"; + String originalDateValue = "12"; + String expectedDateValue = "1970-01-01 12:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testHourMinute() + { + String columnName = "date_field"; + String dateFormat = "hour_minute"; + String originalDateValue = "12:06"; + String expectedDateValue = "1970-01-01 12:06:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testHourMinuteSecond() + { + String columnName = "date_field"; + String dateFormat = "hour_minute_second"; + String originalDateValue = "12:06:45"; + String expectedDateValue = "1970-01-01 12:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testHourMinuteSecondFraction() + { + String columnName = "date_field"; + String dateFormat = "hour_minute_second_fraction"; + String originalDateValue = "12:06:45.123"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testHourMinuteSecondMillis() + { + String columnName = "date_field"; + String dateFormat = "hour_minute_second_millis"; + String originalDateValue = "12:06:45.123"; + String expectedDateValue = "1970-01-01 12:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testOrdinalDate() + { + String columnName = "date_field"; + String dateFormat = "ordinal_date"; + String originalDateValue = "1993-019"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testOrdinalDateTime() + { + String columnName = "date_field"; + String dateFormat = "ordinal_date_time"; + String originalDateValue = "1993-019T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testOrdinalDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "ordinal_date_time_no_millis"; + String originalDateValue = "1993-019T12:06:45-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testTime() + { + String columnName = "date_field"; + String dateFormat = "time"; + String originalDateValue = "12:06:45.123-0800"; + String expectedDateValue = "1970-01-01 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "time_no_millis"; + String originalDateValue = "12:06:45-0800"; + String expectedDateValue = "1970-01-01 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testTTime() + { + String columnName = "date_field"; + String dateFormat = "t_time"; + String originalDateValue = "T12:06:45.123-0800"; + String expectedDateValue = "1970-01-01 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testTTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "t_time_no_millis"; + String originalDateValue = "T12:06:45-0800"; + String expectedDateValue = "1970-01-01 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekDate() + { + String columnName = "date_field"; + String dateFormat = "week_date"; + String originalDateValue = "1993-W04-2"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekDateTime() + { + String columnName = "date_field"; + String dateFormat = "week_date_time"; + String originalDateValue = "1993-W04-2T12:06:45.123-0800"; + String expectedDateValue = "1993-01-19 20:06:45.123"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekDateTimeNoMillis() + { + String columnName = "date_field"; + String dateFormat = "week_date_time_no_millis"; + String originalDateValue = "1993-W04-2T12:06:45-0800"; + String expectedDateValue = "1993-01-19 20:06:45.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekyear() + { + String columnName = "date_field"; + String dateFormat = "week_year"; + String originalDateValue = "1993"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekyearWeek() + { + String columnName = "date_field"; + String dateFormat = "weekyear_week"; + String originalDateValue = "1993-W04"; + String expectedDateValue = "1993-01-17 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testWeekyearWeekDay() + { + String columnName = "date_field"; + String dateFormat = "weekyear_week_day"; + String originalDateValue = "1993-W04-2"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testYear() + { + String columnName = "date_field"; + String dateFormat = "year"; + String originalDateValue = "1993"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testYearMonth() + { + String columnName = "date_field"; + String dateFormat = "year_month"; + String originalDateValue = "1993-01"; + String expectedDateValue = "1993-01-01 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testYearMonthDay() + { + String columnName = "date_field"; + String dateFormat = "year_month_day"; + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testCustomFormat() + { + String columnName = "date_field"; + String dateFormat = "EEE, MMM d, ''yy"; + + String originalDateValue = "Tue, Jan 19, '93"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testMultipleFormats() + { + String columnName = "date_field"; + String dateFormat = "date_optional_time||epoch_millis"; + + String originalDateValue = "1993-01-19"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + + originalDateValue = "727401600000"; + expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testMultipleCustomFormats() + { + String columnName = "date_field"; + String dateFormat = "EEE, MMM d, ''yy||yyMMddHHmmssZ"; + + String originalDateValue = "Tue, Jan 19, '93"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + + originalDateValue = "930119000000-0000"; + expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testNamedAndCustomFormats() + { + String columnName = "date_field"; + String dateFormat = "EEE, MMM d, ''yy||hour_minute_second"; + + String originalDateValue = "Tue, Jan 19, '93"; + String expectedDateValue = "1993-01-19 00:00:00.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + + originalDateValue = "12:06:45"; + expectedDateValue = "1970-01-01 12:06:45.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testIncorrectFormat() + { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1581724085"; + // Invalid format for date value; should return original value + String expectedDateValue = "1581724085"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testNullDateData() + { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = null; + // Nulls should be preserved + String expectedDateValue = null; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + private void verifyFormatting(String columnName, String dateFormatProperty, String originalDateValue, String expectedDateValue) + { + List columns = buildColumnList(columnName); + Map> dateFieldFormatMap = buildDateFieldFormatMap(columnName, dateFormatProperty); + + Map rowSource = new HashMap<>(); + rowSource.put(columnName, originalDateValue); + + DateFieldFormatter dateFieldFormatter = new DateFieldFormatter(dateFieldFormatMap, columns, new HashMap<>()); + executeFormattingAndCompare(dateFieldFormatter, rowSource, columnName, expectedDateValue); + } + + private void executeFormattingAndCompare( + DateFieldFormatter formatter, + Map rowSource, + String columnToCheck, + String expectedDateValue) { + formatter.applyJDBCDateFormat(rowSource); + assertEquals(expectedDateValue, rowSource.get(columnToCheck)); + } + + private List buildColumnList(String columnName) { + return ImmutableList.builder() + .add(new Schema.Column(columnName, null, Schema.Type.DATE)) + .build(); + } + + private Map> buildDateFieldFormatMap(String columnName, String dateFormatProperty) { + return ImmutableMap.>builder() + .put(columnName, Arrays.asList(dateFormatProperty.split("\\|\\|"))) + .build(); + } } \ No newline at end of file From 1532683b470d5018ce8e85ff7f766c4553ac33f8 Mon Sep 17 00:00:00 2001 From: Jordan Wilson Date: Wed, 26 Feb 2020 16:12:49 -0800 Subject: [PATCH 16/16] add case for Kibana flights date data with T but no time field --- .../sql/executor/format/DateFieldFormatter.java | 2 ++ .../sql/executor/format/DateFieldFormatterTest.java | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java index 6f32fe0639..f05d340bad 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java @@ -45,6 +45,7 @@ public class DateFieldFormatter { private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ss"; + private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME = "yyyy-MM-dd'T'"; private static final String FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION = "yyyy-MM-dd'T'HH:mm:ssXXX"; private static final String FORMAT_DOT_DATE = DateFormat.getFormatString("date"); @@ -163,6 +164,7 @@ private Date parseDateString(List formats, String columnOriginalDate) { columnOriginalDate, FORMAT_DOT_KIBANA_SAMPLE_DATA_LOGS_EXCEPTION, FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_KIBANA_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, FORMAT_DOT_KIBANA_SAMPLE_DATA_ECOMMERCE_EXCEPTION, FORMAT_DOT_DATE_AND_TIME, FORMAT_DOT_DATE); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java index c239e77d0e..b08dc21959 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java @@ -50,6 +50,17 @@ public void testKibanaSampleDataFlightsTimestampField() verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } + @Test + public void testKibanaSampleDataFlightsTimestampFieldNoTime() + { + String columnName = "timestamp"; + String dateFormat = "date_optional_time"; + String originalDateValue = "2020-02-03T"; + String expectedDateValue = "2020-02-03 00:00:00.000"; + + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + @Test public void testKibanaSampleDataLogsUtcDateField() {