Skip to content

Commit

Permalink
SQL: Make parsing of date more lenient (#52137)
Browse files Browse the repository at this point in the history 
Make the parsing of date more lenient

- as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}`
- cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)`

Closes: #49379
(cherry picked from commit 5863b27)
  • Loading branch information
@matriv
matriv committed Feb 10, 2020
1 parent 47255c4 commit 6b60085
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 19 deletions.
10 changes: 5 additions & 5 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,9 @@
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly;
import static org.elasticsearch.xpack.sql.util.DateUtils.asTimeOnly;
import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral;
import static org.elasticsearch.xpack.sql.util.DateUtils.dateOfEscapedLiteral;
import static org.elasticsearch.xpack.sql.util.DateUtils.dateTimeOfEscapedLiteral;

abstract class ExpressionBuilder extends IdentifierBuilder {

Expand Down Expand Up @@ -761,9 +761,9 @@ private SqlTypedParamValue param(TerminalNode node) {
public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) {
String string = string(ctx.string());
Source source = source(ctx);
// parse yyyy-MM-dd
// parse yyyy-MM-dd (time optional but is set to 00:00:00.000 because of the conversion to DATE
try {
return new Literal(source, asDateOnly(string), SqlDataTypes.DATE);
return new Literal(source, dateOfEscapedLiteral(string), SqlDataTypes.DATE);
} catch(DateTimeParseException ex) {
throw new ParsingException(source, "Invalid date received; {}", ex.getMessage());
}
Expand All @@ -789,7 +789,7 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx)
Source source = source(ctx);
// parse yyyy-mm-dd hh:mm:ss(.f...)
try {
return new Literal(source, ofEscapedLiteral(string), DataTypes.DATETIME);
return new Literal(source, dateTimeOfEscapedLiteral(string), DataTypes.DATETIME);
} catch (DateTimeParseException ex) {
throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage());
}
Expand Down
56 changes: 50 additions & 6 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,40 @@ public final class DateUtils {
public static final LocalDate EPOCH = LocalDate.of(1970, 1, 1);
public static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L;

private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
private static final DateTimeFormatter DATE_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.appendLiteral(' ')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
private static final DateTimeFormatter DATE_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.appendLiteral('T')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.optionalStart()
.appendLiteral(' ')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.optionalStart()
.appendLiteral('T')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE)
.optionalStart()
.appendZoneOrOffsetId()
.optionalEnd()
.toFormatter().withZone(UTC);
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL)
.optionalStart()
.appendZoneOrOffsetId()
.optionalEnd()
.toFormatter().withZone(UTC);

private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC);
private static final int DEFAULT_PRECISION_FOR_CURRENT_FUNCTIONS = 3;
Expand Down Expand Up @@ -91,7 +115,17 @@ public static ZonedDateTime asDateTime(long millis, ZoneId id) {
* Parses the given string into a Date (SQL DATE type) using UTC as a default timezone.
*/
public static ZonedDateTime asDateOnly(String dateFormat) {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC);
int separatorIdx = dateFormat.indexOf('-');
if (separatorIdx == 0) { // negative year
separatorIdx = dateFormat.indexOf('-', 1);
}
separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
} else {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
}
}

public static ZonedDateTime asDateOnly(ZonedDateTime zdt) {
Expand All @@ -109,13 +143,23 @@ public static ZonedDateTime asDateTime(String dateFormat) {
return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC);
}

public static ZonedDateTime ofEscapedLiteral(String dateFormat) {
public static ZonedDateTime dateOfEscapedLiteral(String dateFormat) {
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return LocalDate.parse(dateFormat, DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
} else {
return LocalDate.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
}
}

public static ZonedDateTime dateTimeOfEscapedLiteral(String dateFormat) {
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL.withZone(UTC));
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_T_LITERAL.withZone(UTC));
} else {
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE.withZone(UTC));
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE.withZone(UTC));
}
}

Expand Down
9 changes: 8 additions & 1 deletion ...ck/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ private String buildDate() {
return sb.toString();
}

private String buildTime() {
if (randomBoolean()) {
return (randomBoolean() ? "T" : " ") + "11:22" + buildSecsAndFractional();
}
return "";
}

private String buildSecsAndFractional() {
if (randomBoolean()) {
return ":55" + randomFrom("", ".1", ".12", ".123", ".1234", ".12345", ".123456",
Expand Down Expand Up @@ -212,7 +219,7 @@ public void testFunctionWithFunctionWithArgAndParams() {
}

public void testDateLiteral() {
Literal l = dateLiteral(buildDate());
Literal l = dateLiteral(buildDate() + buildTime());
assertThat(l.dataType(), is(DATE));
}

Expand Down
31 changes: 24 additions & 7 deletions .../plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,19 +172,37 @@ public void testConversionToDate() {
Converter conversion = converterFor(KEYWORD, to);
assertNull(conversion.convert(null));

assertEquals(date(0L), conversion.convert("1970-01-01"));
assertEquals(date(1483228800000L), conversion.convert("2017-01-01"));
assertEquals(date(-1672531200000L), conversion.convert("1917-01-01"));
assertEquals(date(18000000L), conversion.convert("1970-01-01"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789"));

// double check back and forth conversion
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789"));

assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20+05:00"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123-06:00"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789+03:00"));

assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20+05:00"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123-06:00"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789+03:00"));

// double check back and forth conversion
ZonedDateTime zdt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
Converter forward = converterFor(DATE, KEYWORD);
Converter back = converterFor(KEYWORD, DATE);
assertEquals(asDateOnly(zdt), back.convert(forward.convert(zdt)));
Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-02-"));
assertEquals("cannot cast [2020-02-] to [date]: Text '2020-02-' could not be parsed at index 8", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-"));
assertEquals("cannot cast [2020-] to [date]: Text '2020-' could not be parsed at index 5", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-02-"));
assertEquals("cannot cast [-2020-02-] to [date]: Text '-2020-02-' could not be parsed at index 9", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-"));
assertEquals("cannot cast [-2020-] to [date]: Text '-2020-' could not be parsed at index 6", e.getMessage());
}
}

Expand Down Expand Up @@ -285,7 +303,6 @@ public void testConversionToDateTime() {
assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00"));

// double check back and forth conversion

ZonedDateTime dt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
Converter forward = converterFor(DATETIME, KEYWORD);
Converter back = converterFor(KEYWORD, DATETIME);
Expand Down Expand Up @@ -692,4 +709,4 @@ static ZonedDateTime date(long millisSinceEpoch) {
static OffsetTime time(long millisSinceEpoch) {
return DateUtils.asTimeOnly(millisSinceEpoch);
}
}
}

0 comments on commit 6b60085

Please sign in to comment.