Skip to content

Commit

Permalink
Implement parametric timestamp type
Browse files Browse the repository at this point in the history 
Supports timestamp(p) with p in [0..12]

It introduces a 3-form representation:
* When p <= 3, it uses a long containing the number of milliseconds since the epoch. This
  makes it possible to maintain backward compatibility with existing connectors that rely
  on this encoding
* When 3 < p <= 6, it uses a long containing the number of microseconds since the epoch.
* When 6 < p <= 12, it uses a custom object containing a long+int
  • Loading branch information
@martint
martint committed Jun 6, 2020
1 parent 8d4b906 commit 21e3ddf
Show file tree
Hide file tree
Showing 142 changed files with 7,895 additions and 1,396 deletions.
5 changes: 5 additions & 0 deletions presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import io.prestosql.spi.connector.TableNotFoundException;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.predicate.Marker.Bound;
import io.prestosql.spi.type.TimestampType;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
Expand Down Expand Up @@ -183,6 +184,10 @@ private static void validateColumns(ConnectorTableMetadata meta)
}
}

if (column.getType() instanceof TimestampType && ((TimestampType) column.getType()).getPrecision() != 3) {
throw new PrestoException(NOT_SUPPORTED, format("%s type not supported", column.getType()));
}

columnNameBuilder.add(column.getName().toLowerCase(Locale.ENGLISH));
}

Expand Down
2 changes: 1 addition & 1 deletion presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestJdbcQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ public void testEmptyBuildSql()

private static long toPrestoTimestamp(int year, int month, int day, int hour, int minute, int second)
{
SqlTimestamp sqlTimestamp = DateTimeTestingUtils.sqlTimestampOf(year, month, day, hour, minute, second, 0, UTC, UTC_KEY, SESSION);
SqlTimestamp sqlTimestamp = DateTimeTestingUtils.sqlTimestampOf(3, year, month, day, hour, minute, second, 0, UTC, UTC_KEY, SESSION);
if (SESSION.isLegacyTimestamp()) {
return sqlTimestamp.getMillisUtc();
}
Expand Down
2 changes: 1 addition & 1 deletion presto-docs/src/main/sphinx/language/timestamp.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ Now, Presto treats ``TIMESTAMP`` values as a set of the following fields represe
* ``DAY OF MONTH``
* ``HOUR OF DAY``
* ``MINUTE OF HOUR``
* ``SECOND OF MINUTE`` - as ``DECIMAL(5, 3)``
* ``SECOND OF MINUTE``

For that reason, a ``TIMESTAMP`` value is not linked with the session time zone in any way until
a time zone is needed explicitly, such as when casting to a ``TIMESTAMP WITH TIME ZONE`` or
Expand Down
4 changes: 2 additions & 2 deletions ...to-elasticsearch/src/test/java/io/prestosql/elasticsearch/BaseElasticsearchSmokeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void testDescribeTable()
.row("clerk", "varchar", "", "")
.row("comment", "varchar", "", "")
.row("custkey", "bigint", "", "")
.row("orderdate", "timestamp", "", "")
.row("orderdate", "timestamp(3)", "", "")
.row("orderkey", "bigint", "", "")
.row("orderpriority", "varchar", "", "")
.row("orderstatus", "varchar", "", "")
Expand All @@ -111,7 +111,7 @@ public void testShowCreateTable()
" clerk varchar,\n" +
" comment varchar,\n" +
" custkey bigint,\n" +
" orderdate timestamp,\n" +
" orderdate timestamp(3),\n" +
" orderkey bigint,\n" +
" orderpriority varchar,\n" +
" orderstatus varchar,\n" +
Expand Down
2 changes: 1 addition & 1 deletion presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4199,7 +4199,7 @@ else if (rowNumber % 19 == 1) {
assertNull(row.getField(index));
}
else {
SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123, timeZone, UTC_KEY, SESSION);
SqlTimestamp expected = sqlTimestampOf(3, 2011, 5, 6, 7, 8, 9, 123, timeZone, UTC_KEY, SESSION);
assertEquals(row.getField(index), expected);
}
}
Expand Down
28 changes: 14 additions & 14 deletions presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5216,12 +5216,12 @@ public void testCollectColumnStatisticsOnCreateTable()
" VALUES " +
" (null, null, null, null, null, null, 'p1'), " +
" (null, null, null, null, null, null, 'p1'), " +
" (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), 'p1')," +
" (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), 'p1')," +
" (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00:00.000', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), 'p1')," +
" (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00:00.000', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), 'p1')," +
" (null, null, null, null, null, null, 'p2'), " +
" (null, null, null, null, null, null, 'p2'), " +
" (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), 'p2'), " +
" (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), 'p2') " +
" (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00:00.000', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), 'p2'), " +
" (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00:00.000', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), 'p2') " +
") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar)", tableName), 8);

assertQuery(format("SHOW STATS FOR (SELECT * FROM %s WHERE p_varchar = 'p1')", tableName),
Expand Down Expand Up @@ -6240,23 +6240,23 @@ private void createTableForAnalyzeTest(String tableName, boolean partitioned)
// p_varchar = 'p1', p_bigint = BIGINT '7'
" (null, null, null, null, null, null, 'p1', BIGINT '7'), " +
" (null, null, null, null, null, null, 'p1', BIGINT '7'), " +
" (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', 'abc1', X'bcd1', 'p1', BIGINT '7'), " +
" (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', 'abc2', X'bcd2', 'p1', BIGINT '7'), " +
" (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00:00.000', 'abc1', X'bcd1', 'p1', BIGINT '7'), " +
" (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00:00.000', 'abc2', X'bcd2', 'p1', BIGINT '7'), " +
// p_varchar = 'p2', p_bigint = BIGINT '7'
" (null, null, null, null, null, null, 'p2', BIGINT '7'), " +
" (null, null, null, null, null, null, 'p2', BIGINT '7'), " +
" (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', 'cba1', X'dcb1', 'p2', BIGINT '7'), " +
" (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', 'cba2', X'dcb2', 'p2', BIGINT '7'), " +
" (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00:00.000', 'cba1', X'dcb1', 'p2', BIGINT '7'), " +
" (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00:00.000', 'cba2', X'dcb2', 'p2', BIGINT '7'), " +
// p_varchar = 'p3', p_bigint = BIGINT '8'
" (null, null, null, null, null, null, 'p3', BIGINT '8'), " +
" (null, null, null, null, null, null, 'p3', BIGINT '8'), " +
" (true, BIGINT '3', DOUBLE '4.4', TIMESTAMP '2012-10-10 01:00', 'bca1', X'cdb1', 'p3', BIGINT '8'), " +
" (false, BIGINT '2', DOUBLE '3.4', TIMESTAMP '2012-10-10 00:00', 'bca2', X'cdb2', 'p3', BIGINT '8'), " +
" (true, BIGINT '3', DOUBLE '4.4', TIMESTAMP '2012-10-10 01:00:00.000', 'bca1', X'cdb1', 'p3', BIGINT '8'), " +
" (false, BIGINT '2', DOUBLE '3.4', TIMESTAMP '2012-10-10 00:00:00.000', 'bca2', X'cdb2', 'p3', BIGINT '8'), " +
// p_varchar = NULL, p_bigint = NULL
" (false, BIGINT '7', DOUBLE '7.7', TIMESTAMP '1977-07-07 07:07', 'efa1', X'efa1', NULL, NULL), " +
" (false, BIGINT '6', DOUBLE '6.7', TIMESTAMP '1977-07-07 07:06', 'efa2', X'efa2', NULL, NULL), " +
" (false, BIGINT '5', DOUBLE '5.7', TIMESTAMP '1977-07-07 07:05', 'efa3', X'efa3', NULL, NULL), " +
" (false, BIGINT '4', DOUBLE '4.7', TIMESTAMP '1977-07-07 07:04', 'efa4', X'efa4', NULL, NULL) " +
" (false, BIGINT '7', DOUBLE '7.7', TIMESTAMP '1977-07-07 07:07:00.000', 'efa1', X'efa1', NULL, NULL), " +
" (false, BIGINT '6', DOUBLE '6.7', TIMESTAMP '1977-07-07 07:06:00.000', 'efa2', X'efa2', NULL, NULL), " +
" (false, BIGINT '5', DOUBLE '5.7', TIMESTAMP '1977-07-07 07:05:00.000', 'efa3', X'efa3', NULL, NULL), " +
" (false, BIGINT '4', DOUBLE '4.7', TIMESTAMP '1977-07-07 07:04:00.000', 'efa4', X'efa4', NULL, NULL) " +
") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar, p_bigint)", 16);

if (partitioned) {
Expand Down
2 changes: 1 addition & 1 deletion presto-hive/src/test/java/io/prestosql/plugin/hive/parquet/ParquetTester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,7 @@ private static Object getActualCursorValue(RecordCursor cursor, Type type, int f
return new SqlDate(((Long) fieldFromCursor).intValue());
}
if (TIMESTAMP.equals(type)) {
return new SqlTimestamp((long) fieldFromCursor, UTC_KEY);
return SqlTimestamp.legacyFromMillis(3, (long) fieldFromCursor, UTC_KEY);
}
return fieldFromCursor;
}
Expand Down
4 changes: 3 additions & 1 deletion presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/TypeConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ public final class TypeConverter
.put(RealType.class, Types.FloatType.get())
.put(IntegerType.class, Types.IntegerType.get())
.put(TimeType.class, Types.TimeType.get())
.put(TimestampType.class, Types.TimestampType.withoutZone())
.put(TimestampWithTimeZoneType.class, Types.TimestampType.withZone())
.put(VarcharType.class, Types.StringType.get())
.build();
Expand Down Expand Up @@ -177,6 +176,9 @@ public static org.apache.iceberg.types.Type toIcebergType(Type type)
if (type instanceof MapType) {
return fromMap((MapType) type);
}
if (type.equals(TIMESTAMP)) {
return Types.TimestampType.withoutZone();
}
throw new PrestoException(NOT_SUPPORTED, "Type not supported for Iceberg: " + type.getDisplayName());
}

Expand Down
98 changes: 96 additions & 2 deletions presto-jdbc/src/main/java/io/prestosql/jdbc/PrestoResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
Expand All @@ -65,8 +67,11 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static com.google.common.base.Verify.verify;
import static com.google.common.collect.Iterables.getOnlyElement;
Expand All @@ -81,6 +86,27 @@
public class PrestoResultSet
implements ResultSet
{
private static final Pattern DATETIME_PATTERN = Pattern.compile("" +
"(?<year>\\d\\d\\d\\d)-(?<month>\\d{1,2})-(?<day>\\d{1,2})" +
"(?: (?<hour>\\d{1,2}):(?<minute>\\d{1,2})(?::(?<second>\\d{1,2})(?:\\.(?<fraction>\\d+))?)?)?" +
"\\s*(?<timezone>.+)?");

private static final long[] POWERS_OF_TEN = {
1L,
10L,
100L,
1000L,
10_000L,
100_000L,
1_000_000L,
10_000_000L,
100_000_000L,
1_000_000_000L,
10_000_000_000L,
100_000_000_000L,
1000_000_000_000L
};

static final DateTimeFormatter DATE_FORMATTER = ISODateTimeFormat.date();
static final DateTimeFormatter TIME_FORMATTER = DateTimeFormat.forPattern("HH:mm:ss.SSS");
static final DateTimeFormatter TIME_WITH_TIME_ZONE_FORMATTER = new DateTimeFormatterBuilder()
Expand All @@ -93,6 +119,7 @@ public class PrestoResultSet
.withOffsetParsed();

static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");

static final DateTimeFormatter TIMESTAMP_WITH_TIME_ZONE_FORMATTER = new DateTimeFormatterBuilder()
.append(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ").getPrinter(),
new DateTimeParser[] {
Expand Down Expand Up @@ -329,9 +356,9 @@ private Timestamp getTimestamp(int columnIndex, DateTimeZone localTimeZone)
}

ColumnInfo columnInfo = columnInfo(columnIndex);
if (columnInfo.getColumnTypeName().equalsIgnoreCase("timestamp")) {
if (columnInfo.getColumnTypeSignature().getRawType().equalsIgnoreCase("timestamp")) {
try {
return new Timestamp(TIMESTAMP_FORMATTER.withZone(localTimeZone).parseMillis(String.valueOf(value)));
return parseTimestamp(String.valueOf(value), localTimeZone);
}
catch (IllegalArgumentException e) {
throw new SQLException("Invalid timestamp from server: " + value, e);
Expand Down Expand Up @@ -1924,4 +1951,71 @@ private static List<ColumnInfo> getColumnInfo(List<Column> columns)
}
return list.build();
}

private static Timestamp parseTimestamp(String value, DateTimeZone localTimeZone)
{
Matcher matcher = DATETIME_PATTERN.matcher(value);
if (!matcher.matches() || matcher.group("timezone") != null) {
throw new IllegalArgumentException("Invalid timestamp: " + value);
}

int year = Integer.parseInt(matcher.group("year"));
int month = Integer.parseInt(matcher.group("month"));
int day = Integer.parseInt(matcher.group("day"));
int hour = Integer.parseInt(matcher.group("hour"));
int minute = Integer.parseInt(matcher.group("minute"));
int second = Integer.parseInt(matcher.group("second"));
String fraction = matcher.group("fraction");

long fractionValue = 0;
int precision = 0;
if (fraction != null) {
precision = fraction.length();
fractionValue = Long.parseLong(fraction);
}

long epochSecond = LocalDateTime.of(year, month, day, hour, minute, second, 0)
.atZone(ZoneId.of(localTimeZone.getID()))
.toEpochSecond();

Timestamp timestamp = new Timestamp(epochSecond * 1000);
timestamp.setNanos((int) rescale(fractionValue, precision, 9));
return timestamp;
}

public static long rescale(long value, int fromPrecision, int toPrecision)
{
if (value < 0) {
throw new IllegalArgumentException("value must be >= 0");
}

if (fromPrecision <= toPrecision) {
value *= scaleFactor(fromPrecision, toPrecision);
}
else {
value = roundDiv(value, scaleFactor(toPrecision, fromPrecision));
}

return value;
}

private static long scaleFactor(int fromPrecision, int toPrecision)
{
if (fromPrecision > toPrecision) {
throw new IllegalArgumentException("fromPrecision must be <= toPrecision");
}

return POWERS_OF_TEN[toPrecision - fromPrecision];
}

private static long roundDiv(long value, long factor)
{
checkArgument(factor > 0, "factor must be positive");

if (value >= 0) {
return (value + (factor / 2)) / factor;
}

return (value - (factor / 2)) / factor;
}
}
14 changes: 14 additions & 0 deletions presto-jdbc/src/test/java/io/prestosql/jdbc/TestJdbcResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,20 @@ public void testObjectTypes()
assertEquals(rs.getTimestamp(column), Timestamp.valueOf(LocalDateTime.of(2018, 2, 13, 13, 14, 15, 123_000_000)));
});

checkRepresentation("TIMESTAMP '2018-02-13 13:14:15.111111111111'", Types.TIMESTAMP, (rs, column) -> {
assertEquals(rs.getObject(column), Timestamp.valueOf(LocalDateTime.of(2018, 2, 13, 13, 14, 15, 111_111_111)));
assertThrows(() -> rs.getDate(column));
assertThrows(() -> rs.getTime(column));
assertEquals(rs.getTimestamp(column), Timestamp.valueOf(LocalDateTime.of(2018, 2, 13, 13, 14, 15, 111_111_111)));
});

checkRepresentation("TIMESTAMP '2018-02-13 13:14:15.555555555555'", Types.TIMESTAMP, (rs, column) -> {
assertEquals(rs.getObject(column), Timestamp.valueOf(LocalDateTime.of(2018, 2, 13, 13, 14, 15, 555_555_556)));
assertThrows(() -> rs.getDate(column));
assertThrows(() -> rs.getTime(column));
assertEquals(rs.getTimestamp(column), Timestamp.valueOf(LocalDateTime.of(2018, 2, 13, 13, 14, 15, 555_555_556)));
});

// TODO https://github.com/prestosql/presto/issues/37
// TODO line 1:8: '1970-01-01 00:14:15.123' is not a valid timestamp literal; the expected values will pro
// checkRepresentation("TIMESTAMP '1970-01-01 00:14:15.123'", Types.TIMESTAMP, (rs, column) -> {
Expand Down
4 changes: 2 additions & 2 deletions presto-kafka/src/test/java/io/prestosql/plugin/kafka/util/KafkaLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.util.concurrent.atomic.AtomicReference;

import static com.google.common.base.Preconditions.checkState;
import static io.prestosql.operator.scalar.timestamp.VarcharToTimestampCast.castToLegacyShortTimestamp;
import static io.prestosql.spi.type.BigintType.BIGINT;
import static io.prestosql.spi.type.BooleanType.BOOLEAN;
import static io.prestosql.spi.type.DateTimeEncoding.unpackMillisUtc;
Expand All @@ -46,7 +47,6 @@
import static io.prestosql.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
import static io.prestosql.util.DateTimeUtils.convertToLegacyTimestamp;
import static io.prestosql.util.DateTimeUtils.convertToTimestampWithTimeZone;
import static io.prestosql.util.DateTimeUtils.parseLegacyTime;
import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -146,7 +146,7 @@ private Object convertValue(Object value, Type type)
return ISO8601_FORMATTER.print(parseLegacyTime(timeZoneKey, (String) value));
}
if (TIMESTAMP.equals(type)) {
return ISO8601_FORMATTER.print(convertToLegacyTimestamp(timeZoneKey, (String) value));
return ISO8601_FORMATTER.print(castToLegacyShortTimestamp(TIMESTAMP.getPrecision(), timeZoneKey, (String) value));
}
if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
return ISO8601_FORMATTER.print(unpackMillisUtc(convertToTimestampWithTimeZone(timeZoneKey, (String) value)));
Expand Down
Loading

0 comments on commit 21e3ddf

Please sign in to comment.