From dfd2f2439f11df405ea90199ed3b11e25697dc7c Mon Sep 17 00:00:00 2001 From: Reetika Agrawal Date: Tue, 4 May 2021 16:57:49 +0530 Subject: [PATCH] Add hidden column $file_modified_time with Hive table Cherry pick of trinodb/trino#1428 Co-authored-by: Praveen2112 --- .../facebook/presto/hive/HiveFileInfo.java | 10 ++- .../presto/hive/HiveColumnHandle.java | 15 +++++ .../facebook/presto/hive/HiveMetadata.java | 2 + .../presto/hive/HivePageSourceProvider.java | 13 ++-- .../com/facebook/presto/hive/HiveSplit.java | 9 +++ .../facebook/presto/hive/HiveSplitSource.java | 1 + .../com/facebook/presto/hive/HiveUtil.java | 8 ++- .../presto/hive/InternalHiveSplit.java | 8 +++ .../hive/util/InternalHiveSplitFactory.java | 4 ++ .../presto/hive/TestDynamicPruning.java | 2 + .../presto/hive/TestHiveFileFormats.java | 3 + .../hive/TestHiveIntegrationSmokeTest.java | 66 +++++++++++++++++-- .../presto/hive/TestHivePageSink.java | 1 + .../hive/TestHivePageSourceProvider.java | 3 + .../facebook/presto/hive/TestHiveSplit.java | 2 + .../presto/hive/TestHiveSplitSource.java | 2 + .../TestOrcBatchPageSourceMemoryTracking.java | 2 + 17 files changed, 138 insertions(+), 13 deletions(-) diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveFileInfo.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveFileInfo.java index 7027495c2057..231272f4d02b 100644 --- a/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveFileInfo.java +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveFileInfo.java @@ -28,6 +28,7 @@ public class HiveFileInfo private final boolean isDirectory; private final BlockLocation[] blockLocations; private final long length; + private final long fileModifiedTime; private final Optional extraFileInfo; public static HiveFileInfo createHiveFileInfo(LocatedFileStatus locatedFileStatus, Optional extraFileContext) @@ -37,15 +38,17 @@ public static HiveFileInfo createHiveFileInfo(LocatedFileStatus locatedFileStatu locatedFileStatus.isDirectory(), locatedFileStatus.getBlockLocations(), locatedFileStatus.getLen(), + locatedFileStatus.getModificationTime(), extraFileContext); } - private HiveFileInfo(Path path, boolean isDirectory, BlockLocation[] blockLocations, long length, Optional extraFileInfo) + private HiveFileInfo(Path path, boolean isDirectory, BlockLocation[] blockLocations, long length, long fileModifiedTime, Optional extraFileInfo) { this.path = requireNonNull(path, "path is null"); this.isDirectory = isDirectory; this.blockLocations = blockLocations; this.length = length; + this.fileModifiedTime = fileModifiedTime; this.extraFileInfo = requireNonNull(extraFileInfo, "extraFileInfo is null"); } @@ -69,6 +72,11 @@ public long getLength() return length; } + public long getFileModifiedTime() + { + return fileModifiedTime; + } + public Optional getExtraFileInfo() { return extraFileInfo; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java index e24d3512d55a..6553d285a0c4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java @@ -57,6 +57,11 @@ public class HiveColumnHandle public static final HiveType FILE_SIZE_TYPE = HIVE_LONG; public static final TypeSignature FILE_SIZE_TYPE_SIGNATURE = FILE_SIZE_TYPE.getTypeSignature(); + public static final int FILE_MODIFIED_TIME_COLUMN_INDEX = -14; + public static final String FILE_MODIFIED_TIME_COLUMN_NAME = "$file_modified_time"; + public static final HiveType FILE_MODIFIED_TIME_TYPE = HIVE_LONG; + public static final TypeSignature FILE_MODIFIED_TIME_TYPE_SIGNATURE = FILE_MODIFIED_TIME_TYPE.getTypeSignature(); + private static final String UPDATE_ROW_ID_COLUMN_NAME = "$shard_row_id"; // Ids <= this can be used for distinguishing between different prefilled columns. @@ -253,6 +258,11 @@ public static HiveColumnHandle fileSizeColumnHandle() return new HiveColumnHandle(FILE_SIZE_COLUMN_NAME, FILE_SIZE_TYPE, FILE_SIZE_TYPE_SIGNATURE, FILE_SIZE_COLUMN_INDEX, SYNTHESIZED, Optional.empty(), ImmutableList.of(), Optional.empty()); } + public static HiveColumnHandle fileModifiedTimeColumnHandle() + { + return new HiveColumnHandle(FILE_MODIFIED_TIME_COLUMN_NAME, FILE_MODIFIED_TIME_TYPE, FILE_MODIFIED_TIME_TYPE_SIGNATURE, FILE_MODIFIED_TIME_COLUMN_INDEX, SYNTHESIZED, Optional.empty(), ImmutableList.of(), Optional.empty()); + } + public static boolean isPathColumnHandle(HiveColumnHandle column) { return column.getHiveColumnIndex() == PATH_COLUMN_INDEX; @@ -281,4 +291,9 @@ public static boolean isFileSizeColumnHandle(HiveColumnHandle column) { return column.getHiveColumnIndex() == FILE_SIZE_COLUMN_INDEX; } + + public static boolean isFileModifiedTimeColumnHandle(HiveColumnHandle column) + { + return column.getHiveColumnIndex() == FILE_MODIFIED_TIME_COLUMN_INDEX; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 9c92b55a9981..44fef1dbf24a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -179,6 +179,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED; +import static com.facebook.presto.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle; @@ -3618,6 +3619,7 @@ private static Function columnMetadataGetter(T } builder.put(FILE_SIZE_COLUMN_NAME, Optional.empty()); + builder.put(FILE_MODIFIED_TIME_COLUMN_NAME, Optional.empty()); Map> columnComment = builder.build(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index 11a0f9617662..f980ee6f0f12 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -191,6 +191,7 @@ public ConnectorPageSource createPageSource( hiveSplit.getStart(), hiveSplit.getLength(), hiveSplit.getFileSize(), + hiveSplit.getFileModifiedTime(), hiveSplit.getStorage(), splitContext.getDynamicFilterPredicate().map(filter -> filter.transform(handle -> (HiveColumnHandle) handle).intersect(effectivePredicate)).orElse(effectivePredicate), selectedColumns, @@ -269,7 +270,8 @@ private static Optional createSelectivePageSource( split.getTableToPartitionMapping(), path, split.getTableBucketNumber(), - split.getFileSize()); + split.getFileSize(), + split.getFileModifiedTime()); Optional bucketAdaptation = split.getBucketConversion().map(conversion -> toBucketAdaptation(conversion, columnMappings, split.getTableBucketNumber(), mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex())); @@ -336,6 +338,7 @@ public static Optional createHivePageSource( long start, long length, long fileSize, + long fileModifiedTime, Storage storage, TupleDomain effectivePredicate, List hiveColumns, @@ -382,7 +385,8 @@ public static Optional createHivePageSource( tableToPartitionMapping, path, tableBucketNumber, - fileSize); + fileSize, + fileModifiedTime); Set outputIndices = hiveColumns.stream() .map(HiveColumnHandle::getHiveColumnIndex) @@ -668,7 +672,8 @@ public static List buildColumnMappings( TableToPartitionMapping tableToPartitionMapping, Path path, OptionalInt bucketNumber, - long fileSize) + long fileSize, + long fileModifiedTime) { Map partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey::getName); int regularIndex = 0; @@ -706,7 +711,7 @@ else if (isPushedDownSubfield(column)) { else { columnMappings.add(prefilled( column, - getPrefilledColumnValue(column, partitionKeysByName.get(column.getName()), path, bucketNumber, fileSize), + getPrefilledColumnValue(column, partitionKeysByName.get(column.getName()), path, bucketNumber, fileSize, fileModifiedTime), coercionFrom)); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java index cccee8a71e23..e14aa151664a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java @@ -46,6 +46,7 @@ public class HiveSplit private final long start; private final long length; private final long fileSize; + private final long fileModifiedTime; private final Storage storage; private final List partitionKeys; private final List addresses; @@ -74,6 +75,7 @@ public HiveSplit( @JsonProperty("start") long start, @JsonProperty("length") long length, @JsonProperty("fileSize") long fileSize, + @JsonProperty("fileModifiedTime") long fileModifiedTime, @JsonProperty("storage") Storage storage, @JsonProperty("partitionKeys") List partitionKeys, @JsonProperty("addresses") List addresses, @@ -117,6 +119,7 @@ public HiveSplit( this.start = start; this.length = length; this.fileSize = fileSize; + this.fileModifiedTime = fileModifiedTime; this.storage = storage; this.partitionKeys = ImmutableList.copyOf(partitionKeys); this.addresses = ImmutableList.copyOf(addresses); @@ -176,6 +179,12 @@ public long getFileSize() return fileSize; } + @JsonProperty + public long getFileModifiedTime() + { + return fileModifiedTime; + } + @JsonProperty public Storage getStorage() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java index 0d00d807de33..261898112ea4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java @@ -491,6 +491,7 @@ else if (maxSplitBytes * 2 >= remainingBlockBytes) { internalSplit.getStart(), splitBytes, internalSplit.getFileSize(), + internalSplit.getFileModifiedTime(), internalSplit.getPartitionInfo().getStorage(), internalSplit.getPartitionKeys(), block.getAddresses(), diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 72add0dbb928..76690186ac27 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -130,8 +130,10 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX; import static com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle; +import static com.facebook.presto.hive.HiveColumnHandle.fileModifiedTimeColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.fileSizeColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.isBucketColumnHandle; +import static com.facebook.presto.hive.HiveColumnHandle.isFileModifiedTimeColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.isFileSizeColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle; @@ -890,6 +892,7 @@ public static List hiveColumnHandles(Table table) columns.add(bucketColumnHandle()); } columns.add(fileSizeColumnHandle()); + columns.add(fileModifiedTimeColumnHandle()); return columns.build(); } @@ -934,7 +937,7 @@ public static String columnExtraInfo(boolean partitionKey) return partitionKey ? "partition key" : null; } - public static Optional getPrefilledColumnValue(HiveColumnHandle columnHandle, HivePartitionKey partitionKey, Path path, OptionalInt bucketNumber, long fileSize) + public static Optional getPrefilledColumnValue(HiveColumnHandle columnHandle, HivePartitionKey partitionKey, Path path, OptionalInt bucketNumber, long fileSize, long fileModifiedTime) { if (partitionKey != null) { return partitionKey.getValue(); @@ -951,6 +954,9 @@ public static Optional getPrefilledColumnValue(HiveColumnHandle columnHa if (isFileSizeColumnHandle(columnHandle)) { return Optional.of(String.valueOf(fileSize)); } + if (isFileModifiedTimeColumnHandle(columnHandle)) { + return Optional.of(String.valueOf(fileModifiedTime)); + } throw new PrestoException(NOT_SUPPORTED, "unsupported hidden column: " + columnHandle); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java b/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java index 14a0e6808dd4..993d7662fe6d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java @@ -48,6 +48,7 @@ public class InternalHiveSplit private final byte[] relativeUri; private final long end; private final long fileSize; + private final long fileModifiedTime; // encode the hive blocks as an array of longs and list of list of addresses to save memory //if all blockAddress lists are empty, store only the empty list @@ -75,6 +76,7 @@ public InternalHiveSplit( long start, long end, long fileSize, + long fileModifiedTime, List blocks, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, @@ -101,6 +103,7 @@ public InternalHiveSplit( this.start = start; this.end = end; this.fileSize = fileSize; + this.fileModifiedTime = fileModifiedTime; this.readBucketNumber = readBucketNumber.orElse(-1); this.tableBucketNumber = tableBucketNumber.orElse(-1); this.splittable = splittable; @@ -146,6 +149,11 @@ public long getFileSize() return fileSize; } + public long getFileModifiedTime() + { + return fileModifiedTime; + } + public boolean isS3SelectPushdownEnabled() { return s3SelectPushdownEnabled; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java index c78380275009..6ab415fe71e7 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java @@ -105,6 +105,7 @@ private Optional createInternalHiveSplit(HiveFileInfo fileInf 0, fileInfo.getLength(), fileInfo.getLength(), + fileInfo.getFileModifiedTime(), readBucketNumber, tableBucketNumber, splittable, @@ -123,6 +124,7 @@ public Optional createInternalHiveSplit(FileSplit split) split.getStart(), split.getLength(), file.getLen(), + file.getModificationTime(), OptionalInt.empty(), OptionalInt.empty(), false, @@ -136,6 +138,7 @@ private Optional createInternalHiveSplit( long start, long length, long fileSize, + long fileModificationTime, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable, @@ -195,6 +198,7 @@ private Optional createInternalHiveSplit( start, start + length, fileSize, + fileModificationTime, blocks, readBucketNumber, tableBucketNumber, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestDynamicPruning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestDynamicPruning.java index b2e9474c8fe8..3dd565d61430 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestDynamicPruning.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestDynamicPruning.java @@ -35,6 +35,7 @@ import java.io.File; import java.io.IOException; +import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; @@ -130,6 +131,7 @@ private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle 0, outputFile.length(), outputFile.length(), + Instant.now().toEpochMilli(), new Storage( StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), "location", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java index 5a76cef4519a..1743d28d3f95 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java @@ -61,6 +61,7 @@ import java.io.File; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -913,6 +914,7 @@ private void testCursorProvider(HiveRecordCursorProvider cursorProvider, split.getStart(), split.getLength(), split.getLength(), + Instant.now().toEpochMilli(), new Storage( StorageFormat.create(storageFormat.getSerDe(), storageFormat.getInputFormat(), storageFormat.getOutputFormat()), "location", @@ -977,6 +979,7 @@ private void testPageSourceFactory(HiveBatchPageSourceFactory sourceFactory, split.getStart(), split.getLength(), split.getLength(), + Instant.now().toEpochMilli(), new Storage( StorageFormat.create(storageFormat.getSerDe(), storageFormat.getInputFormat(), storageFormat.getOutputFormat()), "location", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index e596065688af..860d6d19d036 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -61,6 +61,7 @@ import java.io.File; import java.math.BigDecimal; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; @@ -94,6 +95,7 @@ import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.common.type.VarcharType.createVarcharType; import static com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME; +import static com.facebook.presto.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; @@ -2745,7 +2747,7 @@ private void testPathHiddenColumn(Session session, HiveStorageFormat storageForm TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_path"); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); - List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME); + List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME, FILE_MODIFIED_TIME_COLUMN_NAME); List columnMetadatas = tableMetadata.getColumns(); assertEquals(columnMetadatas.size(), columnNames.size()); for (int i = 0; i < columnMetadatas.size(); i++) { @@ -2803,7 +2805,7 @@ public void testBucketHiddenColumn() assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("col0")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 2); - List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, BUCKET_COLUMN_NAME, FILE_SIZE_COLUMN_NAME); + List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, BUCKET_COLUMN_NAME, FILE_SIZE_COLUMN_NAME, FILE_MODIFIED_TIME_COLUMN_NAME); List columnMetadatas = tableMetadata.getColumns(); assertEquals(columnMetadatas.size(), columnNames.size()); for (int i = 0; i < columnMetadatas.size(); i++) { @@ -2840,9 +2842,7 @@ public void testBucketHiddenColumn() public void testFileSizeHiddenColumn() { @Language("SQL") String createTable = "CREATE TABLE test_file_size " + - "WITH (" + - "partitioned_by = ARRAY['col1']" + - ") AS " + + "AS " + "SELECT * FROM (VALUES " + "(0, 0), (3, 0), (6, 0), " + "(1, 1), (4, 1), (7, 1), " + @@ -2853,7 +2853,7 @@ public void testFileSizeHiddenColumn() TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_file_size"); - List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME); + List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME, FILE_MODIFIED_TIME_COLUMN_NAME); List columnMetadatas = tableMetadata.getColumns(); assertEquals(columnMetadatas.size(), columnNames.size()); for (int i = 0; i < columnMetadatas.size(); i++) { @@ -2863,7 +2863,6 @@ public void testFileSizeHiddenColumn() assertTrue(columnMetadata.isHidden()); } } - assertEquals(getPartitions("test_file_size").size(), 3); MaterializedResult results = computeActual(format("SELECT *, \"%s\" FROM test_file_size", FILE_SIZE_COLUMN_NAME)); Map fileSizeMap = new HashMap<>(); @@ -2887,6 +2886,59 @@ public void testFileSizeHiddenColumn() assertUpdate("DROP TABLE test_file_size"); } + @Test + public void testFileModifiedTimeHiddenColumn() + { + long testStartTime = Instant.now().toEpochMilli(); + + @Language("SQL") String createTable = "CREATE TABLE test_file_modified_time " + + "WITH (" + + "partitioned_by = ARRAY['col1']" + + ") AS " + + "SELECT * FROM (VALUES " + + "(0, 0), (3, 0), (6, 0), " + + "(1, 1), (4, 1), (7, 1), " + + "(2, 2), (5, 2) " + + " ) t(col0, col1) "; + assertUpdate(createTable, 8); + assertTrue(getQueryRunner().tableExists(getSession(), "test_file_modified_time")); + + TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_file_modified_time"); + + List columnNames = ImmutableList.of("col0", "col1", PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME, FILE_MODIFIED_TIME_COLUMN_NAME); + List columnMetadatas = tableMetadata.getColumns(); + assertEquals(columnMetadatas.size(), columnNames.size()); + for (int i = 0; i < columnMetadatas.size(); i++) { + ColumnMetadata columnMetadata = columnMetadatas.get(i); + assertEquals(columnMetadata.getName(), columnNames.get(i)); + if (columnMetadata.getName().equals(FILE_MODIFIED_TIME_COLUMN_NAME)) { + assertTrue(columnMetadata.isHidden()); + } + } + assertEquals(getPartitions("test_file_modified_time").size(), 3); + + MaterializedResult results = computeActual(format("SELECT *, \"%s\" FROM test_file_modified_time", FILE_MODIFIED_TIME_COLUMN_NAME)); + Map fileModifiedTimeMap = new HashMap<>(); + for (int i = 0; i < results.getRowCount(); i++) { + MaterializedRow row = results.getMaterializedRows().get(i); + int col0 = (int) row.getField(0); + int col1 = (int) row.getField(1); + long fileModifiedTime = (Long) row.getField(2); + + assertTrue(fileModifiedTime > (testStartTime - 2_000)); + assertEquals(col0 % 3, col1); + if (fileModifiedTimeMap.containsKey(col1)) { + assertEquals(fileModifiedTimeMap.get(col1).longValue(), fileModifiedTime); + } + else { + fileModifiedTimeMap.put(col1, fileModifiedTime); + } + } + assertEquals(fileModifiedTimeMap.size(), 3); + + assertUpdate("DROP TABLE test_file_modified_time"); + } + @Test public void testDeleteAndInsert() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java index 5e12ab781a34..094b754b1bb3 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java @@ -233,6 +233,7 @@ private static ConnectorPageSource createPageSource(HiveTransactionHandle transa 0, outputFile.length(), outputFile.length(), + outputFile.lastModified(), new Storage( StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), "location", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSourceProvider.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSourceProvider.java index dfd63775a9ce..25019a1db378 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSourceProvider.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSourceProvider.java @@ -21,6 +21,7 @@ import io.airlift.units.DataSize; import org.testng.annotations.Test; +import java.time.Instant; import java.util.Optional; import java.util.OptionalInt; @@ -49,6 +50,7 @@ public void testGenerateCacheQuota() 0, 10, 10, + Instant.now().toEpochMilli(), new Storage( StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), "location", @@ -83,6 +85,7 @@ public void testGenerateCacheQuota() 0, 10, 10, + Instant.now().toEpochMilli(), new Storage( StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), "location", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java index dcd847a6f940..71539d275933 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java @@ -42,6 +42,7 @@ import com.google.inject.Scopes; import org.testng.annotations.Test; +import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; @@ -86,6 +87,7 @@ public void testJsonRoundTrip() 42, 87, 88, + Instant.now().toEpochMilli(), new Storage( StorageFormat.create("serde", "input", "output"), "location", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java index 135bb6649699..840aa9523d02 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.testng.annotations.Test; +import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.OptionalInt; @@ -524,6 +525,7 @@ private TestSplit(int id, OptionalInt bucketNumber, DataSize fileSize) 0, fileSize.toBytes(), fileSize.toBytes(), + Instant.now().toEpochMilli(), ImmutableList.of(new InternalHiveBlock(fileSize.toBytes(), ImmutableList.of())), bucketNumber, bucketNumber, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java index 3d9486c8411c..49810a3803bc 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java @@ -87,6 +87,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -460,6 +461,7 @@ public ConnectorPageSource newPageSource(FileFormatDataSourceStats stats, Connec fileSplit.getStart(), fileSplit.getLength(), fileSplit.getLength(), + Instant.now().toEpochMilli(), storage, TupleDomain.all(), columns,