From cd8f939f23bc964870a5741154f14df138e7444f Mon Sep 17 00:00:00 2001 From: David Phillips Date: Mon, 20 Jul 2020 17:35:01 -0700 Subject: [PATCH] Remove deprecated split manager methods from Hive --- .../plugin/hive/HiveSplitManager.java | 18 +++------------ .../plugin/hive/AbstractTestHive.java | 22 ++++++++++++------- .../hive/AbstractTestHiveFileSystem.java | 6 ++--- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitManager.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitManager.java index 99f6ebde9dba9..7919bb45956b6 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitManager.java @@ -29,16 +29,15 @@ import io.prestosql.plugin.hive.util.HiveBucketing.HiveBucketFilter; import io.prestosql.spi.PrestoException; import io.prestosql.spi.VersionEmbedder; -import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ConnectorSession; import io.prestosql.spi.connector.ConnectorSplitManager; import io.prestosql.spi.connector.ConnectorSplitSource; import io.prestosql.spi.connector.ConnectorTableHandle; import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.connector.DynamicFilter; import io.prestosql.spi.connector.FixedSplitSource; import io.prestosql.spi.connector.SchemaTableName; import io.prestosql.spi.connector.TableNotFoundException; -import io.prestosql.spi.predicate.TupleDomain; import io.prestosql.spi.type.TypeManager; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -54,7 +53,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.function.Function; -import java.util.function.Supplier; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; @@ -173,23 +171,13 @@ public HiveSplitManager( this.typeManager = requireNonNull(typeManager, "typeManager is null"); } - @Override - public ConnectorSplitSource getSplits( - ConnectorTransactionHandle transaction, - ConnectorSession session, - ConnectorTableHandle tableHandle, - SplitSchedulingStrategy splitSchedulingStrategy) - { - return getSplits(transaction, session, tableHandle, splitSchedulingStrategy, TupleDomain::all); - } - @Override public ConnectorSplitSource getSplits( ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle tableHandle, SplitSchedulingStrategy splitSchedulingStrategy, - Supplier> dynamicFilter) + DynamicFilter dynamicFilter) { HiveTableHandle hiveTable = (HiveTableHandle) tableHandle; SchemaTableName tableName = hiveTable.getSchemaTableName(); @@ -231,7 +219,7 @@ public ConnectorSplitSource getSplits( table, hivePartitions, hiveTable.getCompactEffectivePredicate(), - dynamicFilter, + dynamicFilter::getCurrentPredicate, typeManager, createBucketSplitInfo(bucketHandle, bucketFilter), session, diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java index 49439b38ca931..1f48861bed9cc 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java @@ -84,6 +84,7 @@ import io.prestosql.spi.connector.Constraint; import io.prestosql.spi.connector.ConstraintApplicationResult; import io.prestosql.spi.connector.DiscretePredicates; +import io.prestosql.spi.connector.DynamicFilter; import io.prestosql.spi.connector.ProjectionApplicationResult; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.connector.RecordPageSource; @@ -1371,7 +1372,7 @@ public void testGetPartitionSplitsBatch() metadata.beginQuery(session); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); - ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, tableHandle); assertEquals(getSplitCount(splitSource), tablePartitionFormatPartitions.size()); } @@ -1386,7 +1387,7 @@ public void testGetPartitionSplitsBatchUnpartitioned() metadata.beginQuery(session); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); - ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, tableHandle); assertEquals(getSplitCount(splitSource), 1); } @@ -1396,7 +1397,7 @@ public void testGetPartitionSplitsBatchUnpartitioned() public void testGetPartitionSplitsBatchInvalidTable() { try (Transaction transaction = newTransaction()) { - splitManager.getSplits(transaction.getTransactionHandle(), newSession(), invalidTableHandle, UNGROUPED_SCHEDULING); + getSplits(splitManager, transaction, newSession(), invalidTableHandle); } } @@ -1434,7 +1435,7 @@ public void testGetPartitionSplitsTableOfflinePartition() tableHandle = applyFilter(metadata, tableHandle, new Constraint(tupleDomain)); try { - getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING)); + getSplitCount(getSplits(splitManager, transaction, session, tableHandle)); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { @@ -1455,7 +1456,7 @@ public void testGetPartitionSplitsTableNotReadablePartition() assertNotNull(tableHandle); try { - getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING)); + getSplitCount(getSplits(splitManager, transaction, session, tableHandle)); fail("Expected HiveNotReadableException"); } catch (HiveNotReadableException e) { @@ -1942,7 +1943,7 @@ public void testPartitionSchemaNonCanonical() HivePartition partition = getOnlyElement(((HiveTableHandle) table).getPartitions().orElseThrow(AssertionError::new)); assertEquals(getPartitionId(partition), "t_boolean=0"); - ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, table, UNGROUPED_SCHEDULING); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, table); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); ImmutableList columnHandles = ImmutableList.of(column); @@ -4388,7 +4389,7 @@ private MaterializedResult readTable( throws Exception { tableHandle = applyFilter(transaction.getMetadata(), tableHandle, new Constraint(tupleDomain)); - List splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING)); + List splits = getAllSplits(getSplits(splitManager, transaction, session, tableHandle)); if (expectedSplitCount.isPresent()) { assertEquals(splits.size(), expectedSplitCount.getAsInt()); } @@ -4425,7 +4426,7 @@ protected static int getSplitCount(ConnectorSplitSource splitSource) private List getAllSplits(ConnectorTableHandle tableHandle, Transaction transaction, ConnectorSession session) { - return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING)); + return getAllSplits(getSplits(splitManager, transaction, session, tableHandle)); } protected static List getAllSplits(ConnectorSplitSource splitSource) @@ -4437,6 +4438,11 @@ protected static List getAllSplits(ConnectorSplitSource splitSou return splits.build(); } + protected static ConnectorSplitSource getSplits(ConnectorSplitManager splitManager, Transaction transaction, ConnectorSession session, ConnectorTableHandle tableHandle) + { + return splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING, DynamicFilter.EMPTY); + } + protected String getPartitionId(Object partition) { return ((HivePartition) partition).getPartitionId(); diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveFileSystem.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveFileSystem.java index 8812a6fa2f605..fce134b8ed211 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveFileSystem.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveFileSystem.java @@ -90,6 +90,7 @@ import static io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnHandles; import static io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata; import static io.prestosql.plugin.hive.AbstractTestHive.getAllSplits; +import static io.prestosql.plugin.hive.AbstractTestHive.getSplits; import static io.prestosql.plugin.hive.HiveTestUtils.PAGE_SORTER; import static io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER; import static io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFactories; @@ -98,7 +99,6 @@ import static io.prestosql.plugin.hive.HiveTestUtils.getHiveSession; import static io.prestosql.plugin.hive.HiveTestUtils.getHiveSessionProperties; import static io.prestosql.plugin.hive.HiveTestUtils.getTypes; -import static io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; import static io.prestosql.spi.type.BigintType.BIGINT; import static io.prestosql.testing.MaterializedResult.materializeSourceDataStream; import static io.prestosql.testing.QueryAssertions.assertEqualsIgnoreOrder; @@ -445,7 +445,7 @@ private void createTable(SchemaTableName tableName, HiveStorageFormat storageFor // verify the data metadata.beginQuery(session); - ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, tableHandle); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles, TupleDomain.all())) { @@ -476,7 +476,7 @@ protected MaterializedResult readTable(SchemaTableName tableName) List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, table).values()); metadata.beginQuery(session); - ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, table, UNGROUPED_SCHEDULING); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, table); List allTypes = getTypes(columnHandles); List dataTypes = getTypes(columnHandles.stream()