From 6eb97e2fb4d7cecde715bb9ca98be4ff2c45a0f2 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 12 Jul 2023 00:21:24 -0700 Subject: [PATCH] Remove obsolete FileSystem creation in HiveWriterFactory --- .../plugin/hive/HivePageSinkProvider.java | 5 ----- .../trino/plugin/hive/HiveWriterFactory.java | 18 ------------------ .../io/trino/plugin/hive/AbstractTestHive.java | 1 - .../hive/AbstractTestHiveFileSystem.java | 1 - .../io/trino/plugin/hive/TestHivePageSink.java | 1 - 5 files changed, 26 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java index 3963acfb826e9..e772b27ad6cab 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java @@ -22,7 +22,6 @@ import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.hdfs.HdfsEnvironment; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.HivePageSinkMetadataProvider; import io.trino.plugin.hive.metastore.SortingColumn; @@ -59,7 +58,6 @@ public class HivePageSinkProvider { private final Set fileWriterFactories; private final TrinoFileSystemFactory fileSystemFactory; - private final HdfsEnvironment hdfsEnvironment; private final PageSorter pageSorter; private final HiveMetastoreFactory metastoreFactory; private final PageIndexerFactory pageIndexerFactory; @@ -82,7 +80,6 @@ public class HivePageSinkProvider public HivePageSinkProvider( Set fileWriterFactories, TrinoFileSystemFactory fileSystemFactory, - HdfsEnvironment hdfsEnvironment, PageSorter pageSorter, HiveMetastoreFactory metastoreFactory, PageIndexerFactory pageIndexerFactory, @@ -98,7 +95,6 @@ public HivePageSinkProvider( { this.fileWriterFactories = ImmutableSet.copyOf(requireNonNull(fileWriterFactories, "fileWriterFactories is null")); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); - this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); @@ -178,7 +174,6 @@ private HivePageSink createPageSink(HiveWritableTableHandle handle, boolean isCr handle.getPageSinkMetadata(), new HiveMetastoreClosure(memoizeMetastore(metastoreFactory.createMetastore(Optional.of(session.getIdentity())), perTransactionMetastoreCacheMaximumSize))), typeManager, - hdfsEnvironment, pageSorter, writerSortBufferSize, maxOpenSortFiles, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java index 87cd7bda893db..4c07825cddca5 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java @@ -24,8 +24,6 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.hdfs.HdfsContext; -import io.trino.hdfs.HdfsEnvironment; import io.trino.hive.formats.compression.CompressionKind; import io.trino.plugin.hive.HiveSessionProperties.InsertExistingPartitionsBehavior; import io.trino.plugin.hive.LocationService.WriteInfo; @@ -48,8 +46,6 @@ import io.trino.spi.type.RowType; import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import java.io.IOException; import java.security.Principal; @@ -177,7 +173,6 @@ public HiveWriterFactory( String queryId, HivePageSinkMetadataProvider pageSinkMetadataProvider, TypeManager typeManager, - HdfsEnvironment hdfsEnvironment, PageSorter pageSorter, DataSize sortBufferSize, int maxOpenSortFiles, @@ -244,17 +239,14 @@ public HiveWriterFactory( this.dataColumns = dataColumns.build(); this.isCreateTransactionalTable = isCreateTable && transaction.isTransactional(); - Location writePath; if (isCreateTable) { this.table = null; WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle); checkArgument(writeInfo.writeMode() != DIRECT_TO_TARGET_EXISTING_DIRECTORY, "CREATE TABLE write mode cannot be DIRECT_TO_TARGET_EXISTING_DIRECTORY"); - writePath = writeInfo.writePath(); } else { this.table = pageSinkMetadataProvider.getTable() .orElseThrow(() -> new TrinoException(HIVE_INVALID_METADATA, format("Table '%s.%s' was dropped during insert", schemaName, tableName))); - writePath = locationService.getQueryWriteInfo(locationHandle).writePath(); } this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); @@ -277,16 +269,6 @@ public HiveWriterFactory( .filter(entry -> entry.getValue() != null) .collect(toImmutableMap(Entry::getKey, entry -> entry.getValue().toString())); - Configuration conf = hdfsEnvironment.getConfiguration(new HdfsContext(session), new Path(writePath.toString())); - - // make sure the FileSystem is created with the correct Configuration object - try { - hdfsEnvironment.getFileSystem(session.getIdentity(), new Path(writePath.toString()), conf); - } - catch (IOException e) { - throw new TrinoException(HIVE_FILESYSTEM_ERROR, "Failed getting FileSystem: " + writePath, e); - } - this.hiveWriterStats = requireNonNull(hiveWriterStats, "hiveWriterStats is null"); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index f031bbec26d35..b555694a69d32 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -924,7 +924,6 @@ public Optional getMaterializedView(Connect pageSinkProvider = new HivePageSinkProvider( getDefaultHiveFileWriterFactories(hiveConfig, hdfsEnvironment), new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS), - hdfsEnvironment, PAGE_SORTER, HiveMetastoreFactory.ofInstance(metastoreClient), new GroupByHashPageIndexerFactory(JOIN_COMPILER, TYPE_OPERATORS), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java index 89d5800758b75..d3911e9056bc2 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java @@ -263,7 +263,6 @@ protected void setup(String host, int port, String databaseName, boolean s3Selec pageSinkProvider = new HivePageSinkProvider( getDefaultHiveFileWriterFactories(config, hdfsEnvironment), new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS), - hdfsEnvironment, PAGE_SORTER, HiveMetastoreFactory.ofInstance(metastoreClient), new GroupByHashPageIndexerFactory(new JoinCompiler(typeOperators), typeOperators), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java index afc607405de6b..9ff58fd5812c7 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java @@ -300,7 +300,6 @@ private static ConnectorPageSink createPageSink(HiveTransactionHandle transactio HivePageSinkProvider provider = new HivePageSinkProvider( getDefaultHiveFileWriterFactories(config, HDFS_ENVIRONMENT), HDFS_FILE_SYSTEM_FACTORY, - HDFS_ENVIRONMENT, PAGE_SORTER, HiveMetastoreFactory.ofInstance(metastore), new GroupByHashPageIndexerFactory(new JoinCompiler(typeOperators), typeOperators),