From 7a64e22feb6899fbb398b136b68482ad5db019e3 Mon Sep 17 00:00:00 2001 From: Ajay George Date: Wed, 15 Nov 2023 21:27:09 -0500 Subject: [PATCH] Refactor Hive Split classes for readability --- .../presto/hive/StoragePartitionLoader.java | 9 ++++-- .../hive/util/InternalHiveSplitFactory.java | 30 +++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java index 591b7d74923a..9985a7a54cb2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java @@ -354,7 +354,12 @@ private ListenableFuture addSplitsToSource(InputSplit[] targetSplits, Interna return lastResult; } - private Iterator createInternalHiveSplitIterator(Path path, ExtendedFileSystem fileSystem, InternalHiveSplitFactory splitFactory, boolean splittable, Optional partition) + private Iterator createInternalHiveSplitIterator( + Path path, + ExtendedFileSystem fileSystem, + InternalHiveSplitFactory splitFactory, + boolean splittable, + Optional partition) { boolean cacheable = isUseListDirectoryCache(session); if (partition.isPresent()) { @@ -368,7 +373,7 @@ private Iterator createInternalHiveSplitIterator(Path path, E hdfsContext.getIdentity(), buildDirectoryContextProperties(session)); return stream(directoryLister.list(fileSystem, table, path, partition, namenodeStats, hiveDirectoryContext)) - .map(status -> splitFactory.createInternalHiveSplit(status, splittable)) + .map(hiveFileInfo -> splitFactory.createInternalHiveSplit(hiveFileInfo, splittable)) .filter(Optional::isPresent) .map(Optional::get) .iterator(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java index 958c7f802149..30a74a600e93 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java @@ -84,35 +84,35 @@ public InternalHiveSplitFactory( checkArgument(minimumTargetSplitSizeInBytes > 0, "minimumTargetSplitSize must be > 0, found: %s", minimumTargetSplitSize); } - public Optional createInternalHiveSplit(HiveFileInfo fileInfo, boolean splittable) + public Optional createInternalHiveSplit(HiveFileInfo hiveFileInfo, boolean splittable) { - return createInternalHiveSplit(fileInfo, OptionalInt.empty(), OptionalInt.empty(), splittable); + return createInternalHiveSplit(hiveFileInfo, OptionalInt.empty(), OptionalInt.empty(), splittable); } - public Optional createInternalHiveSplit(HiveFileInfo fileInfo, int readBucketNumber, int tableBucketNumber, boolean splittable) + public Optional createInternalHiveSplit(HiveFileInfo hiveFileInfo, int readBucketNumber, int tableBucketNumber, boolean splittable) { - return createInternalHiveSplit(fileInfo, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable); + return createInternalHiveSplit(hiveFileInfo, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable); } - private Optional createInternalHiveSplit(HiveFileInfo fileInfo, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable) + private Optional createInternalHiveSplit(HiveFileInfo hiveFileInfo, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable) { splittable = splittable && - fileInfo.getLength() > minimumTargetSplitSizeInBytes && + hiveFileInfo.getLength() > minimumTargetSplitSizeInBytes && (s3SelectPushdownEnabled ? - isSelectSplittable(inputFormat, fileInfo.getPath(), s3SelectPushdownEnabled) : - isSplittable(inputFormat, fileSystem, fileInfo.getPath())); + isSelectSplittable(inputFormat, hiveFileInfo.getPath(), s3SelectPushdownEnabled) : + isSplittable(inputFormat, fileSystem, hiveFileInfo.getPath())); return createInternalHiveSplit( - fileInfo.getPath(), - fileInfo.getBlockLocations(), + hiveFileInfo.getPath(), + hiveFileInfo.getBlockLocations(), 0, - fileInfo.getLength(), - fileInfo.getLength(), - fileInfo.getFileModifiedTime(), + hiveFileInfo.getLength(), + hiveFileInfo.getLength(), + hiveFileInfo.getFileModifiedTime(), readBucketNumber, tableBucketNumber, splittable, - fileInfo.getExtraFileInfo(), - fileInfo.getCustomSplitInfo()); + hiveFileInfo.getExtraFileInfo(), + hiveFileInfo.getCustomSplitInfo()); } public Optional createInternalHiveSplit(FileSplit split)