Skip to content

Commit

Permalink
Refactor Hive Split classes for readability
Browse files Browse the repository at this point in the history
  • Loading branch information
ajaygeorge authored and tdcmeehan committed Nov 16, 2023
1 parent 6470fd5 commit 7a64e22
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,12 @@ private ListenableFuture<?> addSplitsToSource(InputSplit[] targetSplits, Interna
return lastResult;
}

private Iterator<InternalHiveSplit> createInternalHiveSplitIterator(Path path, ExtendedFileSystem fileSystem, InternalHiveSplitFactory splitFactory, boolean splittable, Optional<Partition> partition)
private Iterator<InternalHiveSplit> createInternalHiveSplitIterator(
Path path,
ExtendedFileSystem fileSystem,
InternalHiveSplitFactory splitFactory,
boolean splittable,
Optional<Partition> partition)
{
boolean cacheable = isUseListDirectoryCache(session);
if (partition.isPresent()) {
Expand All @@ -368,7 +373,7 @@ private Iterator<InternalHiveSplit> createInternalHiveSplitIterator(Path path, E
hdfsContext.getIdentity(),
buildDirectoryContextProperties(session));
return stream(directoryLister.list(fileSystem, table, path, partition, namenodeStats, hiveDirectoryContext))
.map(status -> splitFactory.createInternalHiveSplit(status, splittable))
.map(hiveFileInfo -> splitFactory.createInternalHiveSplit(hiveFileInfo, splittable))
.filter(Optional::isPresent)
.map(Optional::get)
.iterator();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,35 +84,35 @@ public InternalHiveSplitFactory(
checkArgument(minimumTargetSplitSizeInBytes > 0, "minimumTargetSplitSize must be > 0, found: %s", minimumTargetSplitSize);
}

public Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo fileInfo, boolean splittable)
public Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo hiveFileInfo, boolean splittable)
{
return createInternalHiveSplit(fileInfo, OptionalInt.empty(), OptionalInt.empty(), splittable);
return createInternalHiveSplit(hiveFileInfo, OptionalInt.empty(), OptionalInt.empty(), splittable);
}

public Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo fileInfo, int readBucketNumber, int tableBucketNumber, boolean splittable)
public Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo hiveFileInfo, int readBucketNumber, int tableBucketNumber, boolean splittable)
{
return createInternalHiveSplit(fileInfo, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable);
return createInternalHiveSplit(hiveFileInfo, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable);
}

private Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo fileInfo, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable)
private Optional<InternalHiveSplit> createInternalHiveSplit(HiveFileInfo hiveFileInfo, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable)
{
splittable = splittable &&
fileInfo.getLength() > minimumTargetSplitSizeInBytes &&
hiveFileInfo.getLength() > minimumTargetSplitSizeInBytes &&
(s3SelectPushdownEnabled ?
isSelectSplittable(inputFormat, fileInfo.getPath(), s3SelectPushdownEnabled) :
isSplittable(inputFormat, fileSystem, fileInfo.getPath()));
isSelectSplittable(inputFormat, hiveFileInfo.getPath(), s3SelectPushdownEnabled) :
isSplittable(inputFormat, fileSystem, hiveFileInfo.getPath()));
return createInternalHiveSplit(
fileInfo.getPath(),
fileInfo.getBlockLocations(),
hiveFileInfo.getPath(),
hiveFileInfo.getBlockLocations(),
0,
fileInfo.getLength(),
fileInfo.getLength(),
fileInfo.getFileModifiedTime(),
hiveFileInfo.getLength(),
hiveFileInfo.getLength(),
hiveFileInfo.getFileModifiedTime(),
readBucketNumber,
tableBucketNumber,
splittable,
fileInfo.getExtraFileInfo(),
fileInfo.getCustomSplitInfo());
hiveFileInfo.getExtraFileInfo(),
hiveFileInfo.getCustomSplitInfo());
}

public Optional<InternalHiveSplit> createInternalHiveSplit(FileSplit split)
Expand Down

0 comments on commit 7a64e22

Please sign in to comment.