Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Minor cleanup in Hudi #24067

Merged
merged 2 commits into from
Nov 8, 2024
Merged

Minor cleanup in Hudi #24067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1b0b451
Remove redundant code from Hudi
ebyhr Nov 8, 2024
90e1b02
Convert some classes to record in Hudi
ebyhr Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,23 +143,23 @@ public Optional<SystemTable> getSystemTable(ConnectorSession session, SchemaTabl
private Optional<SystemTable> getRawSystemTable(SchemaTableName tableName, ConnectorSession session)
{
HudiTableName name = HudiTableName.from(tableName.getTableName());
if (name.getTableType() == TableType.DATA) {
if (name.tableType() == TableType.DATA) {
return Optional.empty();
}

Optional<Table> tableOptional = metastore.getTable(tableName.getSchemaName(), name.getTableName());
Optional<Table> tableOptional = metastore.getTable(tableName.getSchemaName(), name.tableName());
if (tableOptional.isEmpty()) {
return Optional.empty();
}
if (!isHudiTable(tableOptional.get())) {
return Optional.empty();
}
return switch (name.getTableType()) {
return switch (name.tableType()) {
case DATA ->
// TODO (https://github.com/trinodb/trino/issues/17973) remove DATA table type
Optional.empty();
case TIMELINE -> {
SchemaTableName systemTableName = new SchemaTableName(tableName.getSchemaName(), name.getTableNameWithType());
SchemaTableName systemTableName = new SchemaTableName(tableName.getSchemaName(), name.tableNameWithType());
yield Optional.of(new TimelineTable(fileSystemFactory.create(session), systemTableName, tableOptional.get()));
}
};
Expand Down Expand Up @@ -228,9 +228,10 @@ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTable
}

@Override
public Optional<Object> getInfo(ConnectorTableHandle table)
public Optional<Object> getInfo(ConnectorTableHandle tableHandle)
{
return Optional.of(HudiTableInfo.from((HudiTableHandle) table));
HudiTableHandle table = (HudiTableHandle) tableHandle;
return Optional.of(new HudiTableInfo(table.getSchemaTableName(), table.getTableType().name(), table.getBasePath()));
}

@Override
Expand Down
47 changes: 5 additions & 42 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,53 +13,16 @@
*/
package io.trino.plugin.hudi;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.trino.spi.connector.SchemaTableName;

import static java.util.Objects.requireNonNull;

public class HudiTableInfo
public record HudiTableInfo(SchemaTableName table, String tableType, String basePath)
{
private final SchemaTableName table;
private final String tableType;
private final String basePath;

@JsonCreator
public HudiTableInfo(
@JsonProperty("table") SchemaTableName table,
@JsonProperty("tableType") String tableType,
@JsonProperty("basePath") String basePath)
{
this.table = requireNonNull(table, "table is null");
this.tableType = requireNonNull(tableType, "tableType is null");
this.basePath = requireNonNull(basePath, "basePath is null");
}

@JsonProperty
public SchemaTableName getTable()
{
return table;
}

@JsonProperty
public String getTableType()
{
return tableType;
}

@JsonProperty
public String getBasePath()
{
return basePath;
}

public static HudiTableInfo from(HudiTableHandle tableHandle)
public HudiTableInfo
{
requireNonNull(tableHandle, "tableHandle is null");
return new HudiTableInfo(
tableHandle.getSchemaTableName(),
tableHandle.getTableType().name(),
tableHandle.getBasePath());
requireNonNull(table, "table is null");
requireNonNull(tableType, "tableType is null");
requireNonNull(basePath, "basePath is null");
}
}
19 changes: 3 additions & 16 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiTableName.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,40 +23,27 @@
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;

public class HudiTableName
public record HudiTableName(String tableName, TableType tableType)
{
private static final Pattern TABLE_PATTERN = Pattern.compile("" +
"(?<table>[^$@]+)" +
"(?:\\$(?<type>[^@]+))?");

private final String tableName;
private final TableType tableType;

public HudiTableName(String tableName, TableType tableType)
{
this.tableName = requireNonNull(tableName, "tableName is null");
this.tableType = requireNonNull(tableType, "tableType is null");
}

public String getTableName()
{
return tableName;
}

public TableType getTableType()
{
return tableType;
}

public String getTableNameWithType()
public String tableNameWithType()
{
return tableName + "$" + tableType.name().toLowerCase(ENGLISH);
}

@Override
public String toString()
{
return getTableNameWithType();
return tableNameWithType();
}

public static HudiTableName from(String name)
Expand Down
2 changes: 1 addition & 1 deletion plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/TableType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
public enum TableType
{
DATA,
TIMELINE;
TIMELINE
}
7 changes: 1 addition & 6 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/compaction/CompactionOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,9 @@ public CompactionOperation(
this.bootstrapFilePath = requireNonNull(bootstrapFilePath, "bootstrapFilePath is null");
}

public String getFileId()
{
return id.getFileId();
}

public String getPartitionPath()
{
return id.getPartitionPath();
return id.partitionPath();
}

public HudiFileGroupId getFileGroupId()
Expand Down
5 changes: 0 additions & 5 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/files/FileSlice.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,6 @@ public Optional<HudiBaseFile> getBaseFile()
return baseFile;
}

public boolean isEmpty()
{
return (baseFile == null) && logFiles.isEmpty();
}

@Override
public String toString()
{
Expand Down
2 changes: 1 addition & 1 deletion plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/files/HudiFileGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public void addLogFile(HudiLogFile logFile)

public String getPartitionPath()
{
return fileGroupId.getPartitionPath();
return fileGroupId.partitionPath();
}

public HudiFileGroupId getFileGroupId()
Expand Down
43 changes: 4 additions & 39 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/files/HudiFileGroupId.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,50 +13,15 @@
*/
package io.trino.plugin.hudi.files;

import java.util.Objects;

import static java.util.Objects.requireNonNull;

public class HudiFileGroupId
public record HudiFileGroupId(String partitionPath, String fileId)
implements Comparable<HudiFileGroupId>
{
private final String partitionPath;

private final String fileId;

public HudiFileGroupId(String partitionPath, String fileId)
{
this.partitionPath = requireNonNull(partitionPath, "partitionPath is null");
this.fileId = requireNonNull(fileId, "partitionPath is null");
}

public String getPartitionPath()
{
return partitionPath;
}

public String getFileId()
{
return fileId;
}

@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
HudiFileGroupId that = (HudiFileGroupId) o;
return Objects.equals(partitionPath, that.partitionPath) && Objects.equals(fileId, that.fileId);
}

@Override
public int hashCode()
public HudiFileGroupId
{
return Objects.hash(partitionPath, fileId);
requireNonNull(partitionPath, "partitionPath is null");
requireNonNull(fileId, "partitionPath is null");
}

@Override
Expand Down
24 changes: 11 additions & 13 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/model/HudiInstant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,52 +132,50 @@ public String getFileName()
: isRequested() ? HudiTimeline.makeRequestedCommitFileName(timestamp)
: HudiTimeline.makeCommitFileName(timestamp);
}
else if (HudiTimeline.CLEAN_ACTION.equals(action)) {
if (HudiTimeline.CLEAN_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightCleanerFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedCleanerFileName(timestamp)
: HudiTimeline.makeCleanerFileName(timestamp);
}
else if (HudiTimeline.ROLLBACK_ACTION.equals(action)) {
if (HudiTimeline.ROLLBACK_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightRollbackFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedRollbackFileName(timestamp)
: HudiTimeline.makeRollbackFileName(timestamp);
}
else if (HudiTimeline.SAVEPOINT_ACTION.equals(action)) {
if (HudiTimeline.SAVEPOINT_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightSavePointFileName(timestamp)
: HudiTimeline.makeSavePointFileName(timestamp);
}
else if (HudiTimeline.DELTA_COMMIT_ACTION.equals(action)) {
if (HudiTimeline.DELTA_COMMIT_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightDeltaFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedDeltaFileName(timestamp)
: HudiTimeline.makeDeltaFileName(timestamp);
}
else if (HudiTimeline.COMPACTION_ACTION.equals(action)) {
if (HudiTimeline.COMPACTION_ACTION.equals(action)) {
if (isInflight()) {
return HudiTimeline.makeInflightCompactionFileName(timestamp);
}
else if (isRequested()) {
if (isRequested()) {
return HudiTimeline.makeRequestedCompactionFileName(timestamp);
}
else {
return HudiTimeline.makeCommitFileName(timestamp);
}
return HudiTimeline.makeCommitFileName(timestamp);
}
else if (HudiTimeline.RESTORE_ACTION.equals(action)) {
if (HudiTimeline.RESTORE_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightRestoreFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedRestoreFileName(timestamp)
: HudiTimeline.makeRestoreFileName(timestamp);
}
else if (HudiTimeline.REPLACE_COMMIT_ACTION.equals(action)) {
if (HudiTimeline.REPLACE_COMMIT_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightReplaceFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedReplaceFileName(timestamp)
: HudiTimeline.makeReplaceFileName(timestamp);
}
else if (HudiTimeline.INDEXING_ACTION.equals(action)) {
if (HudiTimeline.INDEXING_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightIndexFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestedIndexFileName(timestamp)
: HudiTimeline.makeIndexCommitFileName(timestamp);
}
else if (HudiTimeline.SCHEMA_COMMIT_ACTION.equals(action)) {
if (HudiTimeline.SCHEMA_COMMIT_ACTION.equals(action)) {
return isInflight() ? HudiTimeline.makeInflightSchemaFileName(timestamp)
: isRequested() ? HudiTimeline.makeRequestSchemaFileName(timestamp)
: HudiTimeline.makeSchemaFileName(timestamp);
Expand Down
33 changes: 2 additions & 31 deletions plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/timeline/TimelineLayoutVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,9 @@
*/
package io.trino.plugin.hudi.timeline;

import java.util.Objects;

import static com.google.common.base.Preconditions.checkArgument;

public class TimelineLayoutVersion
public record TimelineLayoutVersion(Integer version)
implements Comparable<TimelineLayoutVersion>
{
public static final Integer VERSION_0 = 0; // pre 0.5.1 version format
Expand All @@ -26,37 +24,10 @@ public class TimelineLayoutVersion
private static final Integer CURRENT_VERSION = VERSION_1;
public static final TimelineLayoutVersion CURRENT_LAYOUT_VERSION = new TimelineLayoutVersion(CURRENT_VERSION);

private final Integer version;

public TimelineLayoutVersion(Integer version)
public TimelineLayoutVersion
{
checkArgument(version <= CURRENT_VERSION);
checkArgument(version >= VERSION_0);
this.version = version;
}

public Integer getVersion()
{
return version;
}

@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TimelineLayoutVersion that = (TimelineLayoutVersion) o;
return Objects.equals(version, that.version);
}

@Override
public int hashCode()
{
return Objects.hash(version);
}

@Override
Expand Down