Skip to content

Commit

Permalink
[bugfix](iceberg)support null values as partition (#35503)
Browse files Browse the repository at this point in the history
#31442
test in #34929

When null value is used as the partition value, BE will return the "null" string, so this string needs to be processed specially.
  • Loading branch information
wuwenchi authored May 28, 2024
1 parent de2d71a commit d86cd1b
Showing 1 changed file with 13 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

public class IcebergTransaction implements Transaction {

Expand Down Expand Up @@ -154,11 +155,22 @@ public CommitTaskData(String path,
this.path = path;
this.fileSizeInBytes = fileSizeInBytes;
this.metrics = metrics;
this.partitionValues = partitionValues;
this.partitionValues = convertPartitionValuesForNull(partitionValues);
this.content = content;
this.referencedDataFiles = referencedDataFiles;
}

private Optional<List<String>> convertPartitionValuesForNull(Optional<List<String>> partitionValues) {
if (!partitionValues.isPresent()) {
return partitionValues;
}
List<String> values = partitionValues.get();
if (!values.contains("null")) {
return partitionValues;
}
return Optional.of(values.stream().map(s -> s.equals("null") ? null : s).collect(Collectors.toList()));
}

public String getPath() {
return path;
}
Expand Down

0 comments on commit d86cd1b

Please sign in to comment.