From 923c875359529f1ca91f3a1e8001361c623da8a8 Mon Sep 17 00:00:00 2001 From: erik-grepr Date: Fri, 27 Sep 2024 11:13:16 -0700 Subject: [PATCH 1/4] Update TableMetadataParser to close streams --- .../apache/iceberg/TableMetadataParser.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 8bda184142cd..44c4f1eabf18 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -122,13 +122,23 @@ public static void write(TableMetadata metadata, OutputFile outputFile) { public static void internalWrite( TableMetadata metadata, OutputFile outputFile, boolean overwrite) { boolean isGzip = Codec.fromFileName(outputFile.location()) == Codec.GZIP; - OutputStream stream = overwrite ? outputFile.createOrOverwrite() : outputFile.create(); - try (OutputStream ou = isGzip ? new GZIPOutputStream(stream) : stream; - OutputStreamWriter writer = new OutputStreamWriter(ou, StandardCharsets.UTF_8)) { - JsonGenerator generator = JsonUtil.factory().createGenerator(writer); - generator.useDefaultPrettyPrinter(); - toJson(metadata, generator); - generator.flush(); + try (OutputStream os = overwrite ? outputFile.createOrOverwrite() : outputFile.create()) { + if (isGzip) { + try (OutputStream gos = new GZIPOutputStream(os); + OutputStream osw = new OutputStreamWriter(gos, StandardCharsets.UTF_8)) { + JsonGenerator generator = JsonUtil.factory().createGenerator(os); + generator.useDefaultPrettyPrinter(); + toJson(metadata, generator); + generator.flush(); + } + } else { + try (OutputStream osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) { + JsonGenerator generator = JsonUtil.factory().createGenerator(os); + generator.useDefaultPrettyPrinter(); + toJson(metadata, generator); + generator.flush(); + } + } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile); } @@ -275,8 +285,12 @@ public static TableMetadata read(FileIO io, String path) { public static TableMetadata read(FileIO io, InputFile file) { Codec codec = Codec.fromFileName(file.location()); - try (InputStream is = - codec == Codec.GZIP ? new GZIPInputStream(file.newStream()) : file.newStream()) { + try (InputStream is = file.newStream()) { + if (codec == Codec.GZIP) { + try (InputStream gis = new GZIPInputStream(is)) { + return fromJson(file, JsonUtil.mapper().readValue(gis, JsonNode.class)); + } + } return fromJson(file, JsonUtil.mapper().readValue(is, JsonNode.class)); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to read file: %s", file); From 4a0fc3cfc836391bf78fbd294a086cdbdbaa1e8e Mon Sep 17 00:00:00 2001 From: erik-grepr Date: Mon, 30 Sep 2024 10:18:13 -0700 Subject: [PATCH 2/4] Fix OutputStreamWriter --- .../apache/iceberg/TableMetadataParser.java | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 44c4f1eabf18..2c96b295eae8 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -122,22 +122,14 @@ public static void write(TableMetadata metadata, OutputFile outputFile) { public static void internalWrite( TableMetadata metadata, OutputFile outputFile, boolean overwrite) { boolean isGzip = Codec.fromFileName(outputFile.location()) == Codec.GZIP; - try (OutputStream os = overwrite ? outputFile.createOrOverwrite() : outputFile.create()) { - if (isGzip) { - try (OutputStream gos = new GZIPOutputStream(os); - OutputStream osw = new OutputStreamWriter(gos, StandardCharsets.UTF_8)) { - JsonGenerator generator = JsonUtil.factory().createGenerator(os); + try (OutputStream os = overwrite ? outputFile.createOrOverwrite() : outputFile.create(); + OutputStream gos = isGzip ? new GZIPOutputStream(os) : os; + OutputStreamWriter writer = new OutputStreamWriter(gos, StandardCharsets.UTF_8) + ) { + JsonGenerator generator = JsonUtil.factory().createGenerator(writer); generator.useDefaultPrettyPrinter(); toJson(metadata, generator); generator.flush(); - } - } else { - try (OutputStream osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) { - JsonGenerator generator = JsonUtil.factory().createGenerator(os); - generator.useDefaultPrettyPrinter(); - toJson(metadata, generator); - generator.flush(); - } } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile); @@ -285,13 +277,10 @@ public static TableMetadata read(FileIO io, String path) { public static TableMetadata read(FileIO io, InputFile file) { Codec codec = Codec.fromFileName(file.location()); - try (InputStream is = file.newStream()) { - if (codec == Codec.GZIP) { - try (InputStream gis = new GZIPInputStream(is)) { - return fromJson(file, JsonUtil.mapper().readValue(gis, JsonNode.class)); - } - } - return fromJson(file, JsonUtil.mapper().readValue(is, JsonNode.class)); + try (InputStream is = file.newStream(); + InputStream gis = codec == Codec.GZIP ? new GZIPInputStream(is) : is; + ) { + return fromJson(file, JsonUtil.mapper().readValue(gis, JsonNode.class)); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to read file: %s", file); } From e61cd0863a87ca7abc65dc7d5080a3f8d82d7176 Mon Sep 17 00:00:00 2001 From: erik-grepr Date: Sun, 13 Oct 2024 22:38:23 -0700 Subject: [PATCH 3/4] Update TableMetadataParser.java --- core/src/main/java/org/apache/iceberg/TableMetadataParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 2c96b295eae8..f1ff5c582c53 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -130,7 +130,6 @@ public static void internalWrite( generator.useDefaultPrettyPrinter(); toJson(metadata, generator); generator.flush(); - } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile); } From b72e6bce539ebdffab188151c5820b77551b4484 Mon Sep 17 00:00:00 2001 From: Erik Lamp Date: Mon, 14 Oct 2024 10:38:01 -0700 Subject: [PATCH 4/4] spotlessApply --- .../org/apache/iceberg/TableMetadataParser.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index f1ff5c582c53..5abfb74974f7 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -123,13 +123,12 @@ public static void internalWrite( TableMetadata metadata, OutputFile outputFile, boolean overwrite) { boolean isGzip = Codec.fromFileName(outputFile.location()) == Codec.GZIP; try (OutputStream os = overwrite ? outputFile.createOrOverwrite() : outputFile.create(); - OutputStream gos = isGzip ? new GZIPOutputStream(os) : os; - OutputStreamWriter writer = new OutputStreamWriter(gos, StandardCharsets.UTF_8) - ) { - JsonGenerator generator = JsonUtil.factory().createGenerator(writer); - generator.useDefaultPrettyPrinter(); - toJson(metadata, generator); - generator.flush(); + OutputStream gos = isGzip ? new GZIPOutputStream(os) : os; + OutputStreamWriter writer = new OutputStreamWriter(gos, StandardCharsets.UTF_8)) { + JsonGenerator generator = JsonUtil.factory().createGenerator(writer); + generator.useDefaultPrettyPrinter(); + toJson(metadata, generator); + generator.flush(); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile); } @@ -277,8 +276,7 @@ public static TableMetadata read(FileIO io, String path) { public static TableMetadata read(FileIO io, InputFile file) { Codec codec = Codec.fromFileName(file.location()); try (InputStream is = file.newStream(); - InputStream gis = codec == Codec.GZIP ? new GZIPInputStream(is) : is; - ) { + InputStream gis = codec == Codec.GZIP ? new GZIPInputStream(is) : is) { return fromJson(file, JsonUtil.mapper().readValue(gis, JsonNode.class)); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to read file: %s", file);