From f896ed63bceec6d32e1d7fed83b838fb6d447107 Mon Sep 17 00:00:00 2001 From: Hussein Awala <21311487+hussein-awala@users.noreply.github.com> Date: Wed, 24 Jul 2024 13:45:31 +0200 Subject: [PATCH 1/4] Hive: close the fileIO client when closing the hive catalog --- .../org/apache/iceberg/hive/HiveCatalog.java | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index b4f49e29fc49..2eba0628dc12 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -18,6 +18,10 @@ */ package org.apache.iceberg.hive; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalListener; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -79,6 +83,7 @@ public class HiveCatalog extends BaseMetastoreCatalog implements SupportsNamespa private ClientPool clients; private boolean listAllTables = false; private Map catalogProperties; + private Cache fileIOCloser; public HiveCatalog() {} @@ -111,6 +116,20 @@ public void initialize(String inputName, Map properties) { : CatalogUtil.loadFileIO(fileIOImpl, properties, conf); this.clients = new CachedClientPool(conf, properties); + this.fileIOCloser = newFileIOCloser(); + } + + private Cache newFileIOCloser() { + return Caffeine.newBuilder() + .weakKeys() + .removalListener( + (RemovalListener) + (ops, fileIOInstance, cause) -> { + if (null != fileIOInstance) { + fileIOInstance.close(); + } + }) + .build(); } @Override @@ -512,7 +531,10 @@ private boolean isValidateNamespace(Namespace namespace) { public TableOperations newTableOps(TableIdentifier tableIdentifier) { String dbName = tableIdentifier.namespace().level(0); String tableName = tableIdentifier.name(); - return new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName); + HiveTableOperations hiveTableOperations = + new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName); + fileIOCloser.put(hiveTableOperations, hiveTableOperations.io()); + return hiveTableOperations; } @Override @@ -636,6 +658,15 @@ protected Map properties() { return catalogProperties == null ? ImmutableMap.of() : catalogProperties; } + @Override + public void close() throws IOException { + super.close(); + if (fileIOCloser != null) { + fileIOCloser.invalidateAll(); + fileIOCloser.cleanUp(); + } + } + @VisibleForTesting void setListAllTables(boolean listAllTables) { this.listAllTables = listAllTables; From 33480d2360865643c58b84082e026a472855a234 Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Thu, 25 Jul 2024 10:33:57 -0600 Subject: [PATCH 2/4] Update hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java Co-authored-by: Eduard Tudenhoefner --- .../src/main/java/org/apache/iceberg/hive/HiveCatalog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index 2eba0628dc12..b24a75080b9d 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -531,7 +531,7 @@ private boolean isValidateNamespace(Namespace namespace) { public TableOperations newTableOps(TableIdentifier tableIdentifier) { String dbName = tableIdentifier.namespace().level(0); String tableName = tableIdentifier.name(); - HiveTableOperations hiveTableOperations = + HiveTableOperations ops = new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName); fileIOCloser.put(hiveTableOperations, hiveTableOperations.io()); return hiveTableOperations; From d234c5706822ae06b329c46bc92b41c0c1e4c0e5 Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Thu, 25 Jul 2024 10:38:05 -0600 Subject: [PATCH 3/4] Update hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java --- .../src/main/java/org/apache/iceberg/hive/HiveCatalog.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index b24a75080b9d..dc0f8df9504a 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -533,8 +533,8 @@ public TableOperations newTableOps(TableIdentifier tableIdentifier) { String tableName = tableIdentifier.name(); HiveTableOperations ops = new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName); - fileIOCloser.put(hiveTableOperations, hiveTableOperations.io()); - return hiveTableOperations; +fileIOCloser.put(ops, ops.io()); +return ops; } @Override From e7e451e27389a238aebbab86ca1aa0a010565650 Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Thu, 25 Jul 2024 10:39:14 -0600 Subject: [PATCH 4/4] Update hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java --- .../src/main/java/org/apache/iceberg/hive/HiveCatalog.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index dc0f8df9504a..8944cf93947b 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -533,8 +533,8 @@ public TableOperations newTableOps(TableIdentifier tableIdentifier) { String tableName = tableIdentifier.name(); HiveTableOperations ops = new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName); -fileIOCloser.put(ops, ops.io()); -return ops; + fileIOCloser.put(ops, ops.io()); + return ops; } @Override