add lazy snapshot loading for RESTCatalogAdapter

apache · Sep 12, 2023 · 5470010 · 5470010
1 parent d8ded68
commit 5470010
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 3 deletions.
diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml
@@ -829,6 +829,12 @@ acceptedBreaks:
     - code: "java.class.removed"
       old: "interface org.apache.iceberg.actions.RewritePositionDeleteStrategy"
       justification: "Removing deprecated code"
+    - code: "java.method.numberOfParametersChanged"
+      old: "method org.apache.iceberg.rest.responses.LoadTableResponse org.apache.iceberg.rest.CatalogHandlers::loadTable(org.apache.iceberg.catalog.Catalog,\
+        \ org.apache.iceberg.catalog.TableIdentifier)"
+      new: "method org.apache.iceberg.rest.responses.LoadTableResponse org.apache.iceberg.rest.CatalogHandlers::loadTable(org.apache.iceberg.catalog.Catalog,\
+        \ org.apache.iceberg.catalog.TableIdentifier, org.apache.iceberg.rest.RESTSessionCatalog.SnapshotMode)"
+      justification: "add snapshot mode when load Table"
     - code: "java.method.removed"
       old: "method java.util.List<org.apache.iceberg.DataFile> org.apache.iceberg.MergingSnapshotProducer<ThisT>::addedFiles()\
         \ @ org.apache.iceberg.BaseOverwriteFiles"

diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java
@@ -35,6 +35,7 @@
 import org.apache.iceberg.BaseTransaction;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
+import org.apache.iceberg.SnapshotRef;
 import org.apache.iceberg.SortOrder;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableMetadata;
@@ -253,12 +254,34 @@ public static void purgeTable(Catalog catalog, TableIdentifier ident) {
     }
   }
 
-  public static LoadTableResponse loadTable(Catalog catalog, TableIdentifier ident) {
+  private static TableMetadata getTableMetaData(
+      BaseTable baseTable, RESTSessionCatalog.SnapshotMode snapshotMode) {
+    TableMetadata tableMetadata = baseTable.operations().current();
+
+    switch (snapshotMode) {
+      case ALL:
+        return tableMetadata;
+      case REFS:
+        {
+          TableMetadata refsMetaData = TableMetadata.buildFrom(tableMetadata).build();
+          Set<Long> referencedSnapshotIds =
+              refsMetaData.refs().values().stream()
+                  .map(SnapshotRef::snapshotId)
+                  .collect(Collectors.toSet());
+          refsMetaData.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId()));
+          return refsMetaData;
+        }
+    }
+    return tableMetadata;
+  }
+
+  public static LoadTableResponse loadTable(
+      Catalog catalog, TableIdentifier ident, RESTSessionCatalog.SnapshotMode snapshotMode) {
     Table table = catalog.loadTable(ident);
 
     if (table instanceof BaseTable) {
       return LoadTableResponse.builder()
-          .withTableMetadata(((BaseTable) table).operations().current())
+          .withTableMetadata(getTableMetaData((BaseTable) table, snapshotMode))
           .build();
     } else if (table instanceof BaseMetadataTable) {
       // metadata tables are loaded on the client side, return NoSuchTableException for now

diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.Consumer;
 import org.apache.iceberg.BaseTable;
@@ -347,8 +348,14 @@ public <T extends RESTResponse> T handleRequest(
 
       case LOAD_TABLE:
         {
+          RESTSessionCatalog.SnapshotMode snapshotMode =
+              RESTSessionCatalog.SnapshotMode.valueOf(
+                  PropertyUtil.propertyAsString(
+                          vars, "snapshots", RESTSessionCatalog.SnapshotMode.ALL.name())
+                      .toUpperCase(Locale.ENGLISH));
           TableIdentifier ident = identFromPathVars(vars);
-          return castResponse(responseType, CatalogHandlers.loadTable(catalog, ident));
+          return castResponse(
+              responseType, CatalogHandlers.loadTable(catalog, ident, snapshotMode));
         }
 
       case REGISTER_TABLE: