From f4bf9896cae31079e1ca7a3ce27d91d47f7659e5 Mon Sep 17 00:00:00 2001 From: Clearvive Date: Tue, 17 Oct 2023 10:30:09 +0800 Subject: [PATCH] [#400] test: Add Catalog-iceberg e2e integration test. --- .../iceberg/IcebergCatalogOperations.java | 3 ++- .../lakehouse/iceberg/TestIcebergTable.java | 15 +++------------ .../lakehouse/iceberg/CatalogIcebergIT.java | 8 ++++++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index 877f0d4b714..226c14cb505 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -29,6 +29,7 @@ import com.datastrato.graviton.rel.TableCatalog; import com.datastrato.graviton.rel.TableChange; import com.datastrato.graviton.rel.transforms.Transform; +import com.datastrato.graviton.utils.MapUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -478,7 +479,7 @@ public Table createTable( SortOrder[] sortOrders) throws NoSuchSchemaException, TableAlreadyExistsException { try { - if(!Distribution.NONE.equals(distribution)) { + if (!Distribution.NONE.equals(distribution)) { throw new UnsupportedOperationException("Iceberg does not support distribution"); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/graviton/catalog/lakehouse/iceberg/TestIcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/graviton/catalog/lakehouse/iceberg/TestIcebergTable.java index a4a5c1b7e74..107080c16a7 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/graviton/catalog/lakehouse/iceberg/TestIcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/graviton/catalog/lakehouse/iceberg/TestIcebergTable.java @@ -17,7 +17,6 @@ import com.datastrato.graviton.meta.CatalogEntity; import com.datastrato.graviton.rel.Column; import com.datastrato.graviton.rel.Distribution; -import com.datastrato.graviton.rel.Distribution.Strategy; import com.datastrato.graviton.rel.SortOrder; import com.datastrato.graviton.rel.SortOrder.Direction; import com.datastrato.graviton.rel.SortOrder.NullOrdering; @@ -107,14 +106,6 @@ private static CatalogEntity createDefaultCatalogEntity() { return entity; } - private Distribution createDistribution() { - return Distribution.builder() - .withNumber(10) - .withTransforms(new Transform[] {Transforms.field(new String[] {"col_1"})}) - .withStrategy(Strategy.EVEN) - .build(); - } - private SortOrder[] createSortOrder() { return new SortOrder[] { SortOrder.builder() @@ -159,7 +150,7 @@ public void testCreateIcebergTable() throws IOException { ICEBERG_COMMENT, properties, new Transform[0], - null, + Distribution.NONE, sortOrders); Assertions.assertEquals(tableIdentifier.name(), table.name()); Assertions.assertEquals(ICEBERG_COMMENT, table.comment()); @@ -198,7 +189,7 @@ public void testCreateIcebergTable() throws IOException { ICEBERG_COMMENT, properties, new Transform[0], - null, + Distribution.NONE, sortOrders)); Assertions.assertTrue(exception.getMessage().contains("Table already exists")); } @@ -380,7 +371,7 @@ public void testAlterIcebergTable() { .build(); Column[] columns = new Column[] {col1, col2}; - Distribution distribution = createDistribution(); + Distribution distribution = Distribution.NONE; SortOrder[] sortOrders = createSortOrder(); Table createdTable = diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index b688c44690a..de4ed2ec905 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -11,6 +11,7 @@ import com.datastrato.graviton.Catalog; import com.datastrato.graviton.NameIdentifier; import com.datastrato.graviton.Namespace; +import com.datastrato.graviton.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.graviton.client.GravitonMetaLake; import com.datastrato.graviton.dto.rel.ColumnDTO; import com.datastrato.graviton.exceptions.NoSuchSchemaException; @@ -42,9 +43,11 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +@Tag("graviton-docker-it") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogIcebergIT extends AbstractIT { public static String metalakeName = GravitonITUtils.genRandomName("CatalogIcebergIT_metalake"); @@ -108,6 +111,11 @@ private static void createCatalog() { Map properties = Maps.newHashMap(); properties.put("key1", "val1"); properties.put("key2", "val2"); + properties.put("catalog-backend", IcebergCatalogBackend.JDBC.name()); + properties.put("jdbc-user", "iceberg"); + properties.put("jdbc-password", "iceberg"); + properties.put("uri", "jdbc:mysql://127.0.0.1:3306/metastore_db?createDatabaseIfNotExist=true"); + properties.put("warehouse", "file:///tmp/iceberg"); Catalog createdCatalog = metalake.createCatalog(