[apache#2417] feat(trino-connector): Support update catalog operation…

…s in trino-connector (apache#2564) ### What changes were proposed in this pull request? Trino connector support stored procedure to update catalog attributes. ### Why are the changes needed? FEATURE: apache#2417 ### Does this PR introduce _any_ user-facing change? Add stored procedure alter_catalog. ### How was this patch tested? Add new IT
coolderli · Mar 28, 2024 · c5e74dc · c5e74dc
1 parent 5ceae91
commit c5e74dc
Show file tree

Hide file tree

Showing 17 changed files with 477 additions and 117 deletions.
diff --git a/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java
@@ -88,7 +88,9 @@ public void testAuditCatalog() throws Exception {
     String catalogName = GravitinoITUtils.genRandomName("audit_mysql_catalog");
     Catalog catalog = createCatalog(catalogName);
     Assertions.assertEquals(expectUser, catalog.auditInfo().creator());
-    Assertions.assertNull(catalog.auditInfo().lastModifier());
+    Assertions.assertEquals(catalog.auditInfo().creator(), catalog.auditInfo().lastModifier());
+    Assertions.assertEquals(
+        catalog.auditInfo().createTime(), catalog.auditInfo().lastModifiedTime());
     catalog =
         metalake.alterCatalog(
             NameIdentifier.of(metalakeName, catalogName),

diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java
@@ -274,6 +274,8 @@ public Catalog createCatalog(
 
     long uid = idGenerator.nextId();
     StringIdentifier stringId = StringIdentifier.fromId(uid);
+    Instant now = Instant.now();
+    String creator = PrincipalUtils.getCurrentPrincipal().getName();
     CatalogEntity e =
         CatalogEntity.builder()
             .withId(uid)
@@ -285,8 +287,10 @@ public Catalog createCatalog(
             .withProperties(StringIdentifier.newPropertiesWithId(stringId, mergedConfig))
             .withAuditInfo(
                 AuditInfo.builder()
-                    .withCreator(PrincipalUtils.getCurrentPrincipal().getName())
-                    .withCreateTime(Instant.now())
+                    .withCreator(creator)
+                    .withCreateTime(now)
+                    .withLastModifier(creator)
+                    .withLastModifiedTime(now)
                     .build())
             .build();
 

diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md
@@ -17,7 +17,8 @@ The catalogs currently supported by the Gravitino connector are as follows:
 
 Trino itself does not support creating catalogs. 
 Users can create catalogs through the Gravitino connector and then load them into Trino. 
-The Gravitino connector provides the following stored procedures to create and delete catalogs.
+The Gravitino connector provides the following stored procedures to create, delete, and alter catalogs.
+User can also use the system table `catalog` to describe all the catalogs.
 
 Create catalog:
 
@@ -43,12 +44,40 @@ Drop catalog:
 drop_catalog(CATALOG varchar, IGNORE_NOT_EXIST boolean);
 ```
 
-- CATALOG: The catalog name to be created.
+- CATALOG: The catalog name to be deleted.
 - IGNORE_NOT_EXIST: The flag to ignore the error if the catalog does not exist. It's optional, the default value is `false`.
 
-The two stored procedures are under the `gravitino` connector, and the `system` schema.
+
+Alter catalog:
+
+```sql
+alter_catalog(CATALOG varchar, SET_PROPERTIES MAP(VARCHAR, VARCHAR), REMOVE_PROPERTIES ARRY[VARCHAR]);
+```
+
+- CATALOG: The catalog name to be altered.
+- SET_PROPERTIES: The properties to be set.
+- REMOVE_PROPERTIES: The properties to be removed.
+
+These stored procedures are under the `gravitino` connector and the `system` schema.
 So you need to use the following SQL to call them in the `trino-cli`:
 
+
+Describe catalogs:
+
+The system table `gravitino.system.catalog` is used to describe all the catalogs.
+
+```sql
+select * from gravitino.system.catalog;
+```
+
+The result is like:
+
+```test
+     name     | provider |                                                 properties
+--------------+----------+-------------------------------------------------------------------------------------------------------------
+ test.gt_hive | hive     | {gravitino.bypass.hive.metastore.client.capability.check=false, metastore.uris=thrift://trino-ci-hive:9083}
+```
+
 Example:
 You can run the following SQL to create a catalog named `mysql` with `jdbc-mysql` provider.
 
@@ -58,8 +87,8 @@ call gravitino.system.create_catalog(
     'mysql',
     'jdbc-mysql',
     Map(
-        Array('jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver'),
-        Array('jdbc:mysql:192.168.164.4:3306?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver')
+        Array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver'],
+        Array['jdbc:mysql:192.168.164.4:3306?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver']
     )
 )
 call gravitino.system.drop_datalog('mysql');
@@ -69,8 +98,8 @@ call gravitino.system.create_catalog(
     catalog =>'mysql',
     provider => 'jdbc-mysql',
     properties => Map(
-        Array('jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver'),
-        Array('jdbc:mysql:192.168.164.4:3306?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver')
+        Array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver'],
+        Array['jdbc:mysql:192.168.164.4:3306?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver']
     ),
     ignore_exist => true
 )
@@ -79,6 +108,15 @@ call gravitino.system.drop_datalog(
     catalog => 'mysql'
     ignore_not_exist => true
 );
+
+call gravitino.system.alter_catalog(
+    catalog => 'mysql',
+    set_properties=> Map(
+        Array['jdbc-url'],
+        Array['jdbc:mysql:127.0.0.1:3306?useSSL=false']
+    ),
+    remove_properties => Array['jdbc-driver']
+)
 ```
 
 if you need more information about catalog, please refer to:

diff --git a/...-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryITBase.java b/...-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryITBase.java
@@ -190,7 +190,7 @@ protected static void dropCatalog(String catalogName) {
     Catalog catalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName));
     SupportsSchemas schemas = catalog.asSchemas();
     Arrays.stream(schemas.listSchemas(Namespace.ofSchema(metalakeName, catalogName)))
-        .filter(schema -> !schema.name().equals("default") && schema.name().startsWith("gt_"))
+        .filter(schema -> !schema.name().equals("default"))
         .forEach(
             schema -> {
               try {

diff --git a/...set/testsets/jdbc-mysql/00005_catalog.sql → ...tsets/jdbc-mysql/00005_create_catalog.sql b/...set/testsets/jdbc-mysql/00005_catalog.sql → ...tsets/jdbc-mysql/00005_create_catalog.sql
diff --git a/...set/testsets/jdbc-mysql/00005_catalog.txt → ...tsets/jdbc-mysql/00005_create_catalog.txt b/...set/testsets/jdbc-mysql/00005_catalog.txt → ...tsets/jdbc-mysql/00005_create_catalog.txt
diff --git a/...tion-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql b/...tion-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql
@@ -0,0 +1,41 @@
+call gravitino.system.create_catalog(
+    'gt_mysql_xxx1',
+    'jdbc-mysql',
+    map(
+        array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver'],
+        array['${mysql_uri}/?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver']
+    )
+);
+
+select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1';
+
+call gravitino.system.alter_catalog(
+    'gt_mysql_xxx1',
+    map(
+        array['join-pushdown.strategy', 'test_key'],
+        array['EAGER', 'test_value']
+    )
+);
+
+select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1';
+
+call gravitino.system.alter_catalog(
+    'gt_mysql_xxx1',
+    map(),
+    array['join-pushdown.strategy']
+);
+
+select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1';
+
+call gravitino.system.alter_catalog(
+    catalog => 'gt_mysql_xxx1',
+    set_properties => map(
+        array['join-pushdown.strategy'],
+        array['EAGER']
+    ),
+    remove_properties => array['test_key']
+);
+
+select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1';
+
+call gravitino.system.drop_catalog('gt_mysql_xxx1');
diff --git a/...tion-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/...tion-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt
@@ -0,0 +1,17 @@
+CALL
+
+"test.gt_mysql_xxx1","jdbc-mysql","{jdbc-url=jdbc:mysql://%/?useSSL=false, jdbc-user=trino, jdbc-password=ds123, jdbc-driver=com.mysql.cj.jdbc.Driver}"
+
+CALL
+
+"test.gt_mysql_xxx1","jdbc-mysql","{join-pushdown.strategy=EAGER, test_key=test_value, jdbc-url=jdbc:mysql://%/?useSSL=false, jdbc-user=trino, jdbc-password=ds123, jdbc-driver=com.mysql.cj.jdbc.Driver}"
+
+CALL
+
+"test.gt_mysql_xxx1","jdbc-mysql","{test_key=test_value, jdbc-url=jdbc:mysql://%/?useSSL=false, jdbc-user=trino, jdbc-password=ds123, jdbc-driver=com.mysql.cj.jdbc.Driver}"
+
+CALL
+
+"test.gt_mysql_xxx1","jdbc-mysql","{join-pushdown.strategy=EAGER, jdbc-url=jdbc:mysql://%/?useSSL=false, jdbc-user=trino, jdbc-password=ds123, jdbc-driver=com.mysql.cj.jdbc.Driver}"
+
+CALL
diff --git a/...tor/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java b/...tor/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java
@@ -12,6 +12,7 @@
 import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager;
 import com.datastrato.gravitino.trino.connector.catalog.CatalogInjector;
 import com.datastrato.gravitino.trino.connector.system.GravitinoSystemConnector;
+import com.datastrato.gravitino.trino.connector.system.storedprocdure.GravitinoStoredProcedureFactory;
 import com.datastrato.gravitino.trino.connector.system.table.GravitinoSystemTableFactory;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -96,8 +97,10 @@ public Connector create(
         throw new TrinoException(GRAVITINO_METALAKE_NOT_EXISTS, "No gravitino metalake selected");
       }
       catalogConnectorManager.addMetalake(metalake);
+      GravitinoStoredProcedureFactory gravitinoStoredProcedureFactory =
+          new GravitinoStoredProcedureFactory(catalogConnectorManager, metalake);
 
-      return new GravitinoSystemConnector(metalake, catalogConnectorManager);
+      return new GravitinoSystemConnector(gravitinoStoredProcedureFactory);
     }
   }