From b90ee992c05ab0dbd2be63bdb339cb4e4957a6ad Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Wed, 15 May 2024 15:27:29 +0800 Subject: [PATCH 1/4] [#3209] test(web): doris web e2e test (#3230) (#3398) ### What changes were proposed in this pull request? Add test create doris catalog to the web e2e test ### Why are the changes needed? Fix: #3209 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? manual Co-authored-by: yuqi --- integration-test/build.gradle.kts | 1 + .../test/web/ui/CatalogsPageDorisTest.java | 267 ++++++++++++++++++ .../test/web/ui/CatalogsPageKafkaTest.java | 35 +-- 3 files changed, 274 insertions(+), 29 deletions(-) create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 95ce862da68..65768e8f251 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -161,6 +161,7 @@ tasks.test { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.5") environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0") + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3") copy { from("${project.rootDir}/dev/docker/trino/conf") diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java new file mode 100644 index 00000000000..0e32e685dbc --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java @@ -0,0 +1,267 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.web.ui; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.DorisContainer; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; +import com.datastrato.gravitino.integration.test.web.ui.pages.CatalogsPage; +import com.datastrato.gravitino.integration.test.web.ui.pages.MetalakePage; +import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.Maps; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +@Tag("gravitino-docker-it") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class CatalogsPageDorisTest extends AbstractWebIT { + MetalakePage metalakePage = new MetalakePage(); + CatalogsPage catalogsPage = new CatalogsPage(); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static GravitinoAdminClient gravitinoClient; + private static GravitinoMetalake metalake; + + protected static String gravitinoUri = "http://127.0.0.1:8090"; + protected static String dorisJdbcConnectionUri; + + private static final String CATALOG_TABLE_TITLE = "Schemas"; + private static final String SCHEMA_TABLE_TITLE = "Tables"; + private static final String TABLE_TABLE_TITLE = "Columns"; + private static final String METALAKE_NAME = "test"; + private static final String CATALOG_TYPE_RELATIONAL = "relational"; + + private static final String DORIS_CATALOG_NAME = "catalog_doris"; + private static final String DORIS_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; + private static final String USER_NAME = "root"; + private static final String PASSWORD = "root"; + + private static final String SCHEMA_NAME_DORIS = "schema_doris"; + private static final String TABLE_NAME = "table1"; + private static final String COLUMN_NAME = "col1"; + private static final String PROPERTIES_KEY1 = "key1"; + private static final String PROPERTIES_VALUE1 = "val1"; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; + + @BeforeAll + public static void before() throws Exception { + + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) { + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } else { + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-doris/build/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } + gravitinoClient = AbstractIT.getGravitinoClient(); + gravitinoUri = String.format("http://127.0.0.1:%d", AbstractIT.getGravitinoServerPort()); + + containerSuite.startDorisContainer(); + + dorisJdbcConnectionUri = + String.format( + "jdbc:mysql://%s:%d/", + containerSuite.getDorisContainer().getContainerIpAddress(), + DorisContainer.FE_MYSQL_PORT); + LOG.info("Doris jdbc url: {}", dorisJdbcConnectionUri); + } + + /** + * Create the specified schema + * + * @param metalakeName The name of the Metalake where the schema will be created. + * @param catalogName The name of the Catalog where the schema will be created. + * @param schemaName The name of the Schema where the schema will be created. + */ + void createSchema(String metalakeName, String catalogName, String schemaName) { + Map properties = Maps.newHashMap(); + properties.put(PROPERTIES_KEY1, PROPERTIES_VALUE1); + Catalog catalog_doris = + metalake.loadCatalog(NameIdentifier.ofCatalog(metalakeName, catalogName)); + catalog_doris + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), "comment", properties); + } + + /** + * Creates a table with a single column in the specified Metalake, Catalog, Schema, and Table. + * + * @param metalakeName The name of the Metalake where the table will be created. + * @param catalogName The name of the Catalog where the table will be created. + * @param schemaName The name of the Schema where the table will be created. + * @param tableName The name of the Table to be created. + * @param colName The name of the Column to be created in the Table. + */ + void createTableAndColumn( + String metalakeName, + String catalogName, + String schemaName, + String tableName, + String colName) { + Map properties = Maps.newHashMap(); + properties.put("replication_allocation", "tag.location.default: 1"); + Column column = Column.of(colName, Types.IntegerType.get(), "column comment"); + Catalog catalog_doris = + metalake.loadCatalog(NameIdentifier.ofCatalog(metalakeName, catalogName)); + catalog_doris + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + new Column[] {column}, + "comment", + properties, + Distributions.hash(2, NamedReference.field(colName)), + new SortOrder[0]); + } + + @Test + @Order(0) + public void testCreateDorisCatalog() throws InterruptedException { + // create metalake + clickAndWait(metalakePage.createMetalakeBtn); + metalakePage.setMetalakeNameField(METALAKE_NAME); + clickAndWait(metalakePage.submitHandleMetalakeBtn); + // load metalake + metalake = gravitinoClient.loadMetalake(NameIdentifier.of(METALAKE_NAME)); + metalakePage.clickMetalakeLink(METALAKE_NAME); + // create doris catalog actions + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(DORIS_CATALOG_NAME); + // select provider as doris + clickAndWait(catalogsPage.catalogProviderSelector); + catalogsPage.clickSelectProvider("jdbc-doris"); + catalogsPage.setCatalogCommentField("doris catalog comment"); + // set doris catalog props + catalogsPage.setCatalogFixedProp("jdbc-driver", DORIS_JDBC_DRIVER); + catalogsPage.setCatalogFixedProp("jdbc-url", dorisJdbcConnectionUri); + catalogsPage.setCatalogFixedProp("jdbc-user", USER_NAME); + catalogsPage.setCatalogFixedProp("jdbc-password", PASSWORD); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(DORIS_CATALOG_NAME)); + } + + @Test + @Order(1) + public void testClickCatalogLink() { + // 1. create schema of doris catalog + createSchema(METALAKE_NAME, DORIS_CATALOG_NAME, SCHEMA_NAME_DORIS); + // 2. click link of doris catalog + catalogsPage.clickCatalogLink(METALAKE_NAME, DORIS_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); + // 3. verify show table title、 schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME_DORIS, false)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(DORIS_CATALOG_NAME)); + } + + @Test + @Order(2) + public void testClickSchemaLink() { + // create table + createTableAndColumn( + METALAKE_NAME, DORIS_CATALOG_NAME, SCHEMA_NAME_DORIS, TABLE_NAME, COLUMN_NAME); + // 2. click link of doris schema + catalogsPage.clickSchemaLink( + METALAKE_NAME, DORIS_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME_DORIS); + // verify show table title、 schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME_DORIS)); + } + + @Test + @Order(3) + public void testClickTableLink() { + catalogsPage.clickTableLink( + METALAKE_NAME, DORIS_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME_DORIS, TABLE_NAME); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyTableColumns()); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(TABLE_NAME)); + } + + @Test + @Order(4) + public void testDorisSchemaTreeNode() throws InterruptedException { + catalogsPage.clickBreadCrumbsToCatalogs(); + // click doris catalog tree node + String dorisCatalogNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, DORIS_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); + catalogsPage.clickTreeNode(dorisCatalogNode); + // verify show table title、 schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME_DORIS, false)); + List treeNodes = Arrays.asList(DORIS_CATALOG_NAME, SCHEMA_NAME_DORIS); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + } + + @Test + @Order(5) + public void testDorisTableTreeNode() throws InterruptedException { + // 1. click schema tree node + String dorisSchemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, DORIS_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME_DORIS); + catalogsPage.clickTreeNode(dorisSchemaNode); + // 2. verify show table title、 default schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); + List treeNodes = Arrays.asList(DORIS_CATALOG_NAME, SCHEMA_NAME_DORIS, TABLE_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + } + + @Test + @Order(6) + public void testDorisTableDetail() throws InterruptedException { + // 1. click doris table tree node + String tableNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, + DORIS_CATALOG_NAME, + CATALOG_TYPE_RELATIONAL, + SCHEMA_NAME_DORIS, + TABLE_NAME); + catalogsPage.clickTreeNode(tableNode); + // 2. verify show table column after click table tree node + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); + clickAndWait(catalogsPage.tabDetailsBtn); + // 3. verify show tab details + Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); + } + + @Test + @Order(7) + public void testBackHomePage() throws InterruptedException { + clickAndWait(catalogsPage.backHomeBtn); + Assertions.assertTrue(catalogsPage.verifyBackHomePage()); + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java index 59badd669d2..407a3bf59d6 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java @@ -42,17 +42,11 @@ public class CatalogsPageKafkaTest extends AbstractWebIT { private static final String SCHEMA_TOPIC_TITLE = "Topics"; private static final String METALAKE_NAME = "test"; private static final String CATALOG_TYPE_MESSAGING = "messaging"; - private static final String HIVE_CATALOG_NAME = "catalog_hive"; - private static final String MODIFIED_HIVE_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; - private static final String ICEBERG_CATALOG_NAME = "catalog_iceberg"; - private static final String FILESET_CATALOG_NAME = "catalog_fileset"; + private static final String KAFKA_CATALOG_NAME = "catalog_kafka"; private static final String SCHEMA_NAME = "default"; private static final String TOPIC_NAME = "topic1"; - - private static final String MYSQL_CATALOG_NAME = "catalog_mysql"; - - private static final String PG_CATALOG_NAME = "catalog_pg"; + public static final int DEFAULT_BROKER_PORT = 9092; @BeforeAll public static void before() throws Exception { @@ -63,7 +57,7 @@ public static void before() throws Exception { containerSuite.startKafkaContainer(); String address = containerSuite.getKafkaContainer().getContainerIpAddress(); - kafkaUri = String.format("%s:%s", address, "9092"); + kafkaUri = String.format("%s:%d", address, DEFAULT_BROKER_PORT); } /** @@ -116,7 +110,7 @@ public void testCreateKafkaCatalog() throws InterruptedException { clickAndWait(catalogsPage.createCatalogBtn); catalogsPage.setCatalogNameField(KAFKA_CATALOG_NAME); clickAndWait(catalogsPage.catalogTypeSelector); - catalogsPage.clickSelectType("messaging"); + catalogsPage.clickSelectType(CATALOG_TYPE_MESSAGING); catalogsPage.setCatalogCommentField("kafka catalog comment"); // set kafka catalog props catalogsPage.setCatalogFixedProp("bootstrap.servers", kafkaUri); @@ -135,15 +129,7 @@ public void testKafkaSchemaTreeNode() throws InterruptedException { // verify show table title、 schema name and tree node Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); - List treeNodes = - Arrays.asList( - MODIFIED_HIVE_CATALOG_NAME, - ICEBERG_CATALOG_NAME, - MYSQL_CATALOG_NAME, - PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME, - SCHEMA_NAME); + List treeNodes = Arrays.asList(KAFKA_CATALOG_NAME, SCHEMA_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); } @@ -161,16 +147,7 @@ public void testKafkaTopicTreeNode() throws InterruptedException { // 3. verify show table title、 default schema name and tree node Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TOPIC_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TOPIC_NAME, false)); - List treeNodes = - Arrays.asList( - MODIFIED_HIVE_CATALOG_NAME, - ICEBERG_CATALOG_NAME, - MYSQL_CATALOG_NAME, - PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME, - SCHEMA_NAME, - TOPIC_NAME); + List treeNodes = Arrays.asList(KAFKA_CATALOG_NAME, SCHEMA_NAME, TOPIC_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); } From aa9da3baf016d9b2a261b5032945923e5603d123 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 15 May 2024 15:35:01 +0800 Subject: [PATCH 2/4] [#3257] fix(web): add a default optional properties for fileset catalog (#3393) ### What changes were proposed in this pull request? Add a default optional properties for fileset catalog image Remove fix width of version component image image ### Why are the changes needed? Although the location property is optional, it is better to be there for ease of use The width of version component is should be flexible Fix: #3257 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? manual Co-authored-by: Qian Xia --- .../metalake/rightContent/CreateCatalogDialog.js | 4 ++-- web/src/app/rootLayout/VersionView.js | 2 +- web/src/lib/utils/initial.js | 9 ++++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index a6ff296de50..2e9b1ab24c4 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -236,7 +236,7 @@ const CreateCatalogDialog = props => { let properties = {} const prevProperties = innerProps - .filter(i => i.key.trim() !== '') + .filter(i => (typeSelect === 'fileset' && i.key === 'location' ? i.value.trim() !== '' : i.key.trim() !== '')) .reduce((acc, item) => { acc[item.key] = item.value @@ -363,7 +363,7 @@ const CreateCatalogDialog = props => { setProviderTypes(providersItems) const providerItem = providersItems.find(i => i.value === data.provider) - let propsItems = [...providerItem.defaultProps] + let propsItems = [...providerItem.defaultProps].filter(i => i.required) propsItems = propsItems.map((it, idx) => { let propItem = { diff --git a/web/src/app/rootLayout/VersionView.js b/web/src/app/rootLayout/VersionView.js index 8cde1aad75b..5831cff41b6 100644 --- a/web/src/app/rootLayout/VersionView.js +++ b/web/src/app/rootLayout/VersionView.js @@ -12,7 +12,7 @@ const VersionView = () => { const store = useAppSelector(state => state.sys) return ( - + {store.version} ) diff --git a/web/src/lib/utils/initial.js b/web/src/lib/utils/initial.js index 63c7f878969..035bb14601f 100644 --- a/web/src/lib/utils/initial.js +++ b/web/src/lib/utils/initial.js @@ -7,7 +7,14 @@ export const filesetProviders = [ { label: 'hadoop', value: 'hadoop', - defaultProps: [] + defaultProps: [ + { + key: 'location', + value: '', + required: false, + description: 'The storage location of the fileset' + } + ] } ] From 8be9686720aaee211f00d48bc1d36d3854abb3d9 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 15 May 2024 15:36:01 +0800 Subject: [PATCH 3/4] [#3291] fix(web): disable link dragging (#3394) ### What changes were proposed in this pull request? Disable link dragging. ### Why are the changes needed? Fix: #3291 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A Co-authored-by: CHEYNE --- web/src/lib/styles/globals.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/lib/styles/globals.css b/web/src/lib/styles/globals.css index 8cf4250b6ab..f8e6edce5a4 100644 --- a/web/src/lib/styles/globals.css +++ b/web/src/lib/styles/globals.css @@ -14,6 +14,10 @@ body { min-height: 100%; } +a { + -webkit-user-drag: none; +} + .logoText { font-family: 'Alata' !important; } From 98db7ecd13cc30b5b54c6bf6ce3fc8b9bf22c653 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 15 May 2024 15:36:41 +0800 Subject: [PATCH 4/4] [#3285] fix(web): disable the metalake select with only one (#3395) ### What changes were proposed in this pull request? disable the metalake select with only one image ### Why are the changes needed? With only one metalake UI presents user with a choice Fix: #3285 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? manual Co-authored-by: Qian Xia --- web/src/app/rootLayout/AppBar.js | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/app/rootLayout/AppBar.js b/web/src/app/rootLayout/AppBar.js index 94d885365b4..47b45c4e389 100644 --- a/web/src/app/rootLayout/AppBar.js +++ b/web/src/app/rootLayout/AppBar.js @@ -90,6 +90,7 @@ const AppBar = () => { data-refer='select-metalake' value={metalake} label='Metalake' + disabled={store.metalakes.length === 1} sx={{ width: '100%' }} > {metalakes.map(item => {