[apache#2292] add web ui support for fileset catalog

LauraXia123 · Mar 28, 2024 · 4fdedc3 · 4fdedc3
1 parent 5e2237b
commit 4fdedc3
Show file tree

Hide file tree

Showing 12 changed files with 437 additions and 105 deletions.
diff --git a/...test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/...test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java
@@ -29,6 +29,7 @@ public class CatalogsPageTest extends AbstractWebIT {
 
   private static final String metalakeName = "metalake_name";
   String catalogName = "catalog_name";
+  String catalogType = "relational";
   String modifiedCatalogName = catalogName + "_edited";
   String schemaName = "default";
   String tableName = "employee";
@@ -124,21 +125,22 @@ public void testEditCatalog() throws InterruptedException {
   @Test
   @Order(6)
   public void testClickCatalogLink() {
-    catalogsPage.clickCatalogLink(metalakeName, modifiedCatalogName);
+    catalogsPage.clickCatalogLink(metalakeName, modifiedCatalogName, catalogType);
     Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Schemas"));
   }
 
   @Test
   @Order(7)
   public void testClickSchemaLink() {
-    catalogsPage.clickSchemaLink(metalakeName, modifiedCatalogName, schemaName);
+    catalogsPage.clickSchemaLink(metalakeName, modifiedCatalogName, catalogType, schemaName);
     Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Tables"));
   }
 
   @Test
   @Order(8)
   public void testClickTableLink() {
-    catalogsPage.clickTableLink(metalakeName, modifiedCatalogName, schemaName, tableName);
+    catalogsPage.clickTableLink(
+        metalakeName, modifiedCatalogName, catalogType, schemaName, tableName);
     Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Columns"));
     Assertions.assertTrue(catalogsPage.verifyTableColumns());
   }

diff --git a/...st/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java b/...st/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java
@@ -174,9 +174,16 @@ public void clickDeleteCatalogBtn(String name) {
     }
   }
 
-  public void clickCatalogLink(String metalakeName, String catalogName) {
+  public void clickCatalogLink(String metalakeName, String catalogName, String catalogType) {
     try {
-      String xpath = "//a[@href='?metalake=" + metalakeName + "&catalog=" + catalogName + "']";
+      String xpath =
+          "//a[@href='?metalake="
+              + metalakeName
+              + "&catalog="
+              + catalogName
+              + "&type="
+              + catalogType
+              + "']";
       WebElement link = tableGrid.findElement(By.xpath(xpath));
       WebDriverWait wait = new WebDriverWait(driver, MAX_TIMEOUT);
       wait.until(ExpectedConditions.elementToBeClickable(By.xpath(xpath)));
@@ -186,13 +193,16 @@ public void clickCatalogLink(String metalakeName, String catalogName) {
     }
   }
 
-  public void clickSchemaLink(String metalakeName, String catalogName, String schemaName) {
+  public void clickSchemaLink(
+      String metalakeName, String catalogName, String catalogType, String schemaName) {
     try {
       String xpath =
           "//a[@href='?metalake="
               + metalakeName
               + "&catalog="
               + catalogName
+              + "&type="
+              + catalogType
               + "&schema="
               + schemaName
               + "']";
@@ -206,13 +216,19 @@ public void clickSchemaLink(String metalakeName, String catalogName, String sche
   }
 
   public void clickTableLink(
-      String metalakeName, String catalogName, String schemaName, String tableName) {
+      String metalakeName,
+      String catalogName,
+      String catalogType,
+      String schemaName,
+      String tableName) {
     try {
       String xpath =
           "//a[@href='?metalake="
               + metalakeName
               + "&catalog="
               + catalogName
+              + "&type="
+              + catalogType
               + "&schema="
               + schemaName
               + "&table="

diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js
@@ -22,7 +22,8 @@ import {
   removeExpandedNode,
   setSelectedNodes,
   setLoadedNodes,
-  getTableDetails
+  getTableDetails,
+  getFilesetDetails
 } from '@/lib/store/metalakes'
 
 import { extractPlaceholder } from '@/lib/utils'
@@ -47,6 +48,12 @@ const MetalakeTree = props => {
         const [metalake, catalog, schema, table] = pathArr
         dispatch(getTableDetails({ init: true, metalake, catalog, schema, table }))
       }
+    } else if (nodeProps.data.node === 'fileset') {
+      if (store.selectedNodes.includes(nodeProps.data.key)) {
+        const pathArr = extractPlaceholder(nodeProps.data.key)
+        const [metalake, catalog, schema, fileset] = pathArr
+        dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset }))
+      }
     } else {
       dispatch(setIntoTreeNodeWithFetch({ key: nodeProps.data.key }))
     }
@@ -143,6 +150,19 @@ const MetalakeTree = props => {
             <Icon icon={isHover !== nodeProps.data.key ? 'bx:table' : 'mdi:reload'} fontSize='inherit' />
           </IconButton>
         )
+      case 'fileset':
+        return (
+          <IconButton
+            disableRipple={!store.selectedNodes.includes(nodeProps.data.key)}
+            size='small'
+            sx={{ color: '#666' }}
+            onClick={e => handleClickIcon(e, nodeProps)}
+            onMouseEnter={e => onMouseEnter(e, nodeProps)}
+            onMouseLeave={e => onMouseLeave(e, nodeProps)}
+          >
+            <Icon icon={isHover !== nodeProps.data.key ? 'bx:file' : 'mdi:reload'} fontSize='inherit' />
+          </IconButton>
+        )
 
       default:
         return <></>

diff --git a/web/src/app/metalakes/metalake/MetalakeView.js b/web/src/app/metalakes/metalake/MetalakeView.js
@@ -18,12 +18,15 @@ import {
   fetchCatalogs,
   fetchSchemas,
   fetchTables,
+  fetchFilesets,
   getMetalakeDetails,
   getCatalogDetails,
   getSchemaDetails,
   getTableDetails,
+  getFilesetDetails,
   setSelectedNodes
 } from '@/lib/store/metalakes'
+import { type } from 'os'
 
 const MetalakeView = () => {
   const dispatch = useAppDispatch()
@@ -35,39 +38,51 @@ const MetalakeView = () => {
     const routeParams = {
       metalake: searchParams.get('metalake'),
       catalog: searchParams.get('catalog'),
+      type: searchParams.get('type'),
       schema: searchParams.get('schema'),
-      table: searchParams.get('table')
+      table: searchParams.get('table'),
+      fileset: searchParams.get('fileset')
     }
     if ([...searchParams.keys()].length) {
-      const { metalake, catalog, schema, table } = routeParams
+      const { metalake, catalog, type, schema, table, fileset } = routeParams
 
       if (paramsSize === 1 && metalake) {
         dispatch(fetchCatalogs({ init: true, page: 'metalakes', metalake }))
         dispatch(getMetalakeDetails({ metalake }))
       }
 
-      if (paramsSize === 2 && catalog) {
-        dispatch(fetchSchemas({ init: true, page: 'catalogs', metalake, catalog }))
-        dispatch(getCatalogDetails({ metalake, catalog }))
+      if (paramsSize === 3 && catalog) {
+        dispatch(fetchSchemas({ init: true, page: 'catalogs', metalake, catalog, type }))
+        dispatch(getCatalogDetails({ metalake, catalog, type }))
       }
 
-      if (paramsSize === 3 && catalog && schema) {
-        dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema }))
+      if (paramsSize === 4 && catalog && type && schema) {
+        if (type === 'fileset') {
+          dispatch(fetchFilesets({ init: true, page: 'schemas', metalake, catalog, schema }))
+        } else {
+          dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema }))
+        }
         dispatch(getSchemaDetails({ metalake, catalog, schema }))
       }
 
-      if (paramsSize === 4 && catalog && schema && table) {
+      if (paramsSize === 5 && catalog && schema && table) {
         dispatch(getTableDetails({ init: true, metalake, catalog, schema, table }))
       }
+
+      if (paramsSize === 5 && catalog && schema && fileset) {
+        dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset }))
+      }
     }
 
     dispatch(
       setSelectedNodes(
         routeParams.catalog
           ? [
-              `{{${routeParams.metalake}}}{{${routeParams.catalog}}}${
+              `{{${routeParams.metalake}}}{{${routeParams.catalog}}}{{${routeParams.type}}}${
                 routeParams.schema ? `{{${routeParams.schema}}}` : ''
-              }${routeParams.table ? `{{${routeParams.table}}}` : ''}`
+              }${routeParams.table ? `{{${routeParams.table}}}` : ''}${
+                routeParams.fileset ? `{{${routeParams.fileset}}}` : ''
+              }`
             ]
           : []
       )

diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js
@@ -43,7 +43,7 @@ import { useSearchParams } from 'next/navigation'
 const defaultValues = {
   name: '',
   type: 'relational',
-  provider: 'hive',
+  provider: '',
   comment: '',
   propItems: providers[0].defaultProps
 }
@@ -58,7 +58,7 @@ const schema = yup.object().shape({
       nameRegex,
       'This field must start with a letter or underscore, and can only contain letters, numbers, and underscores'
     ),
-  type: yup.mixed().oneOf(['relational']).required(),
+  type: yup.mixed().oneOf(['relational', 'fileset']).required(),
   provider: yup.mixed().oneOf(providerTypeValues).required(),
   propItems: yup.array().of(
     yup.object().shape({
@@ -87,6 +87,8 @@ const CreateCatalogDialog = props => {
 
   const [cacheData, setCacheData] = useState()
 
+  const [providerTypes, setProviderTypes] = useState(providers)
+
   const {
     control,
     reset,
@@ -103,6 +105,7 @@ const CreateCatalogDialog = props => {
   })
 
   const providerSelect = watch('provider')
+  const typeSelect = watch('type')
 
   const handleFormChange = ({ index, event }) => {
     let data = [...innerProps]
@@ -280,6 +283,18 @@ const CreateCatalogDialog = props => {
     console.error('fields error', errors)
   }
 
+  useEffect(() => {
+    if (typeSelect === 'fileset') {
+      setProviderTypes(providers.filter(p => p.value === 'hadoop'))
+      setValue('provider', 'hadoop')
+    } else {
+      setProviderTypes(providers.filter(p => p.value !== 'hadoop'))
+      setValue('provider', 'hive')
+    }
+
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [typeSelect, open])
+
   useEffect(() => {
     let defaultProps = []
 
@@ -410,6 +425,7 @@ const CreateCatalogDialog = props => {
                       disabled={type === 'update'}
                     >
                       <MenuItem value={'relational'}>relational</MenuItem>
+                      <MenuItem value={'fileset'}>fileset</MenuItem>
                     </Select>
                   )}
                 />
@@ -436,10 +452,13 @@ const CreateCatalogDialog = props => {
                       labelId='select-catalog-provider'
                       disabled={type === 'update'}
                     >
-                      <MenuItem value={'hive'}>hive</MenuItem>
-                      <MenuItem value={'lakehouse-iceberg'}>iceberg</MenuItem>
-                      <MenuItem value={'jdbc-mysql'}>mysql</MenuItem>
-                      <MenuItem value={'jdbc-postgresql'}>postgresql</MenuItem>
+                      {providerTypes.map(item => {
+                        return (
+                          <MenuItem key={item.label} value={item.value}>
+                            {item.label}
+                          </MenuItem>
+                        )
+                      })}
                     </Select>
                   )}
                 />

diff --git a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js
@@ -24,16 +24,19 @@ const MetalakePath = props => {
   const routeParams = {
     metalake: searchParams.get('metalake'),
     catalog: searchParams.get('catalog'),
+    type: searchParams.get('type'),
     schema: searchParams.get('schema'),
-    table: searchParams.get('table')
+    table: searchParams.get('table'),
+    fileset: searchParams.get('fileset')
   }
 
-  const { metalake, catalog, schema, table } = routeParams
+  const { metalake, catalog, type, schema, table, fileset } = routeParams
 
   const metalakeUrl = `?metalake=${metalake}`
-  const catalogUrl = `?metalake=${metalake}&catalog=${catalog}`
-  const schemaUrl = `?metalake=${metalake}&catalog=${catalog}&schema=${schema}`
-  const tableUrl = `?metalake=${metalake}&catalog=${catalog}&schema=${schema}&table=${table}`
+  const catalogUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}`
+  const schemaUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}`
+  const tableUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&table=${table}`
+  const filesetUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&fileset=${fileset}`
 
   const handleClick = (event, path) => {
     path === `?${searchParams.toString()}` && event.preventDefault()
@@ -91,6 +94,19 @@ const MetalakePath = props => {
           </MUILink>
         </Tooltip>
       )}
+      {fileset && (
+        <Tooltip title={fileset} placement='top'>
+          <MUILink
+            component={Link}
+            href={filesetUrl}
+            onClick={event => handleClick(event, filesetUrl)}
+            underline='hover'
+          >
+            <Icon icon='bx:file' fontSize={20} />
+            <Text>{fileset}</Text>
+          </MUILink>
+        </Tooltip>
+      )}
     </Breadcrumbs>
   )
 }