From 6bf3ae59bf4058841497eee7773adea58d6b6f7b Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Thu, 11 Apr 2024 14:57:31 +0800 Subject: [PATCH 001/106] [MINOR] improvement(catalog-doris): Upgrade Doris CI image version to 0.1.2 (#2878) ### What changes were proposed in this pull request? Change the Doris CI image version from 0.1.1 to 0.1.2. ### Why are the changes needed? 0.1.2 has several improvements to make the CI more stable. ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- catalogs/catalog-jdbc-doris/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index 24652996e8d..a19523406fa 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -79,7 +79,7 @@ tasks.test { dependsOn(tasks.jar) doFirst { - environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.1") + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit From 99f2ede127c6977c0cf7c2a137166a8ca780c369 Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Thu, 11 Apr 2024 16:37:04 +0800 Subject: [PATCH 002/106] [#2614][#2530]feat(web): Add web UI support for Kafka catalog and add default page for no data tree (#2863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …age for no data tree ### What changes were proposed in this pull request? 1. Add web UI support for Kafka catalog image 2. Add default page for no data tree image ### Why are the changes needed? N/A Fix: #2614, #2530 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Local Smoke Test and e2e test image --- .../app/metalakes/metalake/MetalakeTree.js | 154 +++++++++++------- .../app/metalakes/metalake/MetalakeView.js | 29 +++- .../rightContent/CreateCatalogDialog.js | 73 +++++++-- .../metalake/rightContent/MetalakePath.js | 14 +- .../rightContent/tabsContent/TabsContent.js | 13 +- .../tabsContent/detailsView/DetailsView.js | 52 +++++- web/src/app/rootLayout/AppBar.js | 4 +- web/src/lib/api/topics/index.js | 24 +++ web/src/lib/store/metalakes/index.js | 141 +++++++++++++++- web/src/lib/utils/initial.js | 22 ++- 10 files changed, 429 insertions(+), 97 deletions(-) create mode 100644 web/src/lib/api/topics/index.js diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index ec4e9c9af22..0b880448d9d 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -9,7 +9,7 @@ import { useEffect, useRef, useState } from 'react' import { useRouter } from 'next/navigation' -import { IconButton, Typography } from '@mui/material' +import { IconButton, Typography, Box } from '@mui/material' import { Tree } from 'antd' import Icon from '@/components/Icon' @@ -23,7 +23,8 @@ import { setSelectedNodes, setLoadedNodes, getTableDetails, - getFilesetDetails + getFilesetDetails, + getTopicDetails } from '@/lib/store/metalakes' import { extractPlaceholder } from '@/lib/utils' @@ -54,6 +55,8 @@ const MetalakeTree = props => { default: return 'bx:book' } + case 'messaging': + return 'skill-icons:kafka' case 'fileset': default: return 'bx:book' @@ -63,20 +66,33 @@ const MetalakeTree = props => { const handleClickIcon = (e, nodeProps) => { e.stopPropagation() - if (nodeProps.data.node === 'table') { - if (store.selectedNodes.includes(nodeProps.data.key)) { - const pathArr = extractPlaceholder(nodeProps.data.key) - const [metalake, catalog, schema, table] = pathArr - dispatch(getTableDetails({ init: true, metalake, catalog, schema, table })) + switch (nodeProps.data.node) { + case 'table': { + if (store.selectedNodes.includes(nodeProps.data.key)) { + const pathArr = extractPlaceholder(nodeProps.data.key) + const [metalake, catalog, schema, table] = pathArr + dispatch(getTableDetails({ init: true, metalake, catalog, schema, table })) + } + break } - } else if (nodeProps.data.node === 'fileset') { - if (store.selectedNodes.includes(nodeProps.data.key)) { - const pathArr = extractPlaceholder(nodeProps.data.key) - const [metalake, catalog, schema, fileset] = pathArr - dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) + case 'fileset': { + if (store.selectedNodes.includes(nodeProps.data.key)) { + const pathArr = extractPlaceholder(nodeProps.data.key) + const [metalake, catalog, schema, fileset] = pathArr + dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) + } + break } - } else { - dispatch(setIntoTreeNodeWithFetch({ key: nodeProps.data.key })) + case 'topic': { + if (store.selectedNodes.includes(nodeProps.data.key)) { + const pathArr = extractPlaceholder(nodeProps.data.key) + const [metalake, catalog, schema, topic] = pathArr + dispatch(getTopicDetails({ init: true, metalake, catalog, schema, topic })) + } + break + } + default: + dispatch(setIntoTreeNodeWithFetch({ key: nodeProps.data.key })) } } @@ -195,6 +211,22 @@ const MetalakeTree = props => { ) + case 'topic': + return ( + handleClickIcon(e, nodeProps)} + onMouseEnter={e => onMouseEnter(e, nodeProps)} + onMouseLeave={e => onMouseLeave(e, nodeProps)} + > + + + ) default: return <> @@ -225,51 +257,63 @@ const MetalakeTree = props => { useEffect(() => { if (store.selectedNodes.length !== 0) { - treeRef.current.scrollTo({ key: store.selectedNodes[0] }) + treeRef.current && treeRef.current.scrollTo({ key: store.selectedNodes[0] }) } - }, [store.selectedNodes]) + }, [store.selectedNodes, treeRef]) + + useEffect(() => { + dispatch(setExpandedNodes(store.expandedNodes)) + }, [store.metalakeTree, dispatch]) return ( <> - renderIcon(nodeProps)} - titleRender={nodeData => renderNode(nodeData)} - /> + {store.metalakeTree.length ? ( + renderIcon(nodeProps)} + titleRender={nodeData => renderNode(nodeData)} + /> + ) : ( + + theme.palette.text.primary }} data-refer='no-data'> + No data + + + )} ) } diff --git a/web/src/app/metalakes/metalake/MetalakeView.js b/web/src/app/metalakes/metalake/MetalakeView.js index e700ff145fc..44c3e2e2daf 100644 --- a/web/src/app/metalakes/metalake/MetalakeView.js +++ b/web/src/app/metalakes/metalake/MetalakeView.js @@ -19,11 +19,13 @@ import { fetchSchemas, fetchTables, fetchFilesets, + fetchTopics, getMetalakeDetails, getCatalogDetails, getSchemaDetails, getTableDetails, getFilesetDetails, + getTopicDetails, setSelectedNodes } from '@/lib/store/metalakes' @@ -40,10 +42,11 @@ const MetalakeView = () => { type: searchParams.get('type'), schema: searchParams.get('schema'), table: searchParams.get('table'), - fileset: searchParams.get('fileset') + fileset: searchParams.get('fileset'), + topic: searchParams.get('topic') } if ([...searchParams.keys()].length) { - const { metalake, catalog, type, schema, table, fileset } = routeParams + const { metalake, catalog, type, schema, table, fileset, topic } = routeParams if (paramsSize === 1 && metalake) { dispatch(fetchCatalogs({ init: true, page: 'metalakes', metalake })) @@ -56,10 +59,18 @@ const MetalakeView = () => { } if (paramsSize === 4 && catalog && type && schema) { - if (type === 'fileset') { - dispatch(fetchFilesets({ init: true, page: 'schemas', metalake, catalog, schema })) - } else { - dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema })) + switch (type) { + case 'relational': + dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema })) + break + case 'fileset': + dispatch(fetchFilesets({ init: true, page: 'schemas', metalake, catalog, schema })) + break + case 'messaging': + dispatch(fetchTopics({ init: true, page: 'schemas', metalake, catalog, schema })) + break + default: + break } dispatch(getSchemaDetails({ metalake, catalog, schema })) } @@ -71,6 +82,10 @@ const MetalakeView = () => { if (paramsSize === 5 && catalog && schema && fileset) { dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) } + + if (paramsSize === 5 && catalog && schema && topic) { + dispatch(getTopicDetails({ init: true, metalake, catalog, schema, topic })) + } } dispatch( @@ -81,7 +96,7 @@ const MetalakeView = () => { routeParams.schema ? `{{${routeParams.schema}}}` : '' }${routeParams.table ? `{{${routeParams.table}}}` : ''}${ routeParams.fileset ? `{{${routeParams.fileset}}}` : '' - }` + }${routeParams.topic ? `{{${routeParams.topic}}}` : ''}` ] : [] ) diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index 8e6bbde8244..25bb5108df6 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -36,7 +36,7 @@ import { yupResolver } from '@hookform/resolvers/yup' import { groupBy } from 'lodash-es' import { genUpdates } from '@/lib/utils' -import { providers } from '@/lib/utils/initial' +import { providers, filesetProviders, messagingProviders } from '@/lib/utils/initial' import { nameRegex, keyRegex } from '@/lib/utils/regex' import { useSearchParams } from 'next/navigation' @@ -45,11 +45,9 @@ const defaultValues = { type: 'relational', provider: '', comment: '', - propItems: providers[0].defaultProps + propItems: [] } -const providerTypeValues = providers.map(i => i.value) - const schema = yup.object().shape({ name: yup .string() @@ -58,8 +56,19 @@ const schema = yup.object().shape({ nameRegex, 'This field must start with a letter or underscore, and can only contain letters, numbers, and underscores' ), - type: yup.mixed().oneOf(['relational', 'fileset']).required(), - provider: yup.mixed().oneOf(providerTypeValues).required(), + type: yup.mixed().oneOf(['relational', 'fileset', 'messaging']).required(), + provider: yup.string().when('type', (type, schema) => { + switch (type) { + case 'relational': + return schema.oneOf(providers.map(i => i.value)).required() + case 'fileset': + return schema.oneOf(filesetProviders.map(i => i.value)).required() + case 'messaging': + return schema.oneOf(messagingProviders.map(i => i.value)).required() + default: + return schema + } + }), propItems: yup.array().of( yup.object().shape({ required: yup.boolean(), @@ -284,12 +293,22 @@ const CreateCatalogDialog = props => { } useEffect(() => { - if (typeSelect === 'fileset') { - setProviderTypes(providers.filter(p => p.value === 'hadoop')) - setValue('provider', 'hadoop') - } else { - setProviderTypes(providers.filter(p => p.value !== 'hadoop')) - setValue('provider', 'hive') + switch (typeSelect) { + case 'relational': { + setProviderTypes(providers) + setValue('provider', 'hive') + break + } + case 'fileset': { + setProviderTypes(filesetProviders) + setValue('provider', 'hadoop') + break + } + case 'messaging': { + setProviderTypes(messagingProviders) + setValue('provider', 'kafka') + break + } } // eslint-disable-next-line react-hooks/exhaustive-deps @@ -298,16 +317,16 @@ const CreateCatalogDialog = props => { useEffect(() => { let defaultProps = [] - const providerItemIndex = providers.findIndex(i => i.value === providerSelect) + const providerItemIndex = providerTypes.findIndex(i => i.value === providerSelect) if (providerItemIndex !== -1) { - defaultProps = providers[providerItemIndex].defaultProps + defaultProps = providerTypes[providerItemIndex].defaultProps - resetPropsFields(providers, providerItemIndex) + resetPropsFields(providerTypes, providerItemIndex) if (type === 'create') { setInnerProps(defaultProps) - setValue('propItems', providers[providerItemIndex].defaultProps) + setValue('propItems', providerTypes[providerItemIndex].defaultProps) } } @@ -324,7 +343,26 @@ const CreateCatalogDialog = props => { setValue('type', data.type) setValue('provider', data.provider) - const providerItem = providers.find(i => i.value === data.provider) + let providersItems = [] + + switch (data.type) { + case 'relational': { + providersItems = providers + break + } + case 'fileset': { + providersItems = filesetProviders + break + } + case 'messaging': { + providersItems = messagingProviders + break + } + } + + setProviderTypes(providersItems) + + const providerItem = providersItems.find(i => i.value === data.provider) let propsItems = [...providerItem.defaultProps] propsItems = propsItems.map((it, idx) => { @@ -427,6 +465,7 @@ const CreateCatalogDialog = props => { > relational fileset + messaging )} /> diff --git a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js index 4f7a400535a..9274780ef00 100644 --- a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js +++ b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js @@ -27,16 +27,18 @@ const MetalakePath = props => { type: searchParams.get('type'), schema: searchParams.get('schema'), table: searchParams.get('table'), - fileset: searchParams.get('fileset') + fileset: searchParams.get('fileset'), + topic: searchParams.get('topic') } - const { metalake, catalog, type, schema, table, fileset } = routeParams + const { metalake, catalog, type, schema, table, fileset, topic } = routeParams const metalakeUrl = `?metalake=${metalake}` const catalogUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}` const schemaUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}` const tableUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&table=${table}` const filesetUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&fileset=${fileset}` + const topicUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&topic=${topic}` const handleClick = (event, path) => { path === `?${searchParams.toString()}` && event.preventDefault() @@ -107,6 +109,14 @@ const MetalakePath = props => { )} + {topic && ( + + handleClick(event, topicUrl)} underline='hover'> + + {topic} + + + )} ) } diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js index 7ae1a86543b..f7dfb7b0435 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js @@ -54,7 +54,7 @@ const TabsContent = () => { const paramsSize = [...searchParams.keys()].length const type = searchParams.get('type') const [tab, setTab] = useState('table') - const isNotNeedTableTab = type && type === 'fileset' && paramsSize === 5 + const isNotNeedTableTab = type && ['fileset', 'messaging'].includes(type) && paramsSize === 5 const handleChangeTab = (event, newValue) => { setTab(newValue) @@ -68,7 +68,16 @@ const TabsContent = () => { tableTitle = 'Schemas' break case 4: - tableTitle = type === 'fileset' ? 'Filesets' : 'Tables' + switch (type) { + case 'fileset': + tableTitle = 'Filesets' + break + case 'messaging': + tableTitle = 'Topics' + break + default: + tableTitle = 'Tables' + } break case 5: tableTitle = 'Columns' diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js index 82aa6da023b..fb04dc81868 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js @@ -23,12 +23,24 @@ const DetailsView = () => { const audit = activatedItem?.audit || {} - const properties = Object.keys(activatedItem?.properties || []).map(item => { - return { - key: item, - value: JSON.stringify(activatedItem?.properties[item]).replace(/^"|"$/g, '') - } - }) + const properties = Object.keys(activatedItem?.properties || []) + .filter(key => !['partition-count', 'replication-factor'].includes(key)) + .map(item => { + return { + key: item, + value: JSON.stringify(activatedItem?.properties[item]).replace(/^"|"$/g, '') + } + }) + if (paramsSize === 5 && searchParams.get('topic')) { + properties.unshift({ + key: 'replication-factor', + value: JSON.stringify(activatedItem?.properties['replication-factor'])?.replace(/^"|"$/g, '') + }) + properties.unshift({ + key: 'partition-count', + value: JSON.stringify(activatedItem?.properties['partition-count'])?.replace(/^"|"$/g, '') + }) + } const renderFieldText = ({ value, linkBreak = false, isDate = false }) => { if (!value) { @@ -45,7 +57,7 @@ const DetailsView = () => { return ( - {paramsSize == 2 && searchParams.hasOwnProperty('catalog') ? ( + {paramsSize == 3 && searchParams.get('catalog') && searchParams.get('type') ? ( <> @@ -117,8 +129,30 @@ const DetailsView = () => { {properties.map((item, index) => { return ( - `${theme.spacing(2.75)} !important` }}>{item.key} - `${theme.spacing(2.75)} !important` }}>{item.value} + `${theme.spacing(2.75)} !important` }}> + + {item.key} + + + `${theme.spacing(2.75)} !important` }}> + + {item.value} + + ) })} diff --git a/web/src/app/rootLayout/AppBar.js b/web/src/app/rootLayout/AppBar.js index 0998151dadd..2e61c6b599b 100644 --- a/web/src/app/rootLayout/AppBar.js +++ b/web/src/app/rootLayout/AppBar.js @@ -38,13 +38,13 @@ const AppBar = () => { const router = useRouter() useEffect(() => { - if (!store.metalakes.length) { + if (!store.metalakes.length && metalake) { dispatch(fetchMetalakes()) } const metalakeItems = store.metalakes.map(i => i.name) setMetalakes(metalakeItems) // eslint-disable-next-line react-hooks/exhaustive-deps - }, [store.metalakes]) + }, [store.metalakes, metalake, dispatch]) return ( `/api/metalakes/${metalake}/catalogs/${catalog}/schemas/${schema}/topics`, + GET_DETAIL: ({ metalake, catalog, schema, topic }) => + `/api/metalakes/${metalake}/catalogs/${catalog}/schemas/${schema}/topics/${topic}` +} + +export const getTopicsApi = params => { + return defHttp.get({ + url: `${Apis.GET(params)}` + }) +} + +export const getTopicDetailsApi = ({ metalake, catalog, schema, topic }) => { + return defHttp.get({ + url: `${Apis.GET_DETAIL({ metalake, catalog, schema, topic })}` + }) +} diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js index b5b1458cd5e..912f7bac8a6 100644 --- a/web/src/lib/store/metalakes/index.js +++ b/web/src/lib/store/metalakes/index.js @@ -28,6 +28,7 @@ import { import { getSchemasApi, getSchemaDetailsApi } from '@/lib/api/schemas' import { getTablesApi, getTableDetailsApi } from '@/lib/api/tables' import { getFilesetsApi, getFilesetDetailsApi } from '@/lib/api/filesets' +import { getTopicsApi, getTopicDetailsApi } from '@/lib/api/topics' export const fetchMetalakes = createAsyncThunk('appMetalakes/fetchMetalakes', async (params, { getState }) => { const [err, res] = await to(getMetalakesApi()) @@ -136,7 +137,7 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk( children: [] } }) - } else if (pathArr.length === 4 && type !== 'fileset') { + } else if (pathArr.length === 4 && type === 'relational') { const [err, res] = await to(getTablesApi({ metalake, catalog, schema })) const { identifiers = [] } = res @@ -180,6 +181,27 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk( isLeaf: true } }) + } else if (pathArr.length === 4 && type === 'messaging') { + const [err, res] = await to(getTopicsApi({ metalake, catalog, schema })) + + const { identifiers = [] } = res + + if (err || !res) { + throw new Error(err) + } + + result.data = identifiers.map(topicItem => { + return { + ...topicItem, + node: 'topic', + id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${topicItem.name}}}`, + key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${topicItem.name}}}`, + path: `?${new URLSearchParams({ metalake, catalog, type, schema, topic: topicItem.name }).toString()}`, + name: topicItem.name, + title: topicItem.name, + isLeaf: true + } + }) } return result @@ -692,6 +714,104 @@ export const getFilesetDetails = createAsyncThunk( } ) +export const fetchTopics = createAsyncThunk( + 'appMetalakes/fetchTopics', + async ({ init, page, metalake, catalog, schema }, { getState, dispatch }) => { + if (init) { + dispatch(setTableLoading(true)) + } + + const [err, res] = await to(getTopicsApi({ metalake, catalog, schema })) + dispatch(setTableLoading(false)) + + if (init && (err || !res)) { + dispatch(resetTableData()) + throw new Error(err) + } + + const { identifiers = [] } = res + + const topics = identifiers.map(topic => { + return { + ...topic, + node: 'topic', + id: `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}{{${topic.name}}}`, + key: `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}{{${topic.name}}}`, + path: `?${new URLSearchParams({ + metalake, + catalog, + type: 'messaging', + schema, + topic: topic.name + }).toString()}`, + name: topic.name, + title: topic.name, + isLeaf: true + } + }) + + if ( + init && + getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}`) + ) { + dispatch( + setIntoTreeNodes({ + key: `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}`, + data: topics, + tree: getState().metalakes.metalakeTree + }) + ) + } + + if (getState().metalakes.metalakeTree.length === 0) { + dispatch(fetchCatalogs({ metalake })) + } + + dispatch( + setExpandedNodes([ + `{{${metalake}}}`, + `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}`, + `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}` + ]) + ) + + return { topics, page, init } + } +) + +export const getTopicDetails = createAsyncThunk( + 'appMetalakes/getTopicDetails', + async ({ init, metalake, catalog, schema, topic }, { getState, dispatch }) => { + dispatch(resetTableData()) + if (init) { + dispatch(setTableLoading(true)) + } + const [err, res] = await to(getTopicDetailsApi({ metalake, catalog, schema, topic })) + dispatch(setTableLoading(false)) + + if (err || !res) { + dispatch(resetTableData()) + throw new Error(err) + } + + const { topic: resTopic } = res + + if (getState().metalakes.metalakeTree.length === 0) { + dispatch(fetchCatalogs({ metalake })) + } + + dispatch( + setExpandedNodes([ + `{{${metalake}}}`, + `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}`, + `{{${metalake}}}{{${catalog}}}{{${'messaging'}}}{{${schema}}}` + ]) + ) + + return resTopic + } +) + export const appMetalakesSlice = createSlice({ name: 'appMetalakes', initialState: { @@ -703,6 +823,7 @@ export const appMetalakesSlice = createSlice({ tables: [], columns: [], filesets: [], + topics: [], metalakeTree: [], loadedNodes: [], selectedNodes: [], @@ -754,6 +875,7 @@ export const appMetalakesSlice = createSlice({ state.tables = [] state.columns = [] state.filesets = [] + state.topics = [] }, setTableLoading(state, action) { state.tableLoading = action.payload @@ -787,6 +909,7 @@ export const appMetalakesSlice = createSlice({ }) builder.addCase(setIntoTreeNodeWithFetch.fulfilled, (state, action) => { const { key, data, tree } = action.payload + state.metalakeTree = updateTreeData(tree, key, data) }) builder.addCase(setIntoTreeNodeWithFetch.rejected, (state, action) => { @@ -874,6 +997,22 @@ export const appMetalakesSlice = createSlice({ builder.addCase(getFilesetDetails.rejected, (state, action) => { toast.error(action.error.message) }) + builder.addCase(fetchTopics.fulfilled, (state, action) => { + state.topics = action.payload.topics + if (action.payload.init) { + state.tableData = action.payload.topics + } + }) + builder.addCase(fetchTopics.rejected, (state, action) => { + toast.error(action.error.message) + }) + builder.addCase(getTopicDetails.fulfilled, (state, action) => { + state.activatedDetails = action.payload + state.tableData = [] + }) + builder.addCase(getTopicDetails.rejected, (state, action) => { + toast.error(action.error.message) + }) } }) diff --git a/web/src/lib/utils/initial.js b/web/src/lib/utils/initial.js index 476879b1e4b..7679b566387 100644 --- a/web/src/lib/utils/initial.js +++ b/web/src/lib/utils/initial.js @@ -3,12 +3,30 @@ * This software is licensed under the Apache License version 2. */ -export const providers = [ +export const filesetProviders = [ { label: 'hadoop', value: 'hadoop', defaultProps: [] - }, + } +] + +export const messagingProviders = [ + { + label: 'kafka', + value: 'kafka', + defaultProps: [ + { + key: 'bootstrap.servers', + value: '', + required: true, + description: 'The Kafka broker(s) to connect to, allowing for multiple brokers by comma-separating them' + } + ] + } +] + +export const providers = [ { label: 'hive', value: 'hive', From 604e345aefb0f4b8f289d13a6ff686274811edcd Mon Sep 17 00:00:00 2001 From: Yuhui Date: Thu, 11 Apr 2024 17:55:24 +0800 Subject: [PATCH 003/106] [#2844] improve (test): Update the relations log file in the container in Docker Compose when CI fails. (#2847) ### What changes were proposed in this pull request? Use a Docker volume to map the log files directory for Hive and HDFS in the container to the host. When the task fails, upload the relations log file for debugging. ### Why are the changes needed? Fix: #2844 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? Manually test --- .../workflows/backend-integration-test.yml | 4 ++- .../test/container/HiveContainer.java | 2 +- .../test/container/TrinoITContainers.java | 11 +++++-- integration-test/trino-it/docker-compose.yaml | 4 ++- integration-test/trino-it/inspect_ip.sh | 2 +- integration-test/trino-it/launch.sh | 33 +++++++++++++++---- integration-test/trino-it/shutdown.sh | 9 +++++ 7 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index b3221e4c4cc..3cd93760e91 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -106,7 +106,9 @@ jobs: build/reports integration-test/build/*.log integration-test/build/*.tar + integration-test/build/trino-ci-container-log/hive/*.* + integration-test/build/trino-ci-container-log/hdfs/*.* distribution/package/logs/gravitino-server.out distribution/package/logs/gravitino-server.log catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file + catalogs/**/*.tar diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 26dc124ef16..db4a5f8e5ce 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -56,7 +56,7 @@ protected void setupContainer() { public void start() { try { super.start(); - Preconditions.check("Hive container startup failed!", checkContainerStatus(5)); + Preconditions.check("Hive container startup failed!", checkContainerStatus(10)); } finally { copyHiveLog(); } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java index 0b31794d023..6c3805cd10d 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java @@ -49,6 +49,12 @@ public void launch(int gravitinoServerPort) throws Exception { command, false, ProcessData.TypesOfData.STREAMS_MERGED, env); LOG.info("Command {} output:\n{}", command, output); + String outputString = output.toString(); + if (Strings.isNotEmpty(outputString) + && !outputString.contains("All docker compose service is now available")) { + throw new ContainerLaunchException("Failed to start containers:\n " + outputString); + } + resolveServerAddress(); } @@ -67,7 +73,8 @@ private void resolveServerAddress() throws Exception { String containerIpMapping = output.toString(); if (containerIpMapping.isEmpty()) { - throw new ContainerLaunchException("Missing to get container status"); + throw new ContainerLaunchException( + "Failed to get the container status, the containers have not started"); } try { @@ -96,7 +103,7 @@ private void resolveServerAddress() throws Exception { for (String serviceName : servicesName) { if (!servicesUri.containsKey(serviceName)) { throw new ContainerLaunchException( - "The container for the {} service is not started: " + serviceName); + String.format("The container for the %s service is not started: ", serviceName)); } } } diff --git a/integration-test/trino-it/docker-compose.yaml b/integration-test/trino-it/docker-compose.yaml index 427941289bd..6a6b34a8400 100644 --- a/integration-test/trino-it/docker-compose.yaml +++ b/integration-test/trino-it/docker-compose.yaml @@ -15,11 +15,13 @@ services: entrypoint: /bin/bash /tmp/hive/init.sh volumes: - ./init/hive:/tmp/hive + - ../build/trino-ci-container-log/hive:/tmp/root + - ../build/trino-ci-container-log/hdfs:/usr/local/hadoop/logs healthcheck: test: ["CMD", "/tmp/check-status.sh"] interval: 10s timeout: 60s - retries: 5 + retries: 10 postgresql: image: postgres:13 diff --git a/integration-test/trino-it/inspect_ip.sh b/integration-test/trino-it/inspect_ip.sh index b2d539a11c6..aa0152320f3 100755 --- a/integration-test/trino-it/inspect_ip.sh +++ b/integration-test/trino-it/inspect_ip.sh @@ -4,4 +4,4 @@ # This software is licensed under the Apache License version 2. # -docker inspect --format='{{.Name}}:{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq) |grep "/trino-ci-" | sed 's/\/trino-ci-//g' +docker inspect --format='{{.Name}}:{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q) |grep "/trino-ci-" | sed 's/\/trino-ci-//g' diff --git a/integration-test/trino-it/launch.sh b/integration-test/trino-it/launch.sh index 1b349131c90..2de48f86d7e 100755 --- a/integration-test/trino-it/launch.sh +++ b/integration-test/trino-it/launch.sh @@ -17,6 +17,9 @@ fi cd ${playground_dir} +# create log dir +mkdir -p ../build/trino-ci-container-log + docker compose up -d if [ -n "$GRAVITINO_LOG_PATH" ]; then @@ -29,15 +32,33 @@ echo "The docker compose log is: $LOG_PATH" nohup docker compose logs -f -t >> $LOG_PATH & -max_attempts=600 +max_attempts=300 +attempts=0 -for ((i = 0; i < max_attempts; i++)); do +while true; do docker compose exec -T trino trino --execute "SELECT 1" >/dev/null 2>&1 && { - echo "All docker compose service is now available." - exit 0 + break; } + + num_container=$(docker ps --format '{{.Names}}' | grep trino-ci | wc -l) + if [ "$num_container" -lt 4 ]; then + echo "ERROR: Trino-ci containers start failed." + exit 0 + fi + + if [ "$attempts" -ge "$max_attempts" ]; then + echo "ERROR: Trino service did not start within the $max_attempts time." + exit 1 + fi + + ((attempts++)) sleep 1 done -echo "Trino service did not start within the specified time." -exit 1 + +echo "All docker compose service is now available." + +# change the hive container's logs directory permission +docker exec trino-ci-hive chown -R `id -u`:`id -g` /tmp/root +docker exec trino-ci-hive chown -R `id -u`:`id -g` /usr/local/hadoop/logs + diff --git a/integration-test/trino-it/shutdown.sh b/integration-test/trino-it/shutdown.sh index 69c6281d35c..e2621cfd491 100755 --- a/integration-test/trino-it/shutdown.sh +++ b/integration-test/trino-it/shutdown.sh @@ -5,4 +5,13 @@ # cd "$(dirname "$0")" +# change the hive container's logs directory permission +docker exec trino-ci-hive chown -R `id -u`:`id -g` /tmp/root +docker exec trino-ci-hive chown -R `id -u`:`id -g` /usr/local/hadoop/logs + +# for trace file permission +ls -l ../build/trino-ci-container-log +ls -l ../build/trino-ci-container-log/hive +ls -l ../build/trino-ci-container-log/hdfs + docker compose down From b1c5f6529443ce78c73a80411170c3ff6ea6fc33 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 12 Apr 2024 09:51:43 +0800 Subject: [PATCH 004/106] [#2655] improvement(CI): Add a step in CI pipeline to verify `gradle publishToMavenLocal` (#2718) ### What changes were proposed in this pull request? Add a new pipeline to test if `./gradlew publishToMavenLocal` works. ### Why are the changes needed? Some changes may make `gradle publishToMavenLocal` can't work accidently and we can't detect it currently. Fix: #2655 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 93f9ec9af91..f339bb5bbac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,6 +85,9 @@ jobs: java-version: ${{ matrix.java-version }} distribution: 'temurin' + - name: Test publish to local + run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }} + - name: Build with Gradle run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} From 2b32efef3597c37abccd47142a06c308cc5ecf6e Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Fri, 12 Apr 2024 12:52:51 +1000 Subject: [PATCH 005/106] [MINOR] Give file correct .md extension (#2881) ### What changes were proposed in this pull request? Rename ROADMAP to ROADMAP.md ### Why are the changes needed? The file contains markdown. Fix: # N/A ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- ROADMAP => ROADMAP.md | 0 build.gradle.kts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ROADMAP => ROADMAP.md (100%) diff --git a/ROADMAP b/ROADMAP.md similarity index 100% rename from ROADMAP rename to ROADMAP.md diff --git a/build.gradle.kts b/build.gradle.kts index 98b1d48f011..7d383ced5ff 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -538,7 +538,7 @@ tasks.rat { "web/pnpm-lock.yaml", "**/LICENSE.*", "**/NOTICE.*", - "ROADMAP", + "ROADMAP.md", "clients/client-python/.pytest_cache/*" ) From d3023dc9b37acca182925ec6abb5549d09b68b0b Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Fri, 12 Apr 2024 11:03:11 +0800 Subject: [PATCH 006/106] [#2864] test(web): Add test case of create kafka catalog (#2893) ### What changes were proposed in this pull request? Add test case of create kafka catalog ### Why are the changes needed? Fix: #2864 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? image --- .../test/web/ui/CatalogsPageTest.java | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java index 133af12d523..936bd17a15f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java @@ -47,6 +47,7 @@ public class CatalogsPageTest extends AbstractWebIT { protected static String hdfsUri = "hdfs://127.0.0.1:9000"; protected static String mysqlUri = "jdbc:mysql://127.0.0.1"; protected static String postgresqlUri = "jdbc:postgresql://127.0.0.1"; + protected static String kafkaUri = "http://127.0.0.1:9092"; private static final String WEB_TITLE = "Gravitino"; private static final String CATALOG_TABLE_TITLE = "Schemas"; @@ -60,6 +61,7 @@ public class CatalogsPageTest extends AbstractWebIT { private static final String MODIFIED_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; private static final String ICEBERG_CATALOG_NAME = "catalog_iceberg"; private static final String FILESET_CATALOG_NAME = "catalog_fileset"; + private static final String KAFKA_CATALOG_NAME = "catalog_kafka"; private static final String SCHEMA_NAME = "default"; private static final String TABLE_NAME = "table1"; private static final String TABLE_NAME_2 = "table2"; @@ -233,6 +235,20 @@ public void testCreateFilesetCatalog() throws InterruptedException { @Test @Order(6) + public void testCreateKafkaCatalog() throws InterruptedException { + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(KAFKA_CATALOG_NAME); + clickAndWait(catalogsPage.catalogTypeSelector); + catalogsPage.clickSelectType("messaging"); + catalogsPage.setCatalogCommentField("kafka catalog comment"); + // set kafka catalog props + catalogsPage.setCatalogFixedProp("bootstrap.servers", kafkaUri); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(KAFKA_CATALOG_NAME)); + } + + @Test + @Order(7) public void testRefreshPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -243,12 +259,13 @@ public void testRefreshPage() { ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME); + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyCreatedCatalogs(catalogsNames)); } @Test - @Order(7) + @Order(8) public void testViewTabMetalakeDetails() throws InterruptedException { clickAndWait(catalogsPage.tabDetailsBtn); Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); @@ -257,7 +274,7 @@ public void testViewTabMetalakeDetails() throws InterruptedException { } @Test - @Order(8) + @Order(9) public void testViewCatalogDetails() throws InterruptedException { catalogsPage.clickViewCatalogBtn(HIVE_CATALOG_NAME); Assertions.assertTrue( @@ -265,7 +282,7 @@ public void testViewCatalogDetails() throws InterruptedException { } @Test - @Order(9) + @Order(10) public void testEditCatalog() throws InterruptedException { catalogsPage.clickEditCatalogBtn(HIVE_CATALOG_NAME); catalogsPage.setCatalogNameField(MODIFIED_CATALOG_NAME); @@ -275,7 +292,7 @@ public void testEditCatalog() throws InterruptedException { // test catalog show schema list @Test - @Order(10) + @Order(11) public void testClickCatalogLink() { catalogsPage.clickCatalogLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); @@ -284,7 +301,7 @@ public void testClickCatalogLink() { } @Test - @Order(11) + @Order(12) public void testRefreshCatalogPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -297,14 +314,15 @@ public void testRefreshCatalogPage() { ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME); + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); } // test schema show table list @Test - @Order(12) + @Order(13) public void testClickSchemaLink() { // create table createTableAndColumn( @@ -316,7 +334,7 @@ public void testClickSchemaLink() { } @Test - @Order(13) + @Order(14) public void testRefreshSchemaPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -330,14 +348,15 @@ public void testRefreshSchemaPage() { ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME); + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); } // test table show column list @Test - @Order(14) + @Order(15) public void testClickTableLink() { catalogsPage.clickTableLink( METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); @@ -348,7 +367,7 @@ public void testClickTableLink() { } @Test - @Order(15) + @Order(16) public void testRefreshTablePage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -364,12 +383,13 @@ public void testRefreshTablePage() { ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME); + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); } @Test - @Order(16) + @Order(17) public void testSelectMetalake() throws InterruptedException { catalogsPage.metalakeSelectChange(METALAKE_SELECT_NAME); Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); @@ -379,7 +399,7 @@ public void testSelectMetalake() throws InterruptedException { } @Test - @Order(17) + @Order(18) public void testClickTreeList() throws InterruptedException { String icebergNode = String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, ICEBERG_CATALOG_NAME, CATALOG_TYPE); @@ -420,7 +440,7 @@ public void testClickTreeList() throws InterruptedException { } @Test - @Order(18) + @Order(19) public void testTreeNodeRefresh() throws InterruptedException { createTableAndColumn( METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME_2, COLUMN_NAME_2); @@ -443,7 +463,7 @@ public void testTreeNodeRefresh() throws InterruptedException { } @Test - @Order(19) + @Order(20) public void testBackHomePage() throws InterruptedException { clickAndWait(catalogsPage.backHomeBtn); Assertions.assertTrue(catalogsPage.verifyBackHomePage()); From abcf5f7923c8abe8e6867cd3adcf9715798140ac Mon Sep 17 00:00:00 2001 From: mchades Date: Fri, 12 Apr 2024 14:16:07 +0800 Subject: [PATCH 007/106] [#2795] fix(kafka-catalog): Fix Kafka catalog cannot be dropped (#2865) ### What changes were proposed in this pull request? - move creating default schema to catalog initialization - cascade droping the default schema when drop Kafka catalog ### Why are the changes needed? because we don't allow to drop the default schema in Kafka catalog Fix: #2795 ### Does this PR introduce _any_ user-facing change? yes, user now can drop Kafka catalog successfully ### How was this patch tested? tests added --- .../catalog/kafka/KafkaCatalogOperations.java | 10 +-- .../integration/test/CatalogKafkaIT.java | 73 ++++++++++++++++--- .../gravitino/catalog/CatalogManager.java | 30 +++++++- 3 files changed, 92 insertions(+), 21 deletions(-) diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java index db088dee5ef..938c73e5813 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java @@ -133,6 +133,7 @@ public void initialize(Map config, CatalogInfo info) throws Runt AdminClientConfig.CLIENT_ID_CONFIG, String.format(CLIENT_ID_TEMPLATE, config.get(ID_KEY), info.namespace(), info.name())); + createDefaultSchemaIfNecessary(); adminClient = AdminClient.create(adminClientConfig); } @@ -328,7 +329,6 @@ public boolean dropTopic(NameIdentifier ident) { @Override public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogException { - createDefaultSchemaIfNecessary(); try { List schemas = store.list(namespace, SchemaEntity.class, Entity.EntityType.SCHEMA); @@ -352,7 +352,6 @@ public Schema createSchema(NameIdentifier ident, String comment, Map buildNewTopicConfigs(Map properties) return topicConfigs; } - private synchronized void createDefaultSchemaIfNecessary() { + private void createDefaultSchemaIfNecessary() { // If the default schema already exists, do nothing try { if (store.exists(defaultSchemaIdent, Entity.EntityType.SCHEMA)) { diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 765d3364b0d..5c518b7fec5 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -10,9 +10,11 @@ import static com.datastrato.gravitino.integration.test.container.KafkaContainer.DEFAULT_BROKER_PORT; import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.integration.test.container.ContainerSuite; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; @@ -29,6 +31,7 @@ import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -67,7 +70,7 @@ public class CatalogKafkaIT extends AbstractIT { private static AdminClient adminClient; @BeforeAll - public static void startUp() { + public static void startUp() throws ExecutionException, InterruptedException { CONTAINER_SUITE.startKafkaContainer(); kafkaBootstrapServers = String.format( @@ -75,13 +78,24 @@ public static void startUp() { CONTAINER_SUITE.getKafkaContainer().getContainerIpAddress(), DEFAULT_BROKER_PORT); adminClient = AdminClient.create(ImmutableMap.of(BOOTSTRAP_SERVERS, kafkaBootstrapServers)); + // create topics for testing + adminClient + .createTopics( + ImmutableList.of( + new NewTopic("topic1", 1, (short) 1), + new NewTopic("topic2", 1, (short) 1), + new NewTopic("topic3", 1, (short) 1))) + .all() + .get(); + createMetalake(); - createCatalog(); + Map properties = Maps.newHashMap(); + properties.put(BOOTSTRAP_SERVERS, kafkaBootstrapServers); + catalog = createCatalog(CATALOG_NAME, "Kafka catalog for IT", properties); } @AfterAll public static void shutdown() { - // todo: add drop catalog after it's supported client.dropMetalake(NameIdentifier.of(METALAKE_NAME)); if (adminClient != null) { adminClient.close(); @@ -94,6 +108,43 @@ public static void shutdown() { } } + @Test + public void testCatalog() throws ExecutionException, InterruptedException { + // test create catalog + String catalogName = GravitinoITUtils.genRandomName("test-catalog"); + String comment = "test catalog"; + Map properties = + ImmutableMap.of(BOOTSTRAP_SERVERS, kafkaBootstrapServers, "key1", "value1"); + Catalog createdCatalog = createCatalog(catalogName, comment, properties); + Assertions.assertEquals(catalogName, createdCatalog.name()); + Assertions.assertEquals(comment, createdCatalog.comment()); + Assertions.assertEquals(properties, createdCatalog.properties()); + + // test load catalog + Catalog loadedCatalog = metalake.loadCatalog(NameIdentifier.of(METALAKE_NAME, catalogName)); + Assertions.assertEquals(createdCatalog, loadedCatalog); + + // test alter catalog + Catalog alteredCatalog = + metalake.alterCatalog( + NameIdentifier.of(METALAKE_NAME, catalogName), + CatalogChange.updateComment("new comment"), + CatalogChange.removeProperty("key1")); + Assertions.assertEquals("new comment", alteredCatalog.comment()); + Assertions.assertFalse(alteredCatalog.properties().containsKey("key1")); + + // test drop catalog + boolean dropped = metalake.dropCatalog(NameIdentifier.of(METALAKE_NAME, catalogName)); + Assertions.assertTrue(dropped); + Exception exception = + Assertions.assertThrows( + NoSuchCatalogException.class, + () -> metalake.loadCatalog(NameIdentifier.of(METALAKE_NAME, catalogName))); + Assertions.assertTrue(exception.getMessage().contains(catalogName)); + // assert topic exists in Kafka after catalog dropped + Assertions.assertFalse(adminClient.listTopics().names().get().isEmpty()); + } + @Test public void testDefaultSchema() { NameIdentifier[] schemas = @@ -181,8 +232,9 @@ public void testCreateAndListTopic() throws ExecutionException, InterruptedExcep catalog .asTopicCatalog() .listTopics(Namespace.ofTopic(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME)); - Assertions.assertEquals(1, topics.length); - Assertions.assertEquals(topicName, topics[0].name()); + Assertions.assertTrue(topics.length > 0); + Assertions.assertTrue( + ImmutableList.copyOf(topics).stream().anyMatch(topic -> topic.name().equals(topicName))); } @Test @@ -353,15 +405,14 @@ private static void createMetalake() { metalake = loadMetalake; } - private static void createCatalog() { - Map properties = Maps.newHashMap(); - properties.put(BOOTSTRAP_SERVERS, kafkaBootstrapServers); + private static Catalog createCatalog( + String catalogName, String comment, Map properties) { metalake.createCatalog( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME), + NameIdentifier.of(METALAKE_NAME, catalogName), Catalog.Type.MESSAGING, PROVIDER, - "comment", + comment, properties); - catalog = metalake.loadCatalog(NameIdentifier.of(METALAKE_NAME, CATALOG_NAME)); + return metalake.loadCatalog(NameIdentifier.of(METALAKE_NAME, catalogName)); } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index f70d1f8d170..f374ce6e204 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -33,6 +33,7 @@ import com.datastrato.gravitino.messaging.TopicCatalog; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.rel.SupportsSchemas; import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.storage.IdGenerator; @@ -318,6 +319,7 @@ public Catalog createCatalog( .build()) .build(); + boolean createSuccess = false; try { NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); if (!store.exists(metalakeIdent, EntityType.METALAKE)) { @@ -325,9 +327,9 @@ public Catalog createCatalog( throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); } - // TODO: should avoid a race condition here - CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e)); store.put(e, false /* overwrite */); + CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e)); + createSuccess = true; return wrapper.catalog; } catch (EntityAlreadyExistsException e1) { LOG.warn("Catalog {} already exists", ident, e1); @@ -338,6 +340,14 @@ public Catalog createCatalog( catalogCache.invalidate(ident); LOG.error("Failed to create catalog {}", ident, e3); throw new RuntimeException(e3); + } finally { + if (!createSuccess) { + try { + store.delete(ident, EntityType.CATALOG); + } catch (IOException e4) { + LOG.error("Failed to clean up catalog {}", ident, e4); + } + } } } @@ -461,7 +471,23 @@ public boolean dropCatalog(NameIdentifier ident) { catalogCache.invalidate(ident); try { + CatalogEntity catalogEntity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); + if (catalogEntity.getProvider().equals("kafka")) { + // Kafka catalog needs to cascade drop the default schema + List schemas = + store.list( + Namespace.ofSchema(ident.namespace().level(0), ident.name()), + SchemaEntity.class, + EntityType.SCHEMA); + // If there is only one schema, it must be the default schema, because we don't allow to + // drop the default schema. + if (schemas.size() == 1) { + return store.delete(ident, EntityType.CATALOG, true); + } + } return store.delete(ident, EntityType.CATALOG); + } catch (NoSuchEntityException e) { + return false; } catch (IOException ioe) { LOG.error("Failed to drop catalog {}", ident, ioe); throw new RuntimeException(ioe); From cd2e0e759ef7c3bdb10768bc9d5a74ec46e87f12 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 12 Apr 2024 15:21:16 +0800 Subject: [PATCH 008/106] [#2527] fix: Fix catalog isolated classloaders can't be GC (#2548) ### What changes were proposed in this pull request? Call `AbandonedConnectionCleanupThread.uncheckedShutdown()` to close the checking thread for connection. ### Why are the changes needed? It's a bug. Fix: #2527 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Test locally. --- .../catalog/jdbc/JdbcCatalogOperations.java | 12 +++ .../catalog/mysql/MySQLCatalogOperations.java | 57 +++++++++++ .../gravitino/catalog/mysql/MysqlCatalog.java | 16 +++- .../PostgreSQLCatalogOperations.java | 47 +++++++++ .../catalog/postgresql/PostgreSqlCatalog.java | 14 +++ .../iceberg/ops/IcebergTableOps.java | 61 +++++++++++- .../gravitino/utils/IsolatedClassLoader.java | 95 +++++++++++-------- 7 files changed, 257 insertions(+), 45 deletions(-) create mode 100644 catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java create mode 100644 catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java index 874edd33b00..9463478ce1c 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java @@ -38,9 +38,13 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.utils.IsolatedClassLoader; import com.datastrato.gravitino.utils.MapUtils; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -510,4 +514,12 @@ public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationE throw new UnsupportedOperationException( "Jdbc catalog doesn't support topic related operations"); } + + public void deregisterDriver(Driver driver) throws SQLException { + if (driver.getClass().getClassLoader().getClass() + == IsolatedClassLoader.CUSTOM_CLASS_LOADER_CLASS) { + DriverManager.deregisterDriver(driver); + LOG.info("Driver {} has been deregistered...", driver); + } + } } diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java new file mode 100644 index 00000000000..403baff9469 --- /dev/null +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.mysql; + +import com.datastrato.gravitino.catalog.jdbc.JdbcCatalogOperations; +import com.datastrato.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import java.sql.Driver; +import java.sql.DriverManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySQLCatalogOperations extends JdbcCatalogOperations { + private static final Logger LOG = LoggerFactory.getLogger(MySQLCatalogOperations.class); + + public MySQLCatalogOperations( + JdbcExceptionConverter exceptionConverter, + JdbcTypeConverter jdbcTypeConverter, + JdbcDatabaseOperations databaseOperation, + JdbcTableOperations tableOperation, + JdbcTablePropertiesMetadata jdbcTablePropertiesMetadata, + JdbcColumnDefaultValueConverter columnDefaultValueConverter) { + super( + exceptionConverter, + jdbcTypeConverter, + databaseOperation, + tableOperation, + jdbcTablePropertiesMetadata, + columnDefaultValueConverter); + } + + @Override + public void close() { + super.close(); + + try { + // Close thread AbandonedConnectionCleanupThread + Class.forName("com.mysql.cj.jdbc.AbandonedConnectionCleanupThread") + .getMethod("uncheckedShutdown") + .invoke(null); + LOG.info("AbandonedConnectionCleanupThread has been shutdown..."); + + // Unload the MySQL driver, only Unload the driver if it is loaded by + // IsolatedClassLoader. + Driver mysqlDriver = DriverManager.getDriver("jdbc:mysql://dumpy_address"); + deregisterDriver(mysqlDriver); + } catch (Exception e) { + LOG.warn("Failed to shutdown AbandonedConnectionCleanupThread or deregister MySQL driver", e); + } + } +} diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java index 04ad9d009bc..6824d7b46c0 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java @@ -16,6 +16,8 @@ import com.datastrato.gravitino.catalog.mysql.converter.MysqlTypeConverter; import com.datastrato.gravitino.catalog.mysql.operation.MysqlDatabaseOperations; import com.datastrato.gravitino.catalog.mysql.operation.MysqlTableOperations; +import com.datastrato.gravitino.connector.CatalogOperations; +import java.util.Map; /** Implementation of a Mysql catalog in Gravitino. */ public class MysqlCatalog extends JdbcCatalog { @@ -25,13 +27,25 @@ public String shortName() { return "jdbc-mysql"; } + @Override + protected CatalogOperations newOps(Map config) { + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + return new MySQLCatalogOperations( + createExceptionConverter(), + jdbcTypeConverter, + createJdbcDatabaseOperations(), + createJdbcTableOperations(), + createJdbcTablePropertiesMetadata(), + createJdbcColumnDefaultValueConverter()); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new MysqlExceptionConverter(); } @Override - protected JdbcTypeConverter createJdbcTypeConverter() { + protected JdbcTypeConverter createJdbcTypeConverter() { return new MysqlTypeConverter(); } diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java new file mode 100644 index 00000000000..a3e7bd3ebe2 --- /dev/null +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.postgresql; + +import com.datastrato.gravitino.catalog.jdbc.JdbcCatalogOperations; +import com.datastrato.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import java.sql.Driver; +import java.sql.DriverManager; + +public class PostgreSQLCatalogOperations extends JdbcCatalogOperations { + + public PostgreSQLCatalogOperations( + JdbcExceptionConverter exceptionConverter, + JdbcTypeConverter jdbcTypeConverter, + JdbcDatabaseOperations databaseOperation, + JdbcTableOperations tableOperation, + JdbcTablePropertiesMetadata jdbcTablePropertiesMetadata, + JdbcColumnDefaultValueConverter columnDefaultValueConverter) { + super( + exceptionConverter, + jdbcTypeConverter, + databaseOperation, + tableOperation, + jdbcTablePropertiesMetadata, + columnDefaultValueConverter); + } + + @Override + public void close() { + super.close(); + try { + // Unload the PostgreSQL driver, only Unload the driver if it is loaded by + // IsolatedClassLoader. + Driver pgDriver = DriverManager.getDriver("jdbc:postgresql://dummy_address:dummy_port/"); + deregisterDriver(pgDriver); + } catch (Exception e) { + LOG.warn("Failed to deregister PostgreSQL driver", e); + } + } +} diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java index e3644b6e088..2ee675268b8 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java @@ -15,6 +15,8 @@ import com.datastrato.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter; import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlSchemaOperations; import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlTableOperations; +import com.datastrato.gravitino.connector.CatalogOperations; +import java.util.Map; public class PostgreSqlCatalog extends JdbcCatalog { @@ -23,6 +25,18 @@ public String shortName() { return "jdbc-postgresql"; } + @Override + protected CatalogOperations newOps(Map config) { + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + return new PostgreSQLCatalogOperations( + createExceptionConverter(), + jdbcTypeConverter, + createJdbcDatabaseOperations(), + createJdbcTableOperations(), + createJdbcTablePropertiesMetadata(), + createJdbcColumnDefaultValueConverter()); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new PostgreSqlExceptionConverter(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java index d8441cbee81..8288d615600 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java @@ -8,7 +8,10 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper.IcebergTableChange; import com.datastrato.gravitino.catalog.lakehouse.iceberg.utils.IcebergCatalogUtil; +import com.datastrato.gravitino.utils.IsolatedClassLoader; import com.google.common.base.Preconditions; +import java.sql.Driver; +import java.sql.DriverManager; import java.util.Collections; import java.util.Optional; import javax.ws.rs.NotSupportedException; @@ -29,17 +32,22 @@ import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class IcebergTableOps implements AutoCloseable { + public static final Logger LOG = LoggerFactory.getLogger(IcebergTableOps.class); protected Catalog catalog; private SupportsNamespaces asNamespaceCatalog; + private final String catalogType; + private String catalogUri = null; public IcebergTableOps(IcebergConfig icebergConfig) { - String catalogType = icebergConfig.get(IcebergConfig.CATALOG_BACKEND); + this.catalogType = icebergConfig.get(IcebergConfig.CATALOG_BACKEND); if (!IcebergCatalogBackend.MEMORY.name().equalsIgnoreCase(catalogType)) { icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE); - icebergConfig.get(IcebergConfig.CATALOG_URI); + this.catalogUri = icebergConfig.get(IcebergConfig.CATALOG_URI); } catalog = IcebergCatalogUtil.loadCatalogBackend(catalogType, icebergConfig.getAllConfig()); if (catalog instanceof SupportsNamespaces) { @@ -141,5 +149,54 @@ public void close() throws Exception { // JdbcCatalog need close. ((AutoCloseable) catalog).close(); } + + // Because each catalog in Gravitino has its own classloader, after a catalog is no longer used + // for a long time or dropped, the instance of classloader needs to be released. In order to + // let JVM GC remove the classloader, we need to release the resources of the classloader. The + // resources include the driver of the catalog backend and the + // AbandonedConnectionCleanupThread of MySQL. For more information about + // AbandonedConnectionCleanupThread, please refer to the corresponding java doc of MySQL + // driver. + if (catalogUri != null && catalogUri.contains("mysql")) { + closeMySQLCatalogResource(); + } else if (catalogUri != null && catalogUri.contains("postgresql")) { + closePostgreSQLCatalogResource(); + } else if (catalogType.equalsIgnoreCase(IcebergCatalogBackend.HIVE.name())) { + // TODO(yuqi) add close for other catalog types such Hive catalog + } + } + + private void closeMySQLCatalogResource() { + try { + // Close thread AbandonedConnectionCleanupThread if we are using `com.mysql.cj.jdbc.Driver`, + // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thead maybe not this one. + Class.forName("com.mysql.cj.jdbc.AbandonedConnectionCleanupThread") + .getMethod("uncheckedShutdown") + .invoke(null); + LOG.info("AbandonedConnectionCleanupThread has been shutdown..."); + + // Unload the MySQL driver, only Unload the driver if it is loaded by + // IsolatedClassLoader. + closeDriverLoadedByIsolatedClassLoader(catalogUri); + } catch (Exception e) { + LOG.warn("Failed to shutdown AbandonedConnectionCleanupThread or deregister MySQL driver", e); + } + } + + private void closeDriverLoadedByIsolatedClassLoader(String uri) { + try { + Driver driver = DriverManager.getDriver(uri); + if (driver.getClass().getClassLoader().getClass() + == IsolatedClassLoader.CUSTOM_CLASS_LOADER_CLASS) { + DriverManager.deregisterDriver(driver); + LOG.info("Driver {} has been deregistered...", driver); + } + } catch (Exception e) { + LOG.warn("Failed to deregister driver", e); + } + } + + private void closePostgreSQLCatalogResource() { + closeDriverLoadedByIsolatedClassLoader(catalogUri); } } diff --git a/core/src/main/java/com/datastrato/gravitino/utils/IsolatedClassLoader.java b/core/src/main/java/com/datastrato/gravitino/utils/IsolatedClassLoader.java index 05f47a9f66e..1bc541e4d07 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/IsolatedClassLoader.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/IsolatedClassLoader.java @@ -26,6 +26,9 @@ */ public class IsolatedClassLoader implements Closeable { + public static final Class CUSTOM_CLASS_LOADER_CLASS = + IsolatedClassLoader.CustomURLClassLoader.class; + private static final Logger LOG = LoggerFactory.getLogger(IsolatedClassLoader.class); private final List execJars; @@ -140,6 +143,55 @@ public void close() { } } + class CustomURLClassLoader extends URLClassLoader { + private final ClassLoader baseClassLoader; + + public CustomURLClassLoader(URL[] urls, ClassLoader parent, ClassLoader baseClassLoader) { + super(urls, parent); + this.baseClassLoader = baseClassLoader; + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + Class clazz = findLoadedClass(name); + + try { + return clazz == null ? doLoadClass(name, resolve) : clazz; + } catch (Exception e) { + throw new ClassNotFoundException("Class no found " + name, e); + } + } + + private Class doLoadClass(String name, boolean resolve) throws Exception { + if (isBarrierClass(name)) { + // For barrier classes, copy the class bytecode and reconstruct the class. + if (LOG.isDebugEnabled()) { + LOG.debug("barrier class: {}", name); + } + byte[] bytes = loadClassBytes(name); + return defineClass(name, bytes, 0, bytes.length); + + } else if (!isSharedClass(name)) { + if (LOG.isDebugEnabled()) { + LOG.debug("isolated class: {} - {}", name, getResources(classToPath(name))); + } + return super.loadClass(name, resolve); + + } else { + // For shared classes, delegate to base classloader. + if (LOG.isDebugEnabled()) { + LOG.debug("shared class: {}", name); + } + try { + return baseClassLoader.loadClass(name); + } catch (ClassNotFoundException e) { + // Fall through. + return super.loadClass(name, resolve); + } + } + } + } + private synchronized URLClassLoader classLoader() throws Exception { if (classLoader != null) { return classLoader; @@ -147,48 +199,7 @@ private synchronized URLClassLoader classLoader() throws Exception { ClassLoader parent = Thread.currentThread().getContextClassLoader(); this.classLoader = - new URLClassLoader(execJars.toArray(new URL[0]), parent) { - @Override - protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - Class clazz = findLoadedClass(name); - - try { - return clazz == null ? doLoadClass(name, resolve) : clazz; - } catch (Exception e) { - throw new ClassNotFoundException("Class no found " + name, e); - } - } - - private Class doLoadClass(String name, boolean resolve) throws Exception { - if (isBarrierClass(name)) { - // For barrier classes, copy the class bytecode and reconstruct the class. - if (LOG.isDebugEnabled()) { - LOG.debug("barrier class: {}", name); - } - byte[] bytes = loadClassBytes(name); - return defineClass(name, bytes, 0, bytes.length); - - } else if (!isSharedClass(name)) { - if (LOG.isDebugEnabled()) { - LOG.debug("isolated class: {} - {}", name, getResources(classToPath(name))); - } - return super.loadClass(name, resolve); - - } else { - // For shared classes, delegate to base classloader. - if (LOG.isDebugEnabled()) { - LOG.debug("shared class: {}", name); - } - try { - return baseClassLoader.loadClass(name); - } catch (ClassNotFoundException e) { - // Fall through. - return super.loadClass(name, resolve); - } - } - } - }; - + new CustomURLClassLoader(execJars.toArray(new URL[0]), parent, baseClassLoader); return classLoader; } From d349cf7745e18cfa2dde491d89831ce94831bcd5 Mon Sep 17 00:00:00 2001 From: FANNG Date: Fri, 12 Apr 2024 16:43:14 +0800 Subject: [PATCH 009/106] [#2766] feat(core): Gravitino event listener code skeleton (#2787) ### What changes were proposed in this pull request? gravitino event listener code skeleton and supports some table operations. ### Why are the changes needed? Fix: #2766 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? add UT and test in local envirment --- .../datastrato/gravitino/GravitinoEnv.java | 28 +- .../gravitino/catalog/TableDispatcher.java | 16 ++ .../catalog/TableEventDispatcher.java | 127 +++++++++ .../catalog/TableOperationDispatcher.java | 3 +- .../listener/AsyncQueueListener.java | 143 ++++++++++ .../gravitino/listener/EventBus.java | 56 ++++ .../listener/EventListenerConfig.java | 42 +++ .../listener/EventListenerManager.java | 153 +++++++++++ .../listener/EventListenerPluginWrapper.java | 68 +++++ .../listener/api/EventListenerPlugin.java | 106 ++++++++ .../listener/api/event/CreateTableEvent.java | 57 ++++ .../api/event/CreateTableFailureEvent.java | 55 ++++ .../listener/api/event/DropTableEvent.java | 47 ++++ .../api/event/DropTableFailureEvent.java | 36 +++ .../gravitino/listener/api/event/Event.java | 52 ++++ .../listener/api/event/FailureEvent.java | 47 ++++ .../listener/api/event/TableEvent.java | 33 +++ .../listener/api/event/TableFailureEvent.java | 38 +++ .../listener/api/info/TableInfo.java | 130 +++++++++ .../gravitino/utils/PrincipalUtils.java | 4 + .../listener/TestEventListenerManager.java | 252 ++++++++++++++++++ .../gravitino/server/GravitinoServer.java | 6 +- .../server/web/rest/PartitionOperations.java | 6 +- .../server/web/rest/TableOperations.java | 6 +- .../web/rest/TestPartitionOperations.java | 3 +- .../server/web/rest/TestTableOperations.java | 3 +- 26 files changed, 1497 insertions(+), 20 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/EventBus.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 59c25878aef..c357c9bcc0c 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -9,8 +9,12 @@ import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; +import com.datastrato.gravitino.catalog.TableDispatcher; +import com.datastrato.gravitino.catalog.TableEventDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.catalog.TopicOperationDispatcher; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.EventListenerManager; import com.datastrato.gravitino.lock.LockManager; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; @@ -37,7 +41,7 @@ public class GravitinoEnv { private SchemaOperationDispatcher schemaOperationDispatcher; - private TableOperationDispatcher tableOperationDispatcher; + private TableDispatcher tableDispatcher; private FilesetOperationDispatcher filesetOperationDispatcher; @@ -54,6 +58,7 @@ public class GravitinoEnv { private MetricsSystem metricsSystem; private LockManager lockManager; + private EventListenerManager eventListenerManager; private GravitinoEnv() {} @@ -112,6 +117,11 @@ public void initialize(Config config) { // create and initialize a random id generator this.idGenerator = new RandomIdGenerator(); + this.eventListenerManager = new EventListenerManager(); + eventListenerManager.init( + config.getConfigsWithPrefix(EventListenerManager.GRAVITINO_EVENT_LISTENER_PREFIX)); + EventBus eventBus = eventListenerManager.createEventBus(); + // Create and initialize metalake related modules this.metalakeManager = new MetalakeManager(entityStore, idGenerator); @@ -119,8 +129,9 @@ public void initialize(Config config) { this.catalogManager = new CatalogManager(config, entityStore, idGenerator); this.schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); - this.tableOperationDispatcher = + TableOperationDispatcher tableOperationDispatcher = new TableOperationDispatcher(catalogManager, entityStore, idGenerator); + this.tableDispatcher = new TableEventDispatcher(eventBus, tableOperationDispatcher); this.filesetOperationDispatcher = new FilesetOperationDispatcher(catalogManager, entityStore, idGenerator); this.topicOperationDispatcher = @@ -181,12 +192,12 @@ public SchemaOperationDispatcher schemaOperationDispatcher() { } /** - * Get the TableOperationDispatcher associated with the Gravitino environment. + * Get the TableDispatcher associated with the Gravitino environment. * - * @return The TableOperationDispatcher instance. + * @return The TableDispatcher instance. */ - public TableOperationDispatcher tableOperationDispatcher() { - return tableOperationDispatcher; + public TableDispatcher tableDispatcher() { + return tableDispatcher; } /** @@ -250,6 +261,7 @@ public AccessControlManager accessControlManager() { public void start() { auxServiceManager.serviceStart(); metricsSystem.start(); + eventListenerManager.start(); } /** Shutdown the Gravitino environment. */ @@ -280,6 +292,10 @@ public void shutdown() { metricsSystem.close(); } + if (eventListenerManager != null) { + eventListenerManager.stop(); + } + LOG.info("Gravitino Environment is shut down."); } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java new file mode 100644 index 00000000000..7b54ccd5794 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.rel.TableCatalog; + +/** + * {@code TableDispatcher} interface acts as a specialization of the {@link TableCatalog} interface. + * This interface is designed to potentially add custom behaviors or operations related to + * dispatching or handling table-related events or actions that are not covered by the standard + * {@code TableCatalog} operations. + */ +public interface TableDispatcher extends TableCatalog {} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java new file mode 100644 index 00000000000..987b2687b96 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java @@ -0,0 +1,127 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.CreateTableEvent; +import com.datastrato.gravitino.listener.api.event.CreateTableFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropTableEvent; +import com.datastrato.gravitino.listener.api.event.DropTableFailureEvent; +import com.datastrato.gravitino.listener.api.info.TableInfo; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code TableEventDispatcher} is a decorator for {@link TableDispatcher} that not only delegates + * table operations to the underlying catalog dispatcher but also dispatches corresponding events to + * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or + * monitoring of table operations. + */ +public class TableEventDispatcher implements TableDispatcher { + private final EventBus eventBus; + private final TableDispatcher dispatcher; + + /** + * Constructs a TableEventDispatcher with a specified EventBus and TableCatalog. + * + * @param eventBus The EventBus to which events will be dispatched. + * @param dispatcher The underlying {@link TableOperationDispatcher} that will perform the actual + * table operations. + */ + public TableEventDispatcher(EventBus eventBus, TableDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { + return dispatcher.listTables(namespace); + } + + @Override + public Table loadTable(NameIdentifier ident) throws NoSuchTableException { + return dispatcher.loadTable(ident); + } + + @Override + public Table createTable( + NameIdentifier ident, + Column[] columns, + String comment, + Map properties, + Transform[] partitions, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes) + throws NoSuchSchemaException, TableAlreadyExistsException { + try { + Table table = + dispatcher.createTable( + ident, columns, comment, properties, partitions, distribution, sortOrders, indexes); + eventBus.dispatchEvent( + new CreateTableEvent(PrincipalUtils.getCurrentUserName(), ident, new TableInfo(table))); + return table; + } catch (Exception e) { + TableInfo createTableRequest = + new TableInfo( + ident.name(), + columns, + comment, + properties, + partitions, + distribution, + sortOrders, + indexes, + null); + eventBus.dispatchEvent( + new CreateTableFailureEvent( + PrincipalUtils.getCurrentUserName(), ident, e, createTableRequest)); + throw e; + } + } + + @Override + public Table alterTable(NameIdentifier ident, TableChange... changes) + throws NoSuchTableException, IllegalArgumentException { + return dispatcher.alterTable(ident, changes); + } + + @Override + public boolean dropTable(NameIdentifier ident) { + try { + boolean isExists = dispatcher.dropTable(ident); + eventBus.dispatchEvent( + new DropTableEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropTableFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public boolean purgeTable(NameIdentifier ident) { + return dispatcher.purgeTable(ident); + } + + @Override + public boolean tableExists(NameIdentifier ident) { + return dispatcher.tableExists(ident); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index 2947ca3180a..1d58fb0d9ab 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -20,7 +20,6 @@ import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; @@ -36,7 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TableOperationDispatcher extends OperationDispatcher implements TableCatalog { +public class TableOperationDispatcher extends OperationDispatcher implements TableDispatcher { private static final Logger LOG = LoggerFactory.getLogger(TableOperationDispatcher.class); diff --git a/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java b/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java new file mode 100644 index 00000000000..d9beb624351 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java @@ -0,0 +1,143 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.listener.api.event.Event; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * AsyncQueueListener acts as event listener, and internally buffer event to a queue, start a + * dispatcher thread to dispatch event to the real listeners. For default AsyncQueueListener it may + * contain multi listeners share with one queue and dispatcher thread. For other + * AsyncQueueDispatchers, contain only one listener. + */ +public class AsyncQueueListener implements EventListenerPlugin { + private static final Logger LOG = LoggerFactory.getLogger(AsyncQueueListener.class); + private static final String NAME_PREFIX = "async-queue-listener-"; + + private final List eventListeners; + private final BlockingQueue queue; + private final Thread asyncProcessor; + private final int dispatcherJoinSeconds; + private final AtomicBoolean stopped = new AtomicBoolean(false); + private final AtomicLong dropEventCounters = new AtomicLong(0); + private final AtomicLong lastDropEventCounters = new AtomicLong(0); + private Instant lastRecordDropEventTime; + private final String asyncQueueListenerName; + + public AsyncQueueListener( + List listeners, + String name, + int queueCapacity, + int dispatcherJoinSeconds) { + this.asyncQueueListenerName = NAME_PREFIX + name; + this.eventListeners = listeners; + this.queue = new LinkedBlockingQueue<>(queueCapacity); + this.asyncProcessor = new Thread(() -> processEvents()); + this.dispatcherJoinSeconds = dispatcherJoinSeconds; + asyncProcessor.setDaemon(true); + asyncProcessor.setName(asyncQueueListenerName); + } + + @Override + public void onPostEvent(Event event) { + if (stopped.get()) { + LOG.warn( + "{} drop event: {}, since AsyncQueueListener is stopped", + asyncQueueListenerName, + event.getClass().getSimpleName()); + return; + } + + if (queue.offer(event)) { + return; + } + + logDropEventsIfNecessary(); + } + + @Override + public void init(Map properties) { + throw new RuntimeException( + "Should not reach here, the event listener has already been initialized."); + } + + @Override + public void start() { + eventListeners.forEach(listenerPlugin -> listenerPlugin.start()); + asyncProcessor.start(); + } + + @Override + public void stop() { + Preconditions.checkState(!stopped.get(), asyncQueueListenerName + " had already stopped"); + stopped.compareAndSet(false, true); + asyncProcessor.interrupt(); + try { + asyncProcessor.join(dispatcherJoinSeconds * 1000L); + } catch (InterruptedException e) { + LOG.warn("{} interrupt async processor failed.", asyncQueueListenerName, e); + } + eventListeners.forEach(listenerPlugin -> listenerPlugin.stop()); + } + + @VisibleForTesting + List getEventListeners() { + return this.eventListeners; + } + + private void processEvents() { + while (!Thread.currentThread().isInterrupted()) { + try { + Event event = queue.take(); + this.eventListeners.forEach(listener -> listener.onPostEvent(event)); + } catch (InterruptedException e) { + LOG.warn("{} event dispatcher thread is interrupted.", asyncQueueListenerName); + break; + } catch (Exception e) { + LOG.warn("{} throw a exception while processing event", asyncQueueListenerName, e); + } + } + + if (!queue.isEmpty()) { + LOG.warn( + "{} drop {} events since dispatch thread is interrupted", + asyncQueueListenerName, + queue.size()); + } + } + + private void logDropEventsIfNecessary() { + long currentDropEvents = dropEventCounters.incrementAndGet(); + long lastDropEvents = lastDropEventCounters.get(); + // dropEvents may less than zero in such conditions: + // 1. Thread A increment dropEventCounters + // 2. Thread B increment dropEventCounters and update lastDropEventCounters + // 3. Thread A get lastDropEventCounters + long dropEvents = currentDropEvents - lastDropEvents; + if (dropEvents > 0 && Instant.now().isAfter(lastRecordDropEventTime.plusSeconds(60))) { + if (lastDropEventCounters.compareAndSet(lastDropEvents, currentDropEvents)) { + LOG.warn( + "{} drop {} events since {}", + asyncQueueListenerName, + dropEvents, + lastRecordDropEventTime); + lastRecordDropEventTime = Instant.now(); + } + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java b/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java new file mode 100644 index 00000000000..0c186f4d003 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.listener.api.event.Event; +import com.google.common.annotations.VisibleForTesting; +import java.util.List; + +/** + * The {@code EventBus} class serves as a mechanism to dispatch events to registered listeners. It + * supports both synchronous and asynchronous listeners by categorizing them into two distinct types + * within its internal management. + */ +public class EventBus { + // Holds instances of EventListenerPlugin. These instances can either be + // EventListenerPluginWrapper, + // which are meant for synchronous event listening, or AsyncQueueListener, designed for + // asynchronous event processing. + private final List postEventListeners; + + /** + * Constructs an EventBus with a predefined list of event listeners. + * + * @param postEventListeners A list of {@link EventListenerPlugin} instances that are to be + * registered with this EventBus for event dispatch. + */ + public EventBus(List postEventListeners) { + this.postEventListeners = postEventListeners; + } + + /** + * Dispatches an event to all registered listeners. Each listener processes the event based on its + * implementation, which could be either synchronous or asynchronous. + * + * @param event The event to be dispatched to all registered listeners. + */ + public void dispatchEvent(Event event) { + postEventListeners.forEach(postEventListener -> postEventListener.onPostEvent(event)); + } + + /** + * Retrieves the list of registered post-event listeners. This method is primarily intended for + * testing purposes to verify the correct registration and functioning of event listeners. + * + * @return A list of {@link EventListenerPlugin} instances currently registered with this + * EventBus. + */ + @VisibleForTesting + List getPostEventListeners() { + return postEventListeners; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java new file mode 100644 index 00000000000..f8032031758 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; +import com.datastrato.gravitino.config.ConfigEntry; +import java.util.Map; + +class EventListenerConfig extends Config { + static final ConfigEntry LISTENER_NAMES = + new ConfigBuilder(EventListenerManager.GRAVITINO_EVENT_LISTENER_NAMES) + .doc("Gravitino event listener names, comma is utilized to separate multiple names") + .version(ConfigConstants.VERSION_0_5_0) + .stringConf() + .createWithDefault(""); + + static final ConfigEntry QUEUE_CAPACITY = + new ConfigBuilder(EventListenerManager.GRAVITINO_EVENT_LISTENER_QUEUE_CAPACITY) + .doc("Gravitino event listener async queue capacity") + .version(ConfigConstants.VERSION_0_5_0) + .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) + .createWithDefault(3000); + + static final ConfigEntry DISPATCHER_JOIN_SECONDS = + new ConfigBuilder(EventListenerManager.GRAVITINO_EVENT_LISTENER_DISPATCHER_JOIN_SECONDS) + .doc("Gravitino async event dispatcher join seconds") + .version(ConfigConstants.VERSION_0_5_0) + .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) + .createWithDefault(3); + + EventListenerConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java new file mode 100644 index 00000000000..c26e9abcdbd --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java @@ -0,0 +1,153 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.utils.MapUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * EventListenerManager loads listeners according to the configurations, and assemble the listeners + * with following rules: + * + *

Wrap all listener with EventListenerWrapper to do some common process, like exception handing, + * record metrics. + * + *

For async listeners with the shared dispatcher, will create a default AsyncQueueListener to + * assemble the corresponding EventListenerWrappers. + * + *

For async listeners with the isolated dispatcher, will create a separate AsyncQueueListener + * for each EventListenerWrapper. + */ +public class EventListenerManager { + private static final Logger LOG = LoggerFactory.getLogger(EventListenerManager.class); + public static final String GRAVITINO_EVENT_LISTENER_PREFIX = "gravitino.eventListener."; + static final String GRAVITINO_EVENT_LISTENER_NAMES = "names"; + @VisibleForTesting static final String GRAVITINO_EVENT_LISTENER_CLASS = "class"; + static final String GRAVITINO_EVENT_LISTENER_QUEUE_CAPACITY = "queueCapacity"; + static final String GRAVITINO_EVENT_LISTENER_DISPATCHER_JOIN_SECONDS = "dispatcherJoinSeconds"; + private static final Splitter splitter = Splitter.on(","); + private static final Joiner DOT = Joiner.on("."); + + private int queueCapacity; + private int dispatcherJoinSeconds; + private List eventListeners; + + public void init(Map properties) { + EventListenerConfig config = new EventListenerConfig(properties); + this.queueCapacity = config.get(EventListenerConfig.QUEUE_CAPACITY); + this.dispatcherJoinSeconds = config.get(EventListenerConfig.DISPATCHER_JOIN_SECONDS); + + String eventListenerNames = config.get(EventListenerConfig.LISTENER_NAMES); + Map userEventListenerPlugins = + splitter + .omitEmptyStrings() + .trimResults() + .splitToStream(eventListenerNames) + .collect( + Collectors.toMap( + listenerName -> listenerName, + listenerName -> + loadUserEventListenerPlugin( + listenerName, + MapUtils.getPrefixMap(properties, DOT.join(listenerName, ""))), + (existingValue, newValue) -> { + throw new IllegalStateException( + "Duplicate event listener name detected: " + existingValue); + })); + this.eventListeners = assembleEventListeners(userEventListenerPlugins); + } + + public void start() { + eventListeners.stream().forEach(listener -> listener.start()); + } + + public void stop() { + eventListeners.stream().forEach(listener -> listener.stop()); + } + + public EventBus createEventBus() { + return new EventBus(eventListeners); + } + + private List assembleEventListeners( + Map userEventListeners) { + List sharedQueueListeners = new ArrayList<>(); + + List listeners = + userEventListeners.entrySet().stream() + .map( + entrySet -> { + String listenerName = entrySet.getKey(); + EventListenerPlugin listener = entrySet.getValue(); + switch (listener.mode()) { + case SYNC: + return new EventListenerPluginWrapper(listenerName, listener); + case ASYNC_ISOLATED: + return new AsyncQueueListener( + ImmutableList.of(new EventListenerPluginWrapper(listenerName, listener)), + listenerName, + queueCapacity, + dispatcherJoinSeconds); + case ASYNC_SHARED: + sharedQueueListeners.add( + new EventListenerPluginWrapper(listenerName, listener)); + return null; + default: + throw new RuntimeException("Unexpected listener mode:" + listener.mode()); + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (!sharedQueueListeners.isEmpty()) { + listeners.add( + new AsyncQueueListener( + sharedQueueListeners, "default", queueCapacity, dispatcherJoinSeconds)); + } + return listeners; + } + + private EventListenerPlugin loadUserEventListenerPlugin( + String listenerName, Map config) { + LOG.info("EventListener:{}, config:{}.", listenerName, config); + String className = config.get(GRAVITINO_EVENT_LISTENER_CLASS); + Preconditions.checkArgument( + StringUtils.isNotBlank(className), + String.format( + "EventListener:%s, %s%s.%s is not set in configuration.", + listenerName, + GRAVITINO_EVENT_LISTENER_PREFIX, + listenerName, + GRAVITINO_EVENT_LISTENER_CLASS)); + + try { + EventListenerPlugin listenerPlugin = + (EventListenerPlugin) Class.forName(className).getDeclaredConstructor().newInstance(); + listenerPlugin.init(config); + return listenerPlugin; + } catch (Exception e) { + LOG.error( + "Failed to create and initialize event listener {}, className: {}.", + listenerName, + className, + e); + throw new RuntimeException(e); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java new file mode 100644 index 00000000000..2dda84a84b1 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.listener.api.event.Event; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A wrapper for user provided event listener, could contain common logic like exception handling, + * recording metrics, recording slow event process. + */ +public class EventListenerPluginWrapper implements EventListenerPlugin { + private static final Logger LOG = LoggerFactory.getLogger(EventListenerPluginWrapper.class); + private String listenerName; + private EventListenerPlugin userEventListener; + + public EventListenerPluginWrapper(String listenerName, EventListenerPlugin userEventListener) { + this.listenerName = listenerName; + this.userEventListener = userEventListener; + } + + @Override + public void init(Map properties) { + throw new RuntimeException( + "Should not reach here, the event listener has already been initialized."); + } + + @Override + public void start() { + userEventListener.start(); + LOG.info("Start event listener {}.", listenerName); + } + + @Override + public void stop() { + try { + userEventListener.stop(); + LOG.info("Stop event listener {}.", listenerName); + } catch (Exception e) { + LOG.warn("Failed to stop event listener {}.", listenerName, e); + } + } + + @Override + public void onPostEvent(Event event) { + try { + userEventListener.onPostEvent(event); + } catch (Exception e) { + LOG.warn( + "Event listener {} process event {} failed,", + listenerName, + event.getClass().getSimpleName(), + e); + } + } + + @VisibleForTesting + EventListenerPlugin getUserEventListener() { + return userEventListener; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java b/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java new file mode 100644 index 00000000000..24bebbac30f --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api; + +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.event.Event; +import java.util.Map; + +/** + * Defines an interface for event listeners that manage the lifecycle and state of a plugin, + * including its initialization, starting, and stopping processes, as well as the handling of events + * occurring after various operations have been executed. + * + *

This interface is intended for developers who implement plugins within a system, providing a + * structured approach to managing plugin operations and event processing. + */ +@DeveloperApi +public interface EventListenerPlugin { + /** + * Defines the operational modes for event processing within an event listener, catering to both + * synchronous and asynchronous processing strategies. Each mode determines how events are + * handled, balancing between immediacy and resource efficiency. + * + *

    + *
  • {@code SYNC} - Events are processed synchronously, immediately after the associated + * operation completes. While this approach ensures prompt handling of events, it may block + * the main process if the event listener requires significant time to process an event. + *
  • {@code ASYNC_ISOLATED} - Events are handled asynchronously with each listener possessing + * its own distinct event-processing queue. This mode allows for customized and isolated + * event processing but may increase resource consumption due to the necessity of a + * dedicated event dispatcher for each listener. + *
  • {@code ASYNC_SHARED} - In this mode, event listeners share a common event-processing + * queue, processing events asynchronously. This approach enhances resource efficiency by + * utilizing a shared dispatcher for handling events across multiple listeners. + *
+ */ + enum Mode { + SYNC, + ASYNC_ISOLATED, + ASYNC_SHARED + } + + /** + * Initializes the plugin with the given set of properties. This phase may involve setting up + * necessary resources for the plugin's functionality. + * + *

Failure during this phase will prevent the server from starting, highlighting a critical + * setup issue with the plugin. + * + * @param properties A map of properties used for initializing the plugin. + * @throws RuntimeException Indicates a critical failure in plugin setup, preventing server + * startup. + */ + void init(Map properties) throws RuntimeException; + + /** + * Starts the plugin, transitioning it to a ready state. This method is invoked after successful + * initialization. + * + *

A failure to start indicates an inability for the plugin to enter its operational state, + * which will prevent the server from starting. + * + * @throws RuntimeException Indicates a failure to start the plugin, blocking the server's launch. + */ + void start() throws RuntimeException; + + /** + * Stops the plugin, releasing any resources that were allocated during its operation. This method + * aims to ensure a clean termination, mitigating potential resource leaks or incomplete + * shutdowns. + * + *

While the server's operation is unaffected by exceptions thrown during this process, failing + * to properly stop may lead to resource management issues. + * + * @throws RuntimeException Indicates issues during the stopping process, potentially leading to + * resource leaks. + */ + void stop() throws RuntimeException; + + /** + * Handles events generated after the completion of an operation. Implementers are responsible for + * processing these events, which may involve additional logic to respond to the operation + * outcomes. + * + *

This method provides a hook for post-operation event processing, allowing plugins to react + * or adapt based on the event details. + * + * @param event The event to be processed. + * @throws RuntimeException Indicates issues encountered during event processing. + */ + void onPostEvent(Event event) throws RuntimeException; + + /** + * Specifies the default operational mode for event processing by the plugin. The default + * implementation is synchronous, but implementers can override this to utilize asynchronous + * processing modes. + * + * @return The operational {@link Mode} of the plugin for event processing. + */ + default Mode mode() { + return Mode.SYNC; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java new file mode 100644 index 00000000000..b68dee9db0d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TableInfo; + +/** + * Represents an event triggered upon the successful creation of a table. This class extends {@link + * TableEvent} to provide detailed information specifically related to the table's creation. This + * includes the final table information as it is returned to the user once the creation process has + * successfully completed. + * + *

Such an event is instrumental for a variety of purposes including, but not limited to, + * auditing activities associated with table creation, monitoring the creation of tables within a + * system, and acquiring insights into the final configuration and state of a newly created table. + */ +@DeveloperApi +public final class CreateTableEvent extends TableEvent { + private final TableInfo createdTableInfo; + + /** + * Constructs an instance of {@code CreateTableEvent}, capturing essential details about the + * successful creation of a table. + * + *

This constructor documents the successful culmination of the table creation process by + * encapsulating the final state of the table. This includes any adjustments or resolutions made + * to the table's properties or configuration during the creation process. + * + * @param user The username of the individual who initiated the table creation. This information + * is vital for tracking the source of changes and for audit trails. + * @param identifier The unique identifier of the table that was created, providing a precise + * reference to the affected table. + * @param createdTableInfo The final state of the table post-creation. This information is + * reflective of the table's configuration, including any default settings or properties + * applied during the creation process. + */ + public CreateTableEvent(String user, NameIdentifier identifier, TableInfo createdTableInfo) { + super(user, identifier); + this.createdTableInfo = createdTableInfo; + } + + /** + * Retrieves the final state and configuration information of the table as it was returned to the + * user after successful creation. + * + * @return A {@link TableInfo} instance encapsulating the comprehensive details of the newly + * created table, highlighting its configuration and any default settings applied. + */ + public TableInfo createdTableInfo() { + return createdTableInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java new file mode 100644 index 00000000000..6370378b3f6 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TableInfo; +import com.datastrato.gravitino.rel.Table; + +/** + * Represents an event that is generated when an attempt to create a table fails due to an + * exception. This class extends {@link TableFailureEvent} to specifically address failure scenarios + * encountered during the table creation process. It encapsulates both the exception that caused the + * failure and the original request details for the table creation, providing comprehensive context + * for the failure. + */ +@DeveloperApi +public final class CreateTableFailureEvent extends TableFailureEvent { + private final TableInfo createTableRequest; + + /** + * Constructs a {@code CreateTableFailureEvent} instance, capturing detailed information about the + * failed table creation attempt. + * + * @param user The user who initiated the table creation operation. This information is essential + * for auditing and diagnosing the cause of the failure. + * @param identifier The identifier of the table that was attempted to be created. This helps in + * pinpointing the specific table related to the failure. + * @param exception The exception that was thrown during the table creation operation, providing + * insight into what went wrong. + * @param createTableRequest The original request information used to attempt to create the table. + * This includes details such as the intended table schema, properties, and other + * configuration options that were specified. + */ + public CreateTableFailureEvent( + String user, NameIdentifier identifier, Exception exception, TableInfo createTableRequest) { + super(user, identifier, exception); + this.createTableRequest = createTableRequest; + } + + /** + * Retrieves the original request information for the attempted table creation. This information + * can be valuable for understanding the configuration and expectations that led to the failure, + * facilitating analysis and potential corrective actions. + * + * @return The {@link Table} instance representing the request information for the failed table + * creation attempt. + */ + public TableInfo createTableRequest() { + return createTableRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java new file mode 100644 index 00000000000..a75939eaf66 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated after a table is successfully dropped from the database. + * This class extends {@link TableEvent} to capture specific details related to the dropping of a + * table, including the status of the table's existence at the time of the operation and identifying + * information about the table and the user who initiated the drop operation. + */ +@DeveloperApi +public final class DropTableEvent extends TableEvent { + private final boolean isExists; + + /** + * Constructs a new {@code DropTableEvent} instance, encapsulating information about the outcome + * of a table drop operation. + * + * @param user The user who initiated the drop table operation. This information is important for + * auditing purposes and understanding who is responsible for the change. + * @param identifier The identifier of the table that was attempted to be dropped. This provides a + * clear reference to the specific table affected by the operation. + * @param isExists A boolean flag indicating whether the table existed at the time of the drop + * operation. This can be useful to understand the state of the database prior to the + * operation. + */ + public DropTableEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the existence status of the table at the time of the drop operation. + * + * @return A boolean value indicating whether the table existed. {@code true} if the table + * existed, otherwise {@code false}. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java new file mode 100644 index 00000000000..78076fe961d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to drop a table from the database fails due + * to an exception. This class extends {@link TableFailureEvent} to provide specific context related + * to table drop failures, encapsulating details about the user who initiated the operation, the + * identifier of the table that was attempted to be dropped, and the exception that led to the + * failure. This event can be used for auditing purposes and to facilitate error handling and + * diagnostic processes. + */ +@DeveloperApi +public final class DropTableFailureEvent extends TableFailureEvent { + /** + * Constructs a new {@code DropTableFailureEvent} instance, capturing detailed information about + * the failed attempt to drop a table. + * + * @param user The user who initiated the drop table operation. This information is crucial for + * understanding the context of the operation and for auditing who is responsible for the + * attempted change. + * @param identifier The identifier of the table that the operation attempted to drop. This + * provides a clear reference to the specific table involved in the failure. + * @param exception The exception that was thrown during the drop table operation, offering + * insights into what went wrong and why the operation failed. + */ + public DropTableFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java new file mode 100644 index 00000000000..484120a8bca --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * The abstract base class for all events. It encapsulates common information such as the user who + * generated the event and the identifier for the resource associated with the event. Subclasses + * should provide specific details related to their individual event types. + */ +@DeveloperApi +public abstract class Event { + private final String user; + private final NameIdentifier identifier; + + /** + * Constructs an Event instance with the specified user and resource identifier details. + * + * @param user The user associated with this event. It provides context about who triggered the + * event. + * @param identifier The resource identifier associated with this event. This may refer to various + * types of resources such as a metalake, catalog, schema, or table, etc. + */ + protected Event(String user, NameIdentifier identifier) { + this.user = user; + this.identifier = identifier; + } + + /** + * Retrieves the user associated with this event. + * + * @return A string representing the user associated with this event. + */ + public String user() { + return user; + } + + /** + * Retrieves the resource identifier associated with this event. + * + * @return A NameIdentifier object that represents the resource, like a metalake, catalog, schema, + * table, etc., associated with the event. + */ + public NameIdentifier identifier() { + return identifier; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java new file mode 100644 index 00000000000..ca0c263c20a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events that indicate a failure in an operation. This class + * extends {@link Event} to encapsulate common information related to failures, such as the user who + * performed the operation, the resource identifier, and the exception that was thrown during the + * operation. This class serves as a foundation for more specific failure event types that might + * include additional context or details about the failure. + */ +@DeveloperApi +public abstract class FailureEvent extends Event { + private final Exception exception; + + /** + * Constructs a new {@code FailureEvent} instance with the specified user, resource identifier, + * and the exception that was thrown. + * + * @param user The user associated with the operation that resulted in a failure. This information + * is important for auditing and understanding the context of the failure. + * @param identifier The identifier of the resource involved in the operation that failed. This + * provides a clear reference to what was being acted upon when the exception occurred. + * @param exception The exception that was thrown during the operation. This is the primary piece + * of information indicating what went wrong. + */ + protected FailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier); + this.exception = exception; + } + + /** + * Retrieves the exception that was thrown during the operation, indicating the reason for the + * failure. + * + * @return The exception thrown by the operation. + */ + public Exception exception() { + return exception; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java new file mode 100644 index 00000000000..238ad0eea4c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events related to table operations. This class extends + * {@link Event} to provide a more specific context involving operations on tables, such as + * creation, deletion, or modification. It captures essential information including the user + * performing the operation and the identifier of the table being operated on. + * + *

Concrete implementations of this class should provide additional details pertinent to the + * specific type of table operation being represented. + */ +@DeveloperApi +public abstract class TableEvent extends Event { + /** + * Constructs a new {@code TableEvent} with the specified user and table identifier. + * + * @param user The user responsible for triggering the table operation. This information is + * crucial for auditing and tracking purposes. + * @param identifier The identifier of the table involved in the operation. This encapsulates + * details such as the metalake, catalog, schema, and table name. + */ + protected TableEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java new file mode 100644 index 00000000000..8b78df964c4 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a table operation fails due to an + * exception. This class extends {@link FailureEvent} to provide a more specific context related to + * table operations, encapsulating details about the user who initiated the operation, the + * identifier of the table involved, and the exception that led to the failure. + * + *

Implementations of this class can be used to convey detailed information about failures in + * operations such as creating, updating, deleting, or querying tables, making it easier to diagnose + * and respond to issues. + */ +@DeveloperApi +public abstract class TableFailureEvent extends FailureEvent { + /** + * Constructs a new {@code TableFailureEvent} instance, capturing information about the failed + * table operation. + * + * @param user The user associated with the failed table operation. This information helps in + * auditing and understanding the context of the operation that resulted in a failure. + * @param identifier The identifier of the table that was involved in the failed operation. This + * provides a clear reference to the specific table that the operation was attempting to + * modify or interact with. + * @param exception The exception that was thrown during the table operation, indicating the cause + * of the failure. + */ + protected TableFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java new file mode 100644 index 00000000000..2f7329e0502 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java @@ -0,0 +1,130 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * TableInfo exposes table information for event listener, it's supposed to be read only. Most of + * the fields are shallow copied internally not deep copies for performance. + */ +@DeveloperApi +public final class TableInfo { + private final String name; + private final Column[] columns; + @Nullable private final String comment; + private final Map properties; + private final Transform[] partitions; + private final Distribution distribution; + private final SortOrder[] sortOrders; + private final Index[] indexes; + @Nullable private final Audit auditInfo; + + public TableInfo(Table table) { + this( + table.name(), + table.columns(), + table.comment(), + table.properties(), + table.partitioning(), + table.distribution(), + table.sortOrder(), + table.index(), + table.auditInfo()); + } + + public TableInfo( + String name, + Column[] columns, + String comment, + Map properties, + Transform[] partitions, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes, + Audit auditInfo) { + this.name = name; + this.columns = columns.clone(); + this.comment = comment; + if (properties == null) { + this.properties = ImmutableMap.of(); + } else { + this.properties = ImmutableMap.builder().putAll(properties).build(); + } + if (partitions == null) { + this.partitions = new Transform[0]; + } else { + this.partitions = partitions.clone(); + } + if (distribution == null) { + this.distribution = Distributions.NONE; + } else { + this.distribution = distribution; + } + if (sortOrders == null) { + this.sortOrders = new SortOrder[0]; + } else { + this.sortOrders = sortOrders.clone(); + } + if (indexes == null) { + this.indexes = Indexes.EMPTY_INDEXES; + } else { + this.indexes = indexes.clone(); + } + this.auditInfo = auditInfo; + } + + /** Audit information is null when tableInfo is generated from create table request. */ + @Nullable + public Audit auditInfo() { + return this.auditInfo; + } + + public String name() { + return name; + } + + public Column[] columns() { + return columns; + } + + public Transform[] partitioning() { + return partitions; + } + + public SortOrder[] sortOrder() { + return sortOrders; + } + + public Distribution distribution() { + return distribution; + } + + public Index[] index() { + return indexes; + } + + @Nullable + public String comment() { + return comment; + } + + public Map properties() { + return properties; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java index cbaba064ec6..d8a3255f1e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java @@ -38,4 +38,8 @@ public static Principal getCurrentPrincipal() { } return subject.getPrincipals(UserPrincipal.class).iterator().next(); } + + public static String getCurrentUserName() { + return getCurrentPrincipal().getName(); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java b/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java new file mode 100644 index 00000000000..f670a39404c --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java @@ -0,0 +1,252 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.listener.api.event.Event; +import com.google.common.collect.ImmutableSet; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.Getter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestEventListenerManager { + static class DummyEvent extends Event { + protected DummyEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } + } + + static class DummyEventListener implements EventListenerPlugin { + Map properties; + @Getter List events = new ArrayList<>(); + + @Override + public void init(Map properties) { + this.properties = properties; + } + + @Override + public void start() {} + + @Override + public void stop() {} + + @Override + public void onPostEvent(Event event) { + this.events.add(event); + } + + @Override + public Mode mode() { + return Mode.SYNC; + } + } + + static class DummyAsyncEventListener extends DummyEventListener { + List tryGetEvents() { + Instant waitTime = Instant.now().plusSeconds(20); + while (getEvents().size() == 0 && Instant.now().isBefore(waitTime)) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + break; + } + } + return getEvents(); + } + + @Override + public Mode mode() { + return Mode.ASYNC_SHARED; + } + } + + static class DummyAsyncIsolatedEventListener extends DummyAsyncEventListener { + @Override + public Mode mode() { + return Mode.ASYNC_ISOLATED; + } + } + + private static final DummyEvent DUMMY_EVENT_INSTANCE = + new DummyEvent("user", NameIdentifier.of("a", "b")); + + @Test + void testSyncListener() { + String sync1 = "sync1"; + String sync2 = "sync2"; + Map properties = createSyncEventListenerConfig(sync1, sync2); + + EventListenerManager eventListenerManager = new EventListenerManager(); + eventListenerManager.init(properties); + eventListenerManager.start(); + + EventBus eventBus = eventListenerManager.createEventBus(); + eventBus.dispatchEvent(DUMMY_EVENT_INSTANCE); + + List listeners = eventBus.getPostEventListeners(); + Assertions.assertEquals(2, listeners.size()); + Set names = + listeners.stream() + .map( + listener -> { + Assertions.assertTrue(listener instanceof EventListenerPluginWrapper); + EventListenerPluginWrapper wrapper = (EventListenerPluginWrapper) listener; + EventListenerPlugin userListener = wrapper.getUserEventListener(); + Assertions.assertTrue(userListener instanceof DummyEventListener); + checkEvents(((DummyEventListener) userListener).getEvents()); + return ((DummyEventListener) userListener).properties.get("name"); + }) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of(sync1, sync2), names); + + eventListenerManager.stop(); + } + + @Test + void testSharedAsyncListeners() { + String async1 = "async1"; + String async2 = "async2"; + Map properties = createAsyncEventListenerConfig(async1, async2); + + EventListenerManager eventListenerManager = new EventListenerManager(); + eventListenerManager.init(properties); + eventListenerManager.start(); + + EventBus eventBus = eventListenerManager.createEventBus(); + eventBus.dispatchEvent(DUMMY_EVENT_INSTANCE); + List listeners = eventBus.getPostEventListeners(); + + Assertions.assertEquals(1, listeners.size()); + Assertions.assertTrue(listeners.get(0) instanceof AsyncQueueListener); + AsyncQueueListener asyncQueueListener = (AsyncQueueListener) listeners.get(0); + List shareQueueListeners = asyncQueueListener.getEventListeners(); + Assertions.assertEquals(2, shareQueueListeners.size()); + Set sharedQueueListenerNames = + shareQueueListeners.stream() + .map( + shareQueueListener -> { + Assertions.assertTrue(shareQueueListener instanceof EventListenerPluginWrapper); + EventListenerPlugin userListener = + ((EventListenerPluginWrapper) shareQueueListener).getUserEventListener(); + Assertions.assertTrue(userListener instanceof DummyAsyncEventListener); + checkEvents(((DummyAsyncEventListener) userListener).tryGetEvents()); + return ((DummyAsyncEventListener) userListener).properties.get("name"); + }) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of(async1, async2), sharedQueueListenerNames); + + eventListenerManager.stop(); + } + + @Test + void testIsolatedAsyncListeners() { + String async1 = "async1"; + String async2 = "async2"; + Map properties = createIsolatedAsyncEventListenerConfig(async1, async2); + + EventListenerManager eventListenerManager = new EventListenerManager(); + eventListenerManager.init(properties); + eventListenerManager.start(); + + EventBus eventBus = eventListenerManager.createEventBus(); + eventBus.dispatchEvent(DUMMY_EVENT_INSTANCE); + List listeners = eventBus.getPostEventListeners(); + + Assertions.assertEquals(2, listeners.size()); + Set isolatedListenerNames = + listeners.stream() + .map( + listener -> { + Assertions.assertTrue(listener instanceof AsyncQueueListener); + AsyncQueueListener asyncQueueListener = (AsyncQueueListener) listener; + List internalListeners = + asyncQueueListener.getEventListeners(); + Assertions.assertEquals(1, internalListeners.size()); + Assertions.assertTrue( + internalListeners.get(0) instanceof EventListenerPluginWrapper); + EventListenerPlugin userListener = + ((EventListenerPluginWrapper) internalListeners.get(0)) + .getUserEventListener(); + Assertions.assertTrue(userListener instanceof DummyAsyncEventListener); + checkEvents(((DummyAsyncEventListener) userListener).tryGetEvents()); + return ((DummyAsyncEventListener) userListener).properties.get("name"); + }) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of(async1, async2), isolatedListenerNames); + + eventListenerManager.stop(); + } + + private Map createIsolatedAsyncEventListenerConfig(String async1, String async2) { + Map config = new HashMap<>(); + + config.put( + EventListenerManager.GRAVITINO_EVENT_LISTENER_NAMES, String.join(",", async1, async2)); + + config.put( + async1 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyAsyncIsolatedEventListener.class.getName()); + config.put(async1 + ".name", async1); + + config.put( + async2 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyAsyncIsolatedEventListener.class.getName()); + config.put(async2 + ".name", async2); + + return config; + } + + private Map createAsyncEventListenerConfig(String async1, String async2) { + Map config = new HashMap<>(); + + config.put( + EventListenerManager.GRAVITINO_EVENT_LISTENER_NAMES, String.join(",", async1, async2)); + + config.put( + async1 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyAsyncEventListener.class.getName()); + config.put(async1 + ".name", async1); + + config.put( + async2 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyAsyncEventListener.class.getName()); + config.put(async2 + ".name", async2); + + return config; + } + + private Map createSyncEventListenerConfig(String sync1, String sync2) { + Map config = new HashMap<>(); + + config.put(EventListenerManager.GRAVITINO_EVENT_LISTENER_NAMES, String.join(",", sync1, sync2)); + + config.put( + sync1 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyEventListener.class.getName()); + config.put(sync1 + ".name", sync1); + + config.put( + sync2 + "." + EventListenerManager.GRAVITINO_EVENT_LISTENER_CLASS, + DummyEventListener.class.getName()); + config.put(sync2 + ".name", sync2); + + return config; + } + + private void checkEvents(List events) { + Assertions.assertEquals(1, events.size()); + Assertions.assertEquals(DUMMY_EVENT_INSTANCE, events.get(0)); + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index d17e21159b6..67756319a65 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -9,7 +9,7 @@ import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; -import com.datastrato.gravitino.catalog.TableOperationDispatcher; +import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.MetricsSource; @@ -82,9 +82,7 @@ protected void configure() { bind(gravitinoEnv.schemaOperationDispatcher()) .to(SchemaOperationDispatcher.class) .ranked(1); - bind(gravitinoEnv.tableOperationDispatcher()) - .to(TableOperationDispatcher.class) - .ranked(1); + bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); bind(gravitinoEnv.filesetOperationDispatcher()) .to(FilesetOperationDispatcher.class) .ranked(1); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/PartitionOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/PartitionOperations.java index dd43901c7dc..0b8e17ad6fc 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/PartitionOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/PartitionOperations.java @@ -10,7 +10,7 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.catalog.TableOperationDispatcher; +import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.dto.rel.partitions.PartitionDTO; import com.datastrato.gravitino.dto.requests.AddPartitionsRequest; import com.datastrato.gravitino.dto.responses.DropResponse; @@ -44,11 +44,11 @@ public class PartitionOperations { private static final Logger LOG = LoggerFactory.getLogger(PartitionOperations.class); - private final TableOperationDispatcher dispatcher; + private final TableDispatcher dispatcher; @Context private HttpServletRequest httpRequest; @Inject - public PartitionOperations(TableOperationDispatcher dispatcher) { + public PartitionOperations(TableDispatcher dispatcher) { this.dispatcher = dispatcher; } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java index 2c94c7caa82..d45017b7670 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java @@ -11,7 +11,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.catalog.TableOperationDispatcher; +import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.dto.requests.TableCreateRequest; import com.datastrato.gravitino.dto.requests.TableUpdateRequest; import com.datastrato.gravitino.dto.requests.TableUpdatesRequest; @@ -46,12 +46,12 @@ public class TableOperations { private static final Logger LOG = LoggerFactory.getLogger(TableOperations.class); - private final TableOperationDispatcher dispatcher; + private final TableDispatcher dispatcher; @Context private HttpServletRequest httpRequest; @Inject - public TableOperations(TableOperationDispatcher dispatcher) { + public TableOperations(TableDispatcher dispatcher) { this.dispatcher = dispatcher; } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPartitionOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPartitionOperations.java index cf8ffceb84f..a6f7b178b70 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPartitionOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPartitionOperations.java @@ -16,6 +16,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.dto.rel.partitions.PartitionDTO; import com.datastrato.gravitino.dto.requests.AddPartitionsRequest; @@ -117,7 +118,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(dispatcher).to(TableOperationDispatcher.class).ranked(2); + bind(dispatcher).to(TableDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java index 8dcd6630a10..6d46bdf6fcb 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java @@ -17,6 +17,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.DistributionDTO; @@ -120,7 +121,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(dispatcher).to(TableOperationDispatcher.class).ranked(2); + bind(dispatcher).to(TableDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); From 06c75fcccf5ce1b331feb5cafe23f471c79454d4 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Fri, 12 Apr 2024 18:49:13 +1000 Subject: [PATCH 010/106] [Minor] improve docs (#2908) ### What changes were proposed in this pull request? Improve the language used in the documents. ### Why are the changes needed? For clarity and understanding. Fix: # N/A ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? N/A --- docs/how-to-use-gvfs.md | 122 +++++++++--------- ...manage-fileset-metadata-using-gravitino.md | 114 ++++++++-------- 2 files changed, 117 insertions(+), 119 deletions(-) diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index da0c13e1a15..cea93de5e81 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -8,57 +8,56 @@ This software is licensed under the Apache License version 2." ## Introduction `Fileset` is a concept brought in by Gravitino, which is a logical collection of files and -directories, with `fileset` you can manage the non-tabular data through Gravitino. For the -details you can see [How to manage fileset metadata using Gravitino](./manage-fileset-metadata-using-gravitino.md). +directories, with `fileset` you can manage non-tabular data through Gravitino. For +details, you can read [How to manage fileset metadata using Gravitino](./manage-fileset-metadata-using-gravitino.md). To use `Fileset` managed by Gravitino, Gravitino provides a virtual file system layer called -Gravitino Virtual File System (GVFS) that is built on top of the Hadoop Compatible File System +the Gravitino Virtual File System (GVFS) that's built on top of the Hadoop Compatible File System (HCFS) interface. GVFS is a virtual layer that manages the files and directories in the fileset through a virtual path, without needing to understand the specific storage details of the fileset. You can access -the files or folders like below: +the files or folders as shown below: ```text gvfs://fileset/${catalog_name}/${schema_name}/${fileset_name}/sub_dir/ ``` -Here `gvfs` is the scheme of the GVFS, `fileset` is the root directory of the GVFS and cannot be -modified, the `${catalog_name}/${schema_name}/${fileset_name}` is the virtual path of the fileset. -You can access the files and folders under this virtual path by concatenating the file or folder +Here `gvfs` is the scheme of the GVFS, `fileset` is the root directory of the GVFS which can't +modified, and `${catalog_name}/${schema_name}/${fileset_name}` is the virtual path of the fileset. +You can access the files and folders under this virtual path by concatenating a file or folder name to the virtual path. -The usage pattern for GVFS is exactly the same as HDFS, S3 and others, GVFS internally will manage -the path mapping and converting automatically. +The usage pattern for GVFS is the same as HDFS or S3. GVFS internally manages +the path mapping and convert automatically. ## Prerequisites -+ A Hadoop environment (Hadoop 3.1.0 has been tested) with HDFS running. GVFS is built against - Hadoop 3.1.0, it is recommended to use Hadoop 3.1.0 or later, but it should work with Hadoop 2. ++ A Hadoop environment with HDFS running. GVFS has been tested against + Hadoop 3.1.0. It is recommended to use Hadoop 3.1.0 or later, but it should work with Hadoop 2. x. Please create an [issue](https://www.github.com/datastrato/gravitino/issues) if you find any compatibility issues. -+ You already manage the filesets with Gravitino. ## Configuration | Configuration item | Description | Default value | Required | Since version | |-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-----------------------------------|---------------| -| `fs.AbstractFileSystem.gvfs.impl` | The Gravitino Virtual File System abstract class. Please configure it as `com.datastrato.gravitino.filesystem.hadoop.Gvfs`. | (none) | Yes | 0.5.0 | -| `fs.gvfs.impl` | The Gravitino Virtual File System implementation class. Please configure it as `com.datastrato.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem`. | (none) | Yes | 0.5.0 | -| `fs.gvfs.impl.disable.cache` | Close the Gravitino Virtual File System cache in Hadoop environment. If you need to proxy multi-user operations, please set this value to `true` and create a separate File System for each user. | `false` | No | 0.5.0 | -| `fs.gravitino.server.uri` | The Gravitino server uri which gvfs needs to load the fileset meta. | (none) | Yes | 0.5.0 | -| `fs.gravitino.client.metalake` | The metalake which fileset belongs. | (none) | Yes | 0.5.0 | -| `fs.gravitino.client.authType` | The auth type to initialize the Gravitino client in Gravitino Virtual File System. Now only supports `simple` and `oauth2` auth type. | `simple` | No | 0.5.0 | -| `fs.gravitino.client.oauth2.serverUri` | The auth server uri for the Gravitino client which using `oauth2` auth type in Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | -| `fs.gravitino.client.oauth2.credential` | The auth credential for the Gravitino client which using `oauth2` auth type in Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | -| `fs.gravitino.client.oauth2.path` | The auth server path for the Gravitino client which using `oauth2` auth type in Gravitino Virtual File System. Please remove first slash (`/`) for path, which may like `oauth/token`. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | -| `fs.gravitino.client.oauth2.scope` | The auth scope for the Gravitino client which using `oauth2` auth type in Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | -| `fs.gravitino.fileset.cache.maxCapacity` | The cache capacity in the Gravitino Virtual File System. | `20` | No | 0.5.0 | -| `fs.gravitino.fileset.cache.evictionMillsAfterAccess` | The value of time that the cache evicts the element after access in the Gravitino Virtual File System. The value is in `milliseconds`. | `300000` | No | 0.5.0 | +| `fs.AbstractFileSystem.gvfs.impl` | The Gravitino Virtual File System abstract class, set it to `com.datastrato.gravitino.filesystem.hadoop.Gvfs`. | (none) | Yes | 0.5.0 | +| `fs.gvfs.impl` | The Gravitino Virtual File System implementation class, set it to `com.datastrato.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem`. | (none) | Yes | 0.5.0 | +| `fs.gvfs.impl.disable.cache` | Disable the Gravitino Virtual File System cache in the Hadoop environment. If you need to proxy multi-user operations, please set this value to `true` and create a separate File System for each user. | `false` | No | 0.5.0 | +| `fs.gravitino.server.uri` | The Gravitino server URI which GVFS needs to load the fileset metadata. | (none) | Yes | 0.5.0 | +| `fs.gravitino.client.metalake` | The metalake to which the fileset belongs. | (none) | Yes | 0.5.0 | +| `fs.gravitino.client.authType` | The auth type to initialize the Gravitino client to use with the Gravitino Virtual File System. Currently only supports `simple` and `oauth2` auth types. | `simple` | No | 0.5.0 | +| `fs.gravitino.client.oauth2.serverUri` | The auth server URI for the Gravitino client when using `oauth2` auth type with the Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | +| `fs.gravitino.client.oauth2.credential` | The auth credential for the Gravitino client when using `oauth2` auth type in the Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | +| `fs.gravitino.client.oauth2.path` | The auth server path for the Gravitino client when using `oauth2` auth type with the Gravitino Virtual File System. Please remove the first slash `/` from the path, for example `oauth/token`. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | +| `fs.gravitino.client.oauth2.scope` | The auth scope for the Gravitino client when using `oauth2` auth type with the Gravitino Virtual File System. | (none) | Yes if you use `oauth2` auth type | 0.5.0 | +| `fs.gravitino.fileset.cache.maxCapacity` | The cache capacity of the Gravitino Virtual File System. | `20` | No | 0.5.0 | +| `fs.gravitino.fileset.cache.evictionMillsAfterAccess` | The value of time that the cache expires after accessing in the Gravitino Virtual File System. The value is in `milliseconds`. | `300000` | No | 0.5.0 | You can configure these properties in two ways: -1. Before getting the `FileSystem` in the code, construct the `Configuration` and set the properties: +1. Before obtaining the `FileSystem` in the code, construct a `Configuration` object and set its properties: ```java Configuration conf = new Configuration(); @@ -70,7 +69,7 @@ You can configure these properties in two ways: FileSystem fs = filesetPath.getFileSystem(conf); ``` -2. Configure these properties in the `core-site.xml` file of the Hadoop environment: +2. Configure the properties in the `core-site.xml` file of the Hadoop environment: ```xml @@ -94,16 +93,17 @@ You can configure these properties in two ways: ``` -## How to use +## How to use the Gravitino Virtual File System -Please make sure to have the Gravitino Virtual File System runtime jar firstly, you can get it in +First make sure to obtain the Gravitino Virtual File System runtime jar, which you can get in two ways: -1. Download from the maven central repository: You can download the runtime jar which name is like +1. Download from the maven central repository. You can download the runtime jar named `gravitino-filesystem-hadoop3-runtime-{version}.jar` from [Maven repository](https://mvnrepository.com/). + 2. Compile from the source code: - Download the [Gravitino source code](https://github.com/datastrato/gravitino), and compile it + Download or clone the [Gravitino source code](https://github.com/datastrato/gravitino), and compile it locally using the following command in the Gravitino source code directory: ```shell @@ -112,28 +112,28 @@ two ways: ### Use GVFS via Hadoop shell command -You can use the Hadoop shell command to operate the fileset storage. For example: +You can use the Hadoop shell command to perform operations on the fileset storage. For example: ```shell # 1. Configure the hadoop `core-site.xml` configuration -# you should put the properties above into this file +# You should put the required properties into this file vi ${HADOOP_HOME}/etc/hadoop/core-site.xml -# 2. Place the gvfs runtime jar into your Hadoop environment +# 2. Place the GVFS runtime jar into your Hadoop environment cp gravitino-filesystem-hadoop3-runtime-{version}.jar ${HADOOP_HOME}/share/hadoop/common/lib/ -# 3. Complete the Kerberos authentication of the Hadoop environment (if necessary). -# You need to ensure that the Kerberos has permission to operate the HDFS directory. +# 3. Complete the Kerberos authentication setup of the Hadoop environment (if necessary). +# You need to ensure that the Kerberos has permission on the HDFS directory. kinit -kt your_kerberos.keytab your_kerberos@xxx.com # 4. Try to list the fileset ./${HADOOP_HOME}/bin/hadoop dfs -ls gvfs://fileset/test_catalog/test_schema/test_fileset_1 ``` -### Use GVFS via Java code +### Using the GVFS via Java code -You can also operate the files or directories managed by fileset through Java code. -Please make sure that your code is running in the correct Hadoop environment, and the environment +You can also perform operations on the files or directories managed by fileset through Java code. +Make sure that your code is using the correct Hadoop environment, and that your environment has the `gravitino-filesystem-hadoop3-runtime-{version}.jar` dependency. For example: @@ -149,23 +149,22 @@ FileSystem fs = filesetPath.getFileSystem(conf); fs.getFileStatus(filesetPath); ``` -### Use GVFS with Apache Spark +### Using GVFS with Apache Spark -1. Add the gvfs runtime jar to the Spark environment. +1. Add the GVFS runtime jar to the Spark environment. You can use `--packages` or `--jars` in the Spark submit shell to include the Gravitino Virtual - File System runtime jar, like this: + File System runtime jar, like so: ```shell ./${SPARK_HOME}/bin/spark-submit --packages com.datastrato.gravitino:filesystem-hadoop3-runtime:${version} ``` - If you want to include Gravitino Virtual File System runtime jar in your Spark installation, - make sure to add it to the `${SPARK_HOME}/jars/` folder. + If you want to include the Gravitino Virtual File System runtime jar in your Spark installation, add it to the `${SPARK_HOME}/jars/` folder. 2. Configure the Hadoop configuration when submitting the job. - Then, you can configure the Hadoop configuration in the shell command like this: + You can configure in the shell command in this way: ```shell --conf spark.hadoop.fs.AbstractFileSystem.gvfs.impl=com.datastrato.gravitino.filesystem.hadoop.Gvfs @@ -174,12 +173,12 @@ fs.getFileStatus(filesetPath); --conf spark.hadoop.fs.gravitino.client.metalake=${your_gravitino_metalake} ``` -3. Operate the fileset storage in your code. +3. Perform operations on the fileset storage in your code. - Finally, you can operate the fileset storage in your Spark program: + Finally, you can access the fileset storage in your Spark program: ```scala - // Scala + // Scala code val spark = SparkSession.builder() .appName("Gvfs Example") .getOrCreate() @@ -190,20 +189,20 @@ fs.getFileStatus(filesetPath); ``` -### Use GVFS with Tensorflow +### Using GVFS with Tensorflow -In order for tensorflow to support GVFS, you need to recompile the [tensorflow-io](https://github.com/tensorflow/io) module. +For Tensorflow to support GVFS, you need to recompile the [tensorflow-io](https://github.com/tensorflow/io) module. -1. Add a patch and recompile tensorflow-io. +1. First, add a patch and recompile tensorflow-io. You need to add a [patch](https://github.com/tensorflow/io/pull/1970) to support GVFS on tensorflow-io. Then you can follow the [tutorial](https://github.com/tensorflow/io/blob/master/docs/development.md) to recompile your code and release the tensorflow-io module. -2. Configure the Hadoop configuration. +2. Then you need to configure the Hadoop configuration. - You need to configure the hadoop configuration and `gravitino-filesystem-hadoop3-runtime-{version}.jar`, - also kerberos environment according to the [Use GVFS via Hadoop shell command](#use-gvfs-via-hadoop-shell-command) sections. + You need to configure the Hadoop configuration and add `gravitino-filesystem-hadoop3-runtime-{version}.jar`, + and set up the Kerberos environment according to the [Use GVFS via Hadoop shell command](#use-gvfs-via-hadoop-shell-command) sections. Then you need to set your environment as follows: @@ -235,16 +234,16 @@ Currently, Gravitino Virtual File System supports two kinds of authentication ty The type of `simple` is the default authentication type in Gravitino Virtual File System. -### How to use +### How to use simple authentication -#### Use `simple` authentication type +#### Using `simple` authentication type -Please make sure that your Gravitino server also configure the `simple` authentication mode firstly. +First, make sure that your Gravitino server is also configured to use the `simple` authentication mode. Then, you can configure the Hadoop configuration like this: ```java -// Simple type allows the client to use the environment variable `GRAVITINO_USER` as the user. +// Simple type uses the environment variable `GRAVITINO_USER` as the client user. // If the environment variable `GRAVITINO_USER` isn't set, // the client uses the user of the machine that sends requests. System.setProperty("GRAVITINO_USER", "test"); @@ -261,9 +260,9 @@ Path filesetPath = new Path("gvfs://fileset/test_catalog/test_schema/test_filese FileSystem fs = filesetPath.getFileSystem(conf); ``` -#### Use `oauth2` authentication type +#### Using OAuth authentication -If you want to use `oauth2` authentication type for the Gravitino client in Gravitino Virtual File System, +If you want to use `oauth2` authentication for the Gravitino client in the Gravitino Virtual File System, please refer to this document to complete the configuration of the Gravitino server and the OAuth server: [Security](./security.md). Then, you can configure the Hadoop configuration like this: @@ -274,9 +273,9 @@ conf.set("fs.AbstractFileSystem.gvfs.impl","com.datastrato.gravitino.filesystem. conf.set("fs.gvfs.impl","com.datastrato.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem"); conf.set("fs.gravitino.server.uri","http://localhost:8090"); conf.set("fs.gravitino.client.metalake","test_metalake"); -// Configure the auth type to oatuh2. +// Configure the auth type to oauth2. conf.set("fs.gravitino.client.authType", "oauth2"); -// Configure the oauth conifiguration. +// Configure the OAuth configuration. conf.set("fs.gravitino.client.oauth2.serverUri", "${your_oauth_server_uri}"); conf.set("fs.gravitino.client.oauth2.credential", "${your_client_credential}"); conf.set("fs.gravitino.client.oauth2.path", "${your_oauth_server_path}"); @@ -284,4 +283,3 @@ conf.set("fs.gravitino.client.oauth2.scope", "${your_client_scope}"); Path filesetPath = new Path("gvfs://fileset/test_catalog/test_schema/test_fileset_1"); FileSystem fs = filesetPath.getFileSystem(conf); ``` - diff --git a/docs/manage-fileset-metadata-using-gravitino.md b/docs/manage-fileset-metadata-using-gravitino.md index 5dad793dec4..3ed273c28a3 100644 --- a/docs/manage-fileset-metadata-using-gravitino.md +++ b/docs/manage-fileset-metadata-using-gravitino.md @@ -1,5 +1,5 @@ --- -title: "Manage fileset metadata using Gravitino" +title: Manage fileset metadata using Gravitino slug: /manage-fileset-metadata-using-gravitino date: 2024-4-2 keyword: Gravitino fileset metadata manage @@ -9,30 +9,30 @@ license: Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -This page introduces how to manage fileset metadata by Gravitino. Fileset is a concept brought -out in Gravitino, which is a collection of files and directories. Users can leverage -fileset to manage non-tabular data like training datasets, raw data. +This page introduces how to manage fileset metadata in Gravitino. Filesets +are a collection of files and directories. Users can leverage +filesets to manage non-tabular data like training datasets and other raw data. -Typically, a fileset is mapping to a directory on a file system like HDFS, S3, ADLS, GCS, etc. -With fileset managed by Gravitino, the non-tabular data can be managed as assets together with -tabular data and others in Gravitino with a unified way. +Typically, a fileset is mapped to a directory on a file system like HDFS, S3, ADLS, GCS, etc. +With the fileset managed by Gravitino, the non-tabular data can be managed as assets together with +tabular data in Gravitino in a unified way. -After fileset is created, users can easily access, manage the files/directories through -Fileset's identifier, without needing to know the physical path of the managed datasets. Also, with -unified access control mechanism, filesets can also be managed via the same role based access -control mechanism without needing to set access controls to different storages. +After a fileset is created, users can easily access, manage the files/directories through +the fileset's identifier, without needing to know the physical path of the managed dataset. Also, with +unified access control mechanism, filesets can be managed via the same role based access +control mechanism without needing to set access controls across different storage systems. To use fileset, please make sure that: - - Gravitino server is launched, and the host and port is [http://localhost:8090](http://localhost:8090). - - Metalake has been created. + - Gravitino server has started, and the host and port is [http://localhost:8090](http://localhost:8090). + - A metalake has been created. ## Catalog operations ### Create a catalog :::tip -For fileset catalog, you must specify the catalog `type` as `FILESET` when creating a catalog. +For a fileset catalog, you must specify the catalog `type` as `FILESET` when creating the catalog. ::: You can create a catalog by sending a `POST` request to the `/api/metalakes/{metalake_name}/catalogs` @@ -65,15 +65,15 @@ GravitinoClient gravitinoClient = GravitinoClient Map properties = ImmutableMap.builder() .put("location", "file:/tmp/root") - // Property "location" is optional, if specified all the managed fileset without - // specifying storage location will be stored under this location. + // Property "location" is optional. If specified, a managed fileset without + // a storage location will be stored under this location. .build(); Catalog catalog = gravitinoClient.createCatalog( NameIdentifier.of("metalake", "catalog"), Type.FILESET, - "hadoop", // provider, Gravitino support only "hadoop" for now. - "This is a hadoop fileset catalog", + "hadoop", // provider, Gravitino only supports "hadoop" for now. + "This is a Hadoop fileset catalog", properties); // ... ``` @@ -89,37 +89,37 @@ Currently, Gravitino supports the following catalog providers: ### Load a catalog -Please refer to [Load a catalog](./manage-relational-metadata-using-gravitino.md#load-a-catalog) -from relational catalog for more details. For fileset catalog, the load operation is the same. +Refer to [Load a catalog](./manage-relational-metadata-using-gravitino.md#load-a-catalog) +in relational catalog for more details. For a fileset catalog, the load operation is the same. ### Alter a catalog -Please refer to [Alter a catalog](./manage-relational-metadata-using-gravitino.md#alter-a-catalog) -from relational catalog for more details. For fileset catalog, the alter operation is the same. +Refer to [Alter a catalog](./manage-relational-metadata-using-gravitino.md#alter-a-catalog) +in relational catalog for more details. For a fileset catalog, the alter operation is the same. ### Drop a catalog -Please refer to [Drop a catalog](./manage-relational-metadata-using-gravitino.md#drop-a-catalog) -from relational catalog for more details. For fileset catalog, the drop operation is the same. +Refer to [Drop a catalog](./manage-relational-metadata-using-gravitino.md#drop-a-catalog) +in relational catalog for more details. For a fileset catalog, the drop operation is the same. :::note -Currently, Gravitino doesn't support dropping a catalog with schema and filesets under it. You have +Currently, Gravitino doesn't support dropping a catalog with schemas and filesets under it. You have to drop all the schemas and filesets under the catalog before dropping the catalog. ::: ### List all catalogs in a metalake Please refer to [List all catalogs in a metalake](./manage-relational-metadata-using-gravitino.md#list-all-catalogs-in-a-metalake) -from relational catalog for more details. For fileset catalog, the list operation is the same. +in relational catalog for more details. For a fileset catalog, the list operation is the same. ### List all catalogs' information in a metalake Please refer to [List all catalogs' information in a metalake](./manage-relational-metadata-using-gravitino.md#list-all-catalogs-information-in-a-metalake) -from relational catalog for more details. For fileset catalog, the list operation is the same. +in relational catalog for more details. For a fileset catalog, the list operation is the same. ## Schema operations -`Schema` is a virtual namespace in fileset catalog, which is used to organize the filesets. It +`Schema` is a virtual namespace in a fileset catalog, which is used to organize the fileset. It is similar to the concept of `schema` in relational catalog. :::tip @@ -184,35 +184,35 @@ Currently, Gravitino supports the following schema property: ### Load a schema Please refer to [Load a schema](./manage-relational-metadata-using-gravitino.md#load-a-schema) -from relational catalog for more details. For fileset catalog, the schema load operation is the +in relational catalog for more details. For a fileset catalog, the schema load operation is the same. ### Alter a schema Please refer to [Alter a schema](./manage-relational-metadata-using-gravitino.md#alter-a-schema) -from relational catalog for more details. For fileset catalog, the schema alter operation is the +in relational catalog for more details. For a fileset catalog, the schema alter operation is the same. ### Drop a schema Please refer to [Drop a schema](./manage-relational-metadata-using-gravitino.md#drop-a-schema) -from relational catalog for more details. For fileset catalog, the schema drop operation is the +in relational catalog for more details. For a fileset catalog, the schema drop operation is the same. -Note that the drop operation will also remove all the filesets as well as the managed files +Note that the drop operation will also remove all of the filesets as well as the managed files under this schema path if `cascade` is set to `true`. ### List all schemas under a catalog Please refer to [List all schemas under a catalog](./manage-relational-metadata-using-gravitino.md#list-all-schemas-under-a-catalog) -from relational catalog for more details. For fileset catalog, the schema list operation is the +in relational catalog for more details. For a fileset catalog, the schema list operation is the same. ## Fileset operations :::tip - - Users should create a metalake, a catalog and a schema before creating a fileset. - - Current Gravitino only supports managing Hadoop Compatible File System (HCFS) locations. + - Users should create a metalake, a catalog, and a schema before creating a fileset. + - Currently, Gravitino only supports managing Hadoop Compatible File System (HCFS) locations. ::: ### Create a fileset @@ -265,32 +265,32 @@ filesetCatalog.createFileset( -Currently, Gravitino supports two **types** of the fileset: +Currently, Gravitino supports two **types** of filesets: - - `MANAGED`: The storage location of the fileset is managed by Gravitino, when specified as + - `MANAGED`: The storage location of the fileset is managed by Gravitino when specified as `MANAGED`, the physical location of the fileset will be deleted when this fileset is dropped. - `EXTERNAL`: The storage location of the fileset is **not** managed by Gravitino, when - specified as `EXTERNAL`, the physical location of the fileset will **not** be deleted when - this fileset is dropped. + specified as `EXTERNAL`, the files of the fileset will **not** be deleted when + the fileset is dropped. **storageLocation** -The `storageLocation` is the physical location of the fileset, user can specify this location -when creating a fileset, or follow the rule of the catalog/schema location if not specified. +The `storageLocation` is the physical location of the fileset. Users can specify this location +when creating a fileset, or follow the rules of the catalog/schema location if not specified. -For `MANAGED` fileset, the storage location is: +For a `MANAGED` fileset, the storage location is: -1. The one specified by user in the fileset creation. -2. When catalog property `location` is specified but schema property `location` is not specified, - the storage location is `catalog location/schema name/fileset name` if not specified. -3. When catalog property `location` is not specified but schema property `location` is specified, - the storage location is `schema location/fileset name` if not specified. -4. When both catalog property `location` and schema property `location` are specified, the storage - location is `schema location/fileset name` if not specified. -5. When both catalog property `location` and schema property `location` are not specified, user +1. The one specified by the user during the fileset creation. +2. When the catalog property `location` is specified but the schema property `location` isn't specified, + the storage location is `catalog location/schema name/fileset name`. +3. When the catalog property `location` isn't specified but the schema property `location` is specified, + the storage location is `schema location/fileset name`. +4. When both the catalog property `location` and the schema property `location` are specified, the storage + location is `schema location/fileset name`. +5. When both the catalog property `location` and schema property `location` isn't specified, the user should specify the `storageLocation` in the fileset creation. -For `EXTERNAL` fileset, users should specify `storageLocation` during fileset creation, +For `EXTERNAL` fileset, users should specify `storageLocation` during the fileset creation, otherwise, Gravitino will throw an exception. ### Alter a fileset @@ -340,15 +340,15 @@ Currently, Gravitino supports the following changes to a fileset: | Supported modification | JSON | Java | |------------------------------------|--------------------------------------------------------------|-----------------------------------------------| -| Rename fileset | `{"@type":"rename","newName":"fileset_renamed"}` | `FilesetChange.rename("fileset_renamed")` | -| Update comment | `{"@type":"updateComment","newComment":"new_comment"}` | `FilesetChange.updateComment("new_comment")` | +| Rename a fileset | `{"@type":"rename","newName":"fileset_renamed"}` | `FilesetChange.rename("fileset_renamed")` | +| Update a comment | `{"@type":"updateComment","newComment":"new_comment"}` | `FilesetChange.updateComment("new_comment")` | | Set a fileset property | `{"@type":"setProperty","property":"key1","value":"value1"}` | `FilesetChange.setProperty("key1", "value1")` | | Remove a fileset property | `{"@type":"removeProperty","property":"key1"}` | `FilesetChange.removeProperty("key1")` | ### Drop a fileset You can remove a fileset by sending a `DELETE` request to the `/api/metalakes/{metalake_name} -/catalogs/{catalog_name}/schemas/{schema_name}/filesets/{fileset_name}` endpoint or just use the +/catalogs/{catalog_name}/schemas/{schema_name}/filesets/{fileset_name}` endpoint or by using the Gravitino Java client. The following is an example of dropping a fileset: @@ -378,14 +378,14 @@ filesetCatalog.dropFileset(NameIdentifier.of("metalake", "catalog", "schema", "f -For `MANAGED` fileset, the physical location of the fileset will be deleted when this fileset is +For a `MANAGED` fileset, the physical location of the fileset will be deleted when this fileset is dropped. For `EXTERNAL` fileset, only the metadata of the fileset will be removed. ### List filesets You can list all filesets in a schema by sending a `GET` request to the `/api/metalakes/ -{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/filesets` endpoint or just use the -Gravitino Java client. The following is an example of list all the filesets in a schema: +{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/filesets` endpoint or by using the +Gravitino Java client. The following is an example of listing all the filesets in a schema: From 1e7759c3f4d2b4d8ce7736b6bd09c07604777af6 Mon Sep 17 00:00:00 2001 From: mchades Date: Fri, 12 Apr 2024 17:50:36 +0800 Subject: [PATCH 011/106] [#2800][#2801] fix(license): fix incorrect LICENSE information of Kafka source code (#2868) ### What changes were proposed in this pull request? fix incorrect LICENSE information of Kafka source code ### Why are the changes needed? Fix: #2800 #2801 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no need --- LICENSE | 5 +++++ LICENSE.bin | 1 + .../kafka/embedded/KafkaClusterEmbedded.java | 15 +++++++++++++-- .../catalog/kafka/embedded/KafkaEmbedded.java | 15 +++++++++++++-- .../catalog/kafka/embedded/ZooKeeperEmbedded.java | 15 +++++++++++++-- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index 679674056c0..eab8b763316 100644 --- a/LICENSE +++ b/LICENSE @@ -266,6 +266,11 @@ Apache Submarine ./bin/common.sh + Confluent Kafka Streams Examples + ./catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaClusterEmbedded.java + ./catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaEmbedded.java + ./catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/ZooKeeperEmbedded.java + Trino ./integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/CloseableGroup.java ./trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/SortingColumn.java diff --git a/LICENSE.bin b/LICENSE.bin index d590889badf..fc972c8ed07 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -354,6 +354,7 @@ Snappy Java XNIO API WildFly + Confluent Kafka Streams Examples This product bundles various third-party components also under the Apache Software Foundation License 1.1 diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaClusterEmbedded.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaClusterEmbedded.java index 510f115ac66..97ea48a5672 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaClusterEmbedded.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaClusterEmbedded.java @@ -1,6 +1,17 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Copyright Confluent Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.datastrato.gravitino.catalog.kafka.embedded; diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaEmbedded.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaEmbedded.java index 756a68d95df..2499966146b 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaEmbedded.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/KafkaEmbedded.java @@ -1,6 +1,17 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Copyright Confluent Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.datastrato.gravitino.catalog.kafka.embedded; diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/ZooKeeperEmbedded.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/ZooKeeperEmbedded.java index 9b4c89deaec..4099aea1c7a 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/ZooKeeperEmbedded.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/embedded/ZooKeeperEmbedded.java @@ -1,6 +1,17 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Copyright Confluent Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.datastrato.gravitino.catalog.kafka.embedded; From 67f9eca7447268bce4dbd5a2946f804d875d44c7 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 12 Apr 2024 19:10:02 +0800 Subject: [PATCH 012/106] [#2888] fix(core): Fix the possible concurrency for `KvGarbageCollector` (#2890) ### What changes were proposed in this pull request? Add a check mechanism in `KvGarbageCollector` to avoid removing uncommitted data wrote into kv storage a few seconds or minutes ago. ### Why are the changes needed? There is a potential that `KvGarbageCollector` will take normal data as uncommitted data and remove it. For more please see #2888 Fix: #2888 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- .../storage/kv/KvGarbageCollector.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java index a2f39c361d3..d505a541864 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java @@ -11,6 +11,7 @@ import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.generateCommitKey; import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.generateKey; import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.getBinaryTransactionId; +import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.getTransactionId; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Entity.EntityType; @@ -43,6 +44,7 @@ public final class KvGarbageCollector implements Closeable { private final KvBackend kvBackend; private final Config config; private final EntityKeyEncoder entityKeyEncoder; + private long frequencyInMinutes; private static final String TIME_STAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; @@ -69,8 +71,9 @@ public void start() { // We will collect garbage every 10 minutes at least. If the dateTimeLineMinute is larger than // 100 minutes, we would collect garbage every dateTimeLineMinute/10 minutes. - long frequency = Math.max(dateTimeLineMinute / 10, 10); - garbageCollectorPool.scheduleAtFixedRate(this::collectAndClean, 5, frequency, TimeUnit.MINUTES); + this.frequencyInMinutes = Math.max(dateTimeLineMinute / 10, 10); + garbageCollectorPool.scheduleAtFixedRate( + this::collectAndClean, 5, frequencyInMinutes, TimeUnit.MINUTES); } @VisibleForTesting @@ -98,6 +101,16 @@ private void collectAndRemoveUncommittedData() throws IOException { .predicate( (k, v) -> { byte[] transactionId = getBinaryTransactionId(k); + + // Only remove the uncommitted data that were written frequencyInMinutes + // minutes ago. + // It may have concurrency issues with TransactionalKvBackendImpl#commit. + long writeTime = getTransactionId(transactionId) >> 18; + if (writeTime + < (System.currentTimeMillis() - frequencyInMinutes * 60 * 1000 * 2)) { + return false; + } + return kvBackend.get(generateCommitKey(transactionId)) == null; }) .limit(10000) /* Each time we only collect 10000 entities at most*/ @@ -209,7 +222,7 @@ private void collectAndRemoveOldVersionData() throws IOException { // All keys in this transaction have been deleted, we can remove the commit mark. if (keysDeletedCount == keysInTheTransaction.size()) { - long timestamp = TransactionalKvBackendImpl.getTransactionId(transactionId) >> 18; + long timestamp = getTransactionId(transactionId) >> 18; LOG.info( "Physically delete commit mark: {}, createTime: '{}({})', key: '{}'", Bytes.wrap(kv.getKey()), @@ -261,7 +274,7 @@ LogHelper decodeKey(byte[] key, byte[] timestampArray) { LOG.warn("Unable to decode key: {}", Bytes.wrap(key), e); return LogHelper.NONE; } - long timestamp = TransactionalKvBackendImpl.getTransactionId(timestampArray) >> 18; + long timestamp = getTransactionId(timestampArray) >> 18; String ts = DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT); return new LogHelper(entityTypePair.getKey(), entityTypePair.getValue(), timestamp, ts); From 08f9cc4adef185d86621e8aaba4e36c0d2c43e7c Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Fri, 12 Apr 2024 19:28:00 +0800 Subject: [PATCH 013/106] [#2348] Improvement(docs): Add user doc for relational entity store (#2768) ### What changes were proposed in this pull request? Add user documentation for relational entity store. Depends on #2584. ### Why are the changes needed? Fix: #2348 --------- Co-authored-by: xiaojiebao Co-authored-by: Jerry Shao --- docs/how-to-install.md | 8 +- docs/how-to-use-relational-backend-storage.md | 90 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 docs/how-to-use-relational-backend-storage.md diff --git a/docs/how-to-install.md b/docs/how-to-install.md index 02d495c6af7..de3ff653d5a 100644 --- a/docs/how-to-install.md +++ b/docs/how-to-install.md @@ -43,9 +43,15 @@ The Gravitino binary distribution package contains the following files: | └── log4j2.properties # log4j configuration for the Gravitino server. |── libs/ # Gravitino server dependencies libraries. |── logs/ # Gravitino server logs. Automatically created after the Gravitino server starts. - └── data/ # Default directory for the Gravitino server to store data. + |── data/ # Default directory for the Gravitino server to store data. + └── scripts/ # Extra scripts for Gravitino. ``` +#### Initialize the RDBMS (Optional) + +If you want to use the relational backend storage, you need to initialize the RDBMS firstly. For +the details on how to initialize the RDBMS, please check [How to use relational backend storage](./how-to-use-relational-backend-storage.md). + #### Configure the Gravitino server The Gravitino server configuration file is `conf/gravitino.conf`. You can configure the Gravitino diff --git a/docs/how-to-use-relational-backend-storage.md b/docs/how-to-use-relational-backend-storage.md new file mode 100644 index 00000000000..3d6900cd13e --- /dev/null +++ b/docs/how-to-use-relational-backend-storage.md @@ -0,0 +1,90 @@ +--- +title: How to use relational backend storage +slug: /how-to-use-relational-backend-storage +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +## Introduction + +Before the version `0.5.0`, Gravitino only supports KV backend storage to store metadata. Since +RDBMS is widely used in the industry, starting from the version `0.5.0`, Gravitino supports using +RDBMS as relational backend storage to store metadata. This doc will guide you on how to use the +relational backend storage in Gravitino. + +Relational backend storage mainly aims to the users who are accustomed to using RDBMS to +store data or lack available a KV storage, and want to use Gravitino. + +With relational backend storage, you can quickly deploy Gravitino in a production environment and +take advantage of relational storage to manage metadata. + +### What kind of backend storage are supported + +Currently, relational backend storage supports the `JDBCBackend`, and uses `MySQL` as the +default storage for `JDBCBackend`. + +## How to use + +### Prerequisites + ++ MySQL 5.7 or 8.0. ++ Gravitino distribution package. ++ MySQL connector Jar (Should be compatible with the version of MySQL instance). + +### Step 1: Get the initialization script + +You need to `download` and `unzip` the distribution package firstly, please see +[How to install Gravitino](how-to-install.md). + +Then you can get the initialization script in the directory: + +```text +${GRAVITINO_HOME}/scripts/mysql/ +``` + +The script name is like `schema-{version}-mysql.sql`, and the `version` depends on your Gravitino version. +For example, if your Gravitino version is `0.6.0`, then you can choose the **latest version** script +file that is equal or smaller than `0.6.0`, you can choose the `schema-0.5.0-mysql.sql` script. + +### Step 2: Initialize the database + +Please create a database in MySQL in advance, and execute the initialization script obtained above in the database. + +### Step 3: Place the MySQL connector Jar + +You should **download** the MySQL connector Jar for the corresponding version of MySQL you use +(You can download it from the [maven-central-repo](https://repo1.maven.org/maven2/mysql/mysql-connector-java/)), +which is name like `mysql-connector-java-{version}.jar`. + +Then please place it in the distribution package directory: + +```text +${GRAVITINO_HOME}/libs/ +``` + +### Step 4: Set up the Gravitino server configs + +Find the server configuration file which name is `gravitino.conf` in the distribution package directory: + +```text +${GRAVITINO_HOME}/conf/ +``` + +Then set up the following server configs: + +```text +gravitino.entity.store = relational +gravitino.entity.store.relational = JDBCBackend +gravitino.entity.store.relational.jdbcUrl = ${your_jdbc_url} +gravitino.entity.store.relational.jdbcDriver = ${your_driver_name} +gravitino.entity.store.relational.jdbcUser = ${your_username} +gravitino.entity.store.relational.jdbcPassword = ${your_password} +``` + +### Step 5: Start the server + +Finally, you can run the script in the distribution package directory to start the server: + +```shell +./${GRAVITINO_HOME}/bin/gravitino.sh start +``` From 282d08248028bacd673f77af28f70b011c506fc1 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 13 Apr 2024 19:39:19 +0800 Subject: [PATCH 014/106] [#2886] improvement(ci): optimize Docker container to suite CI framework (#2887) ### What changes were proposed in this pull request? - remove log from container stdout - improvement: add config to avoid storage_medium_check ### Why are the changes needed? Fix: #2886 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI --- dev/docker/doris/start.sh | 19 +++++-------------- docs/docker-image-details.md | 22 +++++++++++++--------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/dev/docker/doris/start.sh b/dev/docker/doris/start.sh index b0b84a826bd..64971c88aa0 100644 --- a/dev/docker/doris/start.sh +++ b/dev/docker/doris/start.sh @@ -21,7 +21,6 @@ DISK_FREE=`df -BG | grep '/$' | tr -s ' ' | cut -d ' ' -f4 | grep -o '[0-9]*'` if [ "$DISK_FREE" -le "$THRESHOLD" ] then echo "ERROR: Doris FE (version < 2.1.0) can not start with less than ${THRESHOLD}G disk space." - exit 1 fi # comment a code snippet about max_map_count, it's not necessary for IT environment @@ -37,6 +36,7 @@ PRIORITY_NETWORKS=$(echo "${CONTAINER_IP}" | awk -F '.' '{print$1"."$2"."$3".0/2 echo "add priority_networks = ${PRIORITY_NETWORKS} to fe.conf & be.conf" echo "priority_networks = ${PRIORITY_NETWORKS}" >> ${DORIS_FE_HOME}/conf/fe.conf echo "priority_networks = ${PRIORITY_NETWORKS}" >> ${DORIS_BE_HOME}/conf/be.conf +echo "report_disk_state_interval_seconds = 10" >> ${DORIS_BE_HOME}/conf/be.conf # start doris fe and be in daemon mode ${DORIS_FE_HOME}/bin/start_fe.sh --daemon @@ -62,10 +62,7 @@ for i in {1..10}; do done if [ "$fe_started" = false ]; then - echo "Doris fe failed to start" - - cat ${DORIS_FE_HOME}/log/fe.* - exit 1 + echo "ERROR: Doris fe failed to start" fi # check for be started @@ -88,10 +85,7 @@ for i in {1..10}; do done if [ "$be_started" = false ]; then - echo "Doris be failed to start" - - cat ${DORIS_BE_HOME}/log/* - exit 1 + echo "ERROR: Doris be failed to start" fi @@ -113,11 +107,8 @@ for i in {1..10}; do done if [ "$be_added" = false ]; then - echo "Doris BE failed to add to FE" - cat ${DORIS_FE_HOME}/log/fe.* ${DORIS_BE_HOME}/log/* - - exit 1 + echo "ERROR: Doris BE failed to add to FE" fi # persist the container -tail -f ${DORIS_FE_HOME}/log/fe.log ${DORIS_BE_HOME}/log/be.INFO +tail -f /dev/null diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index 18baf356341..dfda8cfb6e7 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -170,17 +170,21 @@ You can use this image to test Apache Doris. Changelog +- gravitino-ci-doris:0.1.3 + - To adapt to the CI framework, don't exit container when start failed, logs are no longer printed to stdout. + - Add `report_disk_state_interval_seconds` config to decrease report interval. + - gravitino-ci-doris:0.1.2 - - Add a check for the status of Doris BE, add retry for adding BE nodes. + - Add a check for the status of Doris BE, add retry for adding BE nodes. - gravitino-ci-doris:0.1.1 - - Optimize `start.sh`, add disk space check before starting Doris, exit when FE or BE start failed, add log to stdout + - Optimize `start.sh`, add disk space check before starting Doris, exit when FE or BE start failed, add log to stdout - gravitino-ci-doris:0.1.0 - - Docker image `datastrato/gravitino-ci-doris:0.1.0` - - Start Doris BE & FE in one container - - Please set table properties `"replication_num" = "1"` when creating a table in Doris, because the default replication number is 3, but the Doris container only has one BE. - - Username: `root`, Password: N/A (password is empty) - - Expose ports: - - `8030` Doris FE HTTP port - - `9030` Doris FE MySQL server port + - Docker image `datastrato/gravitino-ci-doris:0.1.0` + - Start Doris BE & FE in one container + - Please set table properties `"replication_num" = "1"` when creating a table in Doris, because the default replication number is 3, but the Doris container only has one BE. + - Username: `root`, Password: N/A (password is empty) + - Expose ports: + - `8030` Doris FE HTTP port + - `9030` Doris FE MySQL server port From 139a40a0918536e5a2419c423ee116f1961e1a7a Mon Sep 17 00:00:00 2001 From: Xun Liu Date: Sat, 13 Apr 2024 22:45:36 +0800 Subject: [PATCH 015/106] [#2252][#2253][#2470] (PyClient): Add Fileset APIs in Gravitino Python client (#2898) ### What changes were proposed in this pull request? Support Fileset Catalog in Python client, support below functions: 1. list_filesets() 2. load_fileset() 3. create_fileset() 4. alter_fileset() 5. drop_fileset() ### Why are the changes needed? Fix: #2470 #2252 #2253 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI Passed --- .../client-python/gravitino/api/__init__.py | 4 + clients/client-python/gravitino/api/audit.py | 44 +++ .../client-python/gravitino/api/auditable.py | 18 ++ .../client-python/gravitino/api/catalog.py | 129 +++++++++ .../gravitino/api/catalog_change.py | 254 ++++++++++++++++++ .../client-python/gravitino/api/fileset.py | 98 +++++++ .../gravitino/api/fileset_change.py | 253 +++++++++++++++++ .../metalake_change.py} | 6 +- clients/client-python/gravitino/api/schema.py | 32 +++ .../gravitino/api/schema_change.py | 133 +++++++++ .../gravitino/api/supports_schemas.py | 126 +++++++++ .../gravitino/catalog/base_schema_catalog.py | 165 ++++++++++++ .../gravitino/catalog/fileset_catalog.py | 184 +++++++++++++ .../client/gravitino_admin_client.py | 45 ++-- .../gravitino/client/gravitino_client.py | 76 ++++++ .../gravitino/client/gravitino_client_base.py | 24 +- .../gravitino/client/gravitino_metalake.py | 181 ++++++++++++- .../client-python/gravitino/dto/audit_dto.py | 50 +++- .../gravitino/dto/catalog_dto.py | 51 ++++ .../gravitino/dto/dto_converters.py | 34 ++- .../gravitino/dto/fileset_dto.py | 41 +++ .../gravitino/dto/metalake_dto.py | 7 - .../dto/requests/catalog_create_request.py | 38 +++ .../dto/requests/catalog_update_request.py | 75 ++++++ .../dto/requests/catalog_updates_request.py | 31 +++ .../dto/requests/fileset_create_request.py | 29 ++ .../dto/requests/fileset_update_request.py | 131 +++++++++ .../dto/requests/fileset_updates_request.py | 21 ++ .../dto/requests/metalake_update_request.py | 49 ++-- .../dto/requests/schema_create_request.py | 20 ++ .../dto/requests/schema_update_request.py | 79 ++++++ .../dto/requests/schema_updates_request.py | 27 ++ .../gravitino/dto/responses/base_response.py | 8 +- .../dto/responses/catalog_list_response.py | 24 ++ .../dto/responses/catalog_response.py | 32 +++ .../gravitino/dto/responses/drop_response.py | 9 +- .../dto/responses/entity_list_response.py | 27 ++ .../dto/responses/fileset_response.py | 26 ++ .../dto/responses/metalake_list_response.py | 5 + .../dto/responses/schema_response.py | 29 ++ .../client-python/gravitino/dto/schema_dto.py | 34 +++ .../gravitino/name_identifier.py | 87 +++--- clients/client-python/gravitino/namespace.py | 68 +++-- .../gravitino/rest/rest_message.py | 42 +++ .../gravitino/utils/exceptions.py | 2 +- .../tests/integration/__init__.py | 4 + .../{ => integration}/integration_test_env.py | 7 +- .../tests/integration/test_fileset_catalog.py | 127 +++++++++ .../test_gravitino_admin_client.py | 46 ++-- 49 files changed, 2842 insertions(+), 190 deletions(-) create mode 100644 clients/client-python/gravitino/api/__init__.py create mode 100644 clients/client-python/gravitino/api/audit.py create mode 100644 clients/client-python/gravitino/api/auditable.py create mode 100644 clients/client-python/gravitino/api/catalog.py create mode 100644 clients/client-python/gravitino/api/catalog_change.py create mode 100644 clients/client-python/gravitino/api/fileset.py create mode 100644 clients/client-python/gravitino/api/fileset_change.py rename clients/client-python/gravitino/{meta_change.py => api/metalake_change.py} (98%) create mode 100644 clients/client-python/gravitino/api/schema.py create mode 100644 clients/client-python/gravitino/api/schema_change.py create mode 100644 clients/client-python/gravitino/api/supports_schemas.py create mode 100644 clients/client-python/gravitino/catalog/base_schema_catalog.py create mode 100644 clients/client-python/gravitino/catalog/fileset_catalog.py create mode 100644 clients/client-python/gravitino/client/gravitino_client.py create mode 100644 clients/client-python/gravitino/dto/catalog_dto.py create mode 100644 clients/client-python/gravitino/dto/fileset_dto.py create mode 100644 clients/client-python/gravitino/dto/requests/catalog_create_request.py create mode 100644 clients/client-python/gravitino/dto/requests/catalog_update_request.py create mode 100644 clients/client-python/gravitino/dto/requests/catalog_updates_request.py create mode 100644 clients/client-python/gravitino/dto/requests/fileset_create_request.py create mode 100644 clients/client-python/gravitino/dto/requests/fileset_update_request.py create mode 100644 clients/client-python/gravitino/dto/requests/fileset_updates_request.py create mode 100644 clients/client-python/gravitino/dto/requests/schema_create_request.py create mode 100644 clients/client-python/gravitino/dto/requests/schema_update_request.py create mode 100644 clients/client-python/gravitino/dto/requests/schema_updates_request.py create mode 100644 clients/client-python/gravitino/dto/responses/catalog_list_response.py create mode 100644 clients/client-python/gravitino/dto/responses/catalog_response.py create mode 100644 clients/client-python/gravitino/dto/responses/entity_list_response.py create mode 100644 clients/client-python/gravitino/dto/responses/fileset_response.py create mode 100644 clients/client-python/gravitino/dto/responses/schema_response.py create mode 100644 clients/client-python/gravitino/dto/schema_dto.py create mode 100644 clients/client-python/gravitino/rest/rest_message.py create mode 100644 clients/client-python/tests/integration/__init__.py rename clients/client-python/tests/{ => integration}/integration_test_env.py (95%) create mode 100644 clients/client-python/tests/integration/test_fileset_catalog.py rename clients/client-python/tests/{ => integration}/test_gravitino_admin_client.py (83%) diff --git a/clients/client-python/gravitino/api/__init__.py b/clients/client-python/gravitino/api/__init__.py new file mode 100644 index 00000000000..5779a3ad252 --- /dev/null +++ b/clients/client-python/gravitino/api/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" diff --git a/clients/client-python/gravitino/api/audit.py b/clients/client-python/gravitino/api/audit.py new file mode 100644 index 00000000000..08dc1407568 --- /dev/null +++ b/clients/client-python/gravitino/api/audit.py @@ -0,0 +1,44 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC, abstractmethod +from datetime import datetime + + +class Audit(ABC): + """Represents the audit information of an entity.""" + + @abstractmethod + def creator(self) -> str: + """The creator of the entity. + + Returns: + the creator of the entity. + """ + pass + + @abstractmethod + def create_time(self) -> datetime: + """The creation time of the entity. + + Returns: + The creation time of the entity. + """ + pass + + @abstractmethod + def last_modifier(self) -> str: + """ + Returns: + The last modifier of the entity. + """ + pass + + @abstractmethod + def last_modified_time(self) -> datetime: + """ + Returns: + The last modified time of the entity. + """ + pass diff --git a/clients/client-python/gravitino/api/auditable.py b/clients/client-python/gravitino/api/auditable.py new file mode 100644 index 00000000000..fd7c6e8ad67 --- /dev/null +++ b/clients/client-python/gravitino/api/auditable.py @@ -0,0 +1,18 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC, abstractmethod + +from gravitino.api.audit import Audit + + +class Auditable(ABC): + """ + An auditable entity is an entity that has audit information associated with it. This audit + information is used to track changes to the entity. + """ + + @abstractmethod + def audit_info(self) -> Audit: + pass diff --git a/clients/client-python/gravitino/api/catalog.py b/clients/client-python/gravitino/api/catalog.py new file mode 100644 index 00000000000..fef0de70735 --- /dev/null +++ b/clients/client-python/gravitino/api/catalog.py @@ -0,0 +1,129 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from enum import Enum +from typing import Dict, Optional + +from gravitino.api.auditable import Auditable +from gravitino.api.supports_schemas import SupportsSchemas + + +class Catalog(Auditable): + """The interface of a catalog. The catalog is the second level entity in the gravitino system, + containing a set of tables. + """ + class Type(Enum): + """The type of the catalog.""" + + RELATIONAL = "relational" + """"Catalog Type for Relational Data Structure, like db.table, catalog.db.table.""" + + FILESET = "fileset" + """Catalog Type for Fileset System (including HDFS, S3, etc.), like path/to/file""" + + MESSAGING = "messaging" + """Catalog Type for Message Queue, like kafka://topic""" + + UNSUPPORTED = "unsupported" + """Catalog Type for test only.""" + + PROPERTY_PACKAGE = "package" + """A reserved property to specify the package location of the catalog. The "package" is a string + of path to the folder where all the catalog related dependencies is located. The dependencies + under the "package" will be loaded by Gravitino to create the catalog. + + The property "package" is not needed if the catalog is a built-in one, Gravitino will search + the proper location using "provider" to load the dependencies. Only when the folder is in + different location, the "package" property is needed. + """ + + @abstractmethod + def name(self) -> str: + """ + Returns: + The name of the catalog. + """ + pass + + @abstractmethod + def type(self) -> Type: + """ + Returns: + The type of the catalog. + """ + pass + + @abstractmethod + def provider(self) -> str: + """ + Returns: + The provider of the catalog. + """ + pass + + @abstractmethod + def comment(self) -> Optional[str]: + """The comment of the catalog. Note. this method will return null if the comment is not set for + this catalog. + + Returns: + The provider of the catalog. + """ + pass + + @abstractmethod + def properties(self) -> Optional[Dict[str, str]]: + """ + The properties of the catalog. Note, this method will return null if the properties are not set. + + Returns: + The properties of the catalog. + """ + pass + + def as_schemas(self) -> SupportsSchemas: + """Return the {@link SupportsSchemas} if the catalog supports schema operations. + + Raises: + UnsupportedOperationException if the catalog does not support schema operations. + + Returns: + The {@link SupportsSchemas} if the catalog supports schema operations. + """ + raise UnsupportedOperationException("Catalog does not support schema operations") + + def as_table_catalog(self) -> 'TableCatalog': + """ + Raises: + UnsupportedOperationException if the catalog does not support table operations. + + Returns: + the {@link TableCatalog} if the catalog supports table operations. + """ + raise UnsupportedOperationException("Catalog does not support table operations") + + def as_fileset_catalog(self) -> 'FilesetCatalog': + """ + Raises: + UnsupportedOperationException if the catalog does not support fileset operations. + + Returns: + the FilesetCatalog if the catalog supports fileset operations. + """ + raise UnsupportedOperationException("Catalog does not support fileset operations") + + def as_topic_catalog(self) -> 'TopicCatalog': + """ + Returns: + the {@link TopicCatalog} if the catalog supports topic operations. + + Raises: + UnsupportedOperationException if the catalog does not support topic operations. + """ + raise UnsupportedOperationException("Catalog does not support topic operations") + + +class UnsupportedOperationException(Exception): + pass \ No newline at end of file diff --git a/clients/client-python/gravitino/api/catalog_change.py b/clients/client-python/gravitino/api/catalog_change.py new file mode 100644 index 00000000000..f971a9b159d --- /dev/null +++ b/clients/client-python/gravitino/api/catalog_change.py @@ -0,0 +1,254 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC + + +class CatalogChange(ABC): + """ + A catalog change is a change to a catalog. It can be used to rename a catalog, update the comment + of a catalog, set a property and value pair for a catalog, or remove a property from a catalog. + """ + + @staticmethod + def rename(new_name): + """Creates a new catalog change to rename the catalog. + + Args: + new_name: The new name of the catalog. + + Returns: + The catalog change. + """ + return CatalogChange.RenameCatalog(new_name) + + @staticmethod + def update_comment(new_comment): + """Creates a new catalog change to update the catalog comment. + + Args: + new_comment: The new comment for the catalog. + + Returns: + The catalog change. + """ + return CatalogChange.UpdateCatalogComment(new_comment) + + @staticmethod + def set_property(property, value): + """Creates a new catalog change to set the property and value for the catalog. + + Args: + property: The property name to set. + value: The value to set the property to. + + Returns: + The catalog change. + """ + return CatalogChange.SetProperty(property, value) + + @staticmethod + def remove_property(property): + """Creates a new catalog change to remove a property from the catalog. + + Args: + property: The property name to remove. + + Returns: + The catalog change. + """ + return CatalogChange.RemoveProperty(property) + + class RenameCatalog: + """A catalog change to rename the catalog.""" + + def __init__(self, new_name): + self.new_name = new_name + + def get_new_name(self): + """Retrieves the new name set for the catalog. + + Returns: + The new name of the catalog. + """ + return self.new_name + + def __eq__(self, other): + """Compares this RenameCatalog instance with another object for equality. Two instances are + considered equal if they designate the same new name for the catalog. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents an identical catalog renaming operation; false otherwise. + """ + if not isinstance(other, CatalogChange.RenameCatalog): + return False + return self.new_name == other.new_name + + def __hash__(self): + """Generates a hash code for this RenameCatalog instance. The hash code is primarily based on + the new name for the catalog. + + Returns: + A hash code value for this renaming operation. + """ + return hash(self.new_name) + + def __str__(self): + """Provides a string representation of the RenameCatalog instance. This string includes the + class name followed by the new name of the catalog. + + Returns: + A string summary of this renaming operation. + """ + return f"RENAMECATALOG {self.new_name}" + + class UpdateCatalogComment: + """A catalog change to update the catalog comment.""" + + def __init__(self, new_comment): + self.new_comment = new_comment + + def get_new_comment(self): + """Retrieves the new comment intended for the catalog. + + Returns: + The new comment that has been set for the catalog. + """ + return self.new_comment + + def __eq__(self, other): + """Compares this UpdateCatalogComment instance with another object for equality. + Two instances are considered equal if they designate the same new comment for the catalog. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same comment update; false otherwise. + """ + if not isinstance(other, CatalogChange.UpdateCatalogComment): + return False + return self.new_comment == other.new_comment + + def __hash__(self): + """Generates a hash code for this UpdateCatalogComment instance. + The hash code is based on the new comment for the catalog. + + Returns: + A hash code representing this comment update operation. + """ + return hash(self.new_comment) + + def __str__(self): + """Provides a string representation of the UpdateCatalogComment instance. + This string format includes the class name followed by the new comment for the catalog. + + Returns: + A string summary of this comment update operation. + """ + return f"UPDATECATALOGCOMMENT {self.new_comment}" + + class SetProperty: + """A catalog change to set the property and value for the catalog.""" + + def __init__(self, property, value): + self.property = property + self.value = value + + def get_property(self): + """Retrieves the name of the property being set in the catalog. + + Returns: + The name of the property. + """ + return self.property + + def get_value(self): + """Retrieves the value assigned to the property in the catalog. + + Returns: + The value of the property. + """ + return self.value + + def __eq__(self, other): + """Compares this SetProperty instance with another object for equality. + Two instances are considered equal if they have the same property and value for the catalog. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same property setting; false otherwise. + """ + if not isinstance(other, CatalogChange.SetProperty): + return False + return self.property == other.property and self.value == other.value + + def __hash__(self): + """Generates a hash code for this SetProperty instance. + The hash code is based on both the property name and its assigned value. + + Returns: + A hash code value for this property setting. + """ + return hash((self.property, self.value)) + + def __str__(self): + """Provides a string representation of the SetProperty instance. + This string format includes the class name followed by the property and its value. + + Returns: + A string summary of the property setting. + """ + return f"SETPROPERTY {self.property} {self.value}" + + class RemoveProperty: + """A catalog change to remove a property from the catalog.""" + + def __init__(self, property): + self.property = property + + def get_property(self): + """Retrieves the name of the property to be removed from the catalog. + + Returns: + The name of the property for removal. + """ + return self.property + + def __eq__(self, other): + """Compares this RemoveProperty instance with another object for equality. + Two instances are considered equal if they target the same property for removal from the catalog. + + Args: + other The object to compare with this instance. + + Returns: + true if the given object represents the same property removal; false otherwise. + """ + if not isinstance(other, CatalogChange.RemoveProperty): + return False + return self.property == other.property + + def __hash__(self): + """Generates a hash code for this RemoveProperty instance. + The hash code is based on the property name that is to be removed from the catalog. + + Returns: + A hash code value for this property removal operation. + """ + return hash(self.property) + + def __str__(self): + """Provides a string representation of the RemoveProperty instance. + This string format includes the class name followed by the property name to be removed. + + Returns: + A string summary of the property removal operation. + """ + return f"REMOVEPROPERTY {self.property}" diff --git a/clients/client-python/gravitino/api/fileset.py b/clients/client-python/gravitino/api/fileset.py new file mode 100644 index 00000000000..213257931af --- /dev/null +++ b/clients/client-python/gravitino/api/fileset.py @@ -0,0 +1,98 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from enum import Enum +from typing import Optional, Dict + +from gravitino.api.auditable import Auditable + + +class Fileset(Auditable): + """An interface representing a fileset under a schema {@link Namespace}. A fileset is a virtual + concept of the file or directory that is managed by Gravitino. Users can create a fileset object + to manage the non-tabular data on the FS-like storage. The typical use case is to manage the + training data for AI workloads. The major difference compare to the relational table is that the + fileset is schema-free, the main property of the fileset is the storage location of the + underlying data. + + Fileset defines the basic properties of a fileset object. A catalog implementation + with FilesetCatalog should implement this interface. + """ + class Type(Enum): + """An enum representing the type of the fileset object.""" + + MANAGED = "managed" + """Fileset is managed by Gravitino. + When specified, the data will be deleted when the fileset object is deleted""" + + EXTERNAL = "external" + """Fileset is not managed by Gravitino. + When specified, the data will not be deleted when the fileset object is deleted""" + + @abstractmethod + def name(self) -> str: + """ + Returns: + Name of the fileset object. + """ + pass + + @abstractmethod + def comment(self) -> Optional[str]: + """ + Returns: + The comment of the fileset object. Null is returned if no comment is set. + """ + pass + + @abstractmethod + def type(self) -> Type: + """ + @Returns: + The type of the fileset object. + """ + pass + + @abstractmethod + def storage_location(self) -> str: + """Get the storage location of the file or directory path that is managed by this fileset object. + + The returned storageLocation can either be the one specified when creating the fileset + object, or the one specified in the catalog / schema level if the fileset object is created + under this catalog / schema. + + For managed fileset, the storageLocation can be: + + 1) The one specified when creating the fileset object. + + 2) When catalog property "location" is specified but schema property "location" is not + specified, then the storageLocation will be "{catalog location}/schemaName/filesetName". + + 3) When catalog property "location" is not specified but schema property "location" is + specified, then the storageLocation will be "{schema location}/filesetName". + + 4) When both catalog property "location" and schema property "location" are specified, then + the storageLocation will be "{schema location}/filesetName". + + 5) When both catalog property "location" and schema property "location" are not specified, + and storageLocation specified when creating the fileset object is null, this situation is + illegal. + + For external fileset, the storageLocation can be: + + 1) The one specified when creating the fileset object. + + Returns: + The storage location of the fileset object. + """ + pass + + @abstractmethod + def properties(self) -> Dict[str, str]: + """ + Returns: + The properties of the fileset object. Empty map is returned if no properties are set. + """ + pass diff --git a/clients/client-python/gravitino/api/fileset_change.py b/clients/client-python/gravitino/api/fileset_change.py new file mode 100644 index 00000000000..82604ed7ce7 --- /dev/null +++ b/clients/client-python/gravitino/api/fileset_change.py @@ -0,0 +1,253 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC + + +class FilesetChange(ABC): + """A fileset change is a change to a fileset. It can be used to rename a fileset, update the comment + of a fileset, set a property and value pair for a fileset, or remove a property from a fileset. + """ + + @staticmethod + def rename(new_name): + """Creates a new fileset change to rename the fileset. + + Args: + new_name: The new name of the fileset. + + Returns: + The fileset change. + """ + return FilesetChange.RenameFileset(new_name) + + @staticmethod + def update_comment(new_comment): + """Creates a new fileset change to update the fileset comment. + + Args: + new_comment: The new comment for the fileset. + + Returns: + The fileset change. + """ + return FilesetChange.UpdateFilesetComment(new_comment) + + @staticmethod + def set_property(property, value): + """Creates a new fileset change to set the property and value for the fileset. + + Args: + property: The property name to set. + value: The value to set the property to. + + Returns: + The fileset change. + """ + return FilesetChange.SetProperty(property, value) + + @staticmethod + def remove_property(property): + """Creates a new fileset change to remove a property from the fileset. + + Args: + property: The property name to remove. + + Returns: + The fileset change. + """ + return FilesetChange.RemoveProperty(property) + + class RenameFileset: + """A fileset change to rename the fileset.""" + + def __init__(self, new_name): + self.new_name = new_name + + def get_new_name(self): + """Retrieves the new name set for the fileset. + + Returns: + The new name of the fileset. + """ + return self.new_name + + def __eq__(self, other): + """Compares this RenameFileset instance with another object for equality. + Two instances are considered equal if they designate the same new name for the fileset. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents an identical fileset renaming operation; false otherwise. + """ + if not isinstance(other, FilesetChange.RenameFileset): + return False + return self.new_name == other.new_name + + def __hash__(self): + """Generates a hash code for this RenameFileset instance. + The hash code is primarily based on the new name for the fileset. + + Returns: + A hash code value for this renaming operation. + """ + return hash(self.new_name) + + def __str__(self): + """Provides a string representation of the RenameFile instance. + This string includes the class name followed by the new name of the fileset. + + Returns: + A string summary of this renaming operation. + """ + return f"RENAMEFILESET {self.new_name}" + + class UpdateFilesetComment: + """A fileset change to update the fileset comment.""" + + def __init__(self, new_comment): + self.new_comment = new_comment + + def get_new_comment(self): + """Retrieves the new comment intended for the fileset. + + Returns: + The new comment that has been set for the fileset. + """ + return self.new_comment + + def __eq__(self, other): + """Compares this UpdateFilesetComment instance with another object for equality. + Two instances are considered equal if they designate the same new comment for the fileset. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same comment update; false otherwise. + """ + if not isinstance(other, FilesetChange.UpdateFilesetComment): + return False + return self.new_comment == other.new_comment + + def __hash__(self): + """Generates a hash code for this UpdateFileComment instance. + The hash code is based on the new comment for the fileset. + + Returns: + A hash code representing this comment update operation. + """ + return hash(self.new_comment) + + def __str__(self): + """Provides a string representation of the UpdateFilesetComment instance. + This string format includes the class name followed by the new comment for the fileset. + + Returns: + A string summary of this comment update operation. + """ + return f"UPDATEFILESETCOMMENT {self.new_comment}" + + class SetProperty: + """A fileset change to set the property and value for the fileset.""" + + def __init__(self, property, value): + self.property = property + self.value = value + + def get_property(self): + """Retrieves the name of the property being set in the fileset. + + Returns: + The name of the property. + """ + return self.property + + def get_value(self): + """Retrieves the value assigned to the property in the fileset. + + Returns: + The value of the property. + """ + return self.value + + def __eq__(self, other): + """Compares this SetProperty instance with another object for equality. + Two instances are considered equal if they have the same property and value for the fileset. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same property setting; false otherwise. + """ + if not isinstance(other, FilesetChange.SetProperty): + return False + return self.property == other.property and self.value == other.value + + def __hash__(self): + """Generates a hash code for this SetProperty instance. + The hash code is based on both the property name and its assigned value. + + Returns: + A hash code value for this property setting. + """ + return hash((self.property, self.value)) + + def __str__(self): + """Provides a string representation of the SetProperty instance. + This string format includes the class name followed by the property and its value. + + Returns: + A string summary of the property setting. + """ + return f"SETPROPERTY {self.property} {self.value}" + + class RemoveProperty: + """A fileset change to remove a property from the fileset.""" + + def __init__(self, property): + self.property = property + + def get_property(self): + """Retrieves the name of the property to be removed from the fileset. + + Returns: + The name of the property for removal. + """ + return self.property + + def __eq__(self, other): + """Compares this RemoveProperty instance with another object for equality. + Two instances are considered equal if they target the same property for removal from the fileset. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same property removal; false otherwise. + """ + if not isinstance(other, FilesetChange.RemoveProperty): + return False + return self.property == other.property + + def __hash__(self): + """Generates a hash code for this RemoveProperty instance. + The hash code is based on the property name that is to be removed from the fileset. + + Returns: + A hash code value for this property removal operation. + """ + return hash(self.property) + + def __str__(self): + """Provides a string representation of the RemoveProperty instance. + This string format includes the class name followed by the property name to be removed. + + Returns: + A string summary of the property removal operation. + """ + return f"REMOVEPROPERTY {self.property}" diff --git a/clients/client-python/gravitino/meta_change.py b/clients/client-python/gravitino/api/metalake_change.py similarity index 98% rename from clients/client-python/gravitino/meta_change.py rename to clients/client-python/gravitino/api/metalake_change.py index 809c87427fc..db3ad87076e 100644 --- a/clients/client-python/gravitino/meta_change.py +++ b/clients/client-python/gravitino/api/metalake_change.py @@ -30,7 +30,7 @@ def update_comment(new_comment: str) -> 'MetalakeChange.UpdateMetalakeComment': Args: new_comment: The new comment of the metalake. - Return: + Returns: The metalake change. """ return MetalakeChange.UpdateMetalakeComment(new_comment) @@ -43,7 +43,7 @@ def set_property(property: str, value: str) -> 'SetProperty': property: The property name to set. value: The value to set the property to. - Return: + Returns: The metalake change. """ return MetalakeChange.SetProperty(property, value) @@ -55,7 +55,7 @@ def remove_property(property: str) -> 'RemoveProperty': Args: property: The property name to remove. - Return: + Returns: The metalake change. """ return MetalakeChange.RemoveProperty(property) diff --git a/clients/client-python/gravitino/api/schema.py b/clients/client-python/gravitino/api/schema.py new file mode 100644 index 00000000000..7773835b2c7 --- /dev/null +++ b/clients/client-python/gravitino/api/schema.py @@ -0,0 +1,32 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC, abstractmethod +from typing import Optional, Dict + +from gravitino.api.auditable import Auditable + + +class Schema(Auditable, ABC): + """ + An interface representing a schema in the Catalog. A Schema is a + basic container of relational objects, like tables, views, etc. A Schema can be self-nested, + which means it can be schema1.schema2.table. + + This defines the basic properties of a schema. A catalog implementation with SupportsSchemas + should implement this interface. + """ + + @abstractmethod + def name(self) -> str: + """Returns the name of the Schema.""" + pass + + def comment(self) -> Optional[str]: + """Returns the comment of the Schema. None is returned if the comment is not set.""" + return None + + def properties(self) -> Dict[str, str]: + """Returns the properties of the Schema. An empty dictionary is returned if no properties are set.""" + return {} diff --git a/clients/client-python/gravitino/api/schema_change.py b/clients/client-python/gravitino/api/schema_change.py new file mode 100644 index 00000000000..f7cb296e5ae --- /dev/null +++ b/clients/client-python/gravitino/api/schema_change.py @@ -0,0 +1,133 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC + + +class SchemaChange(ABC): + """NamespaceChange class to set the property and value pairs for the namespace.""" + + @staticmethod + def set_property(property, value): + """SchemaChange class to set the property and value pairs for the schema. + + Args: + property: The property name to set. + value: The value to set the property to. + + Returns: + The SchemaChange object. + """ + return SchemaChange.SetProperty(property, value) + + @staticmethod + def remove_property(property): + """SchemaChange class to remove a property from the schema. + + Args: + property: The property name to remove. + + Returns: + The SchemaChange object. + """ + return SchemaChange.RemoveProperty(property) + + class SetProperty: + """SchemaChange class to set the property and value pairs for the schema.""" + def __init__(self, property, value): + self.property = property + self.value = value + + def get_property(self): + """Retrieves the name of the property to be set. + + Returns: + The name of the property. + """ + return self.property + + def get_value(self): + """Retrieves the value of the property to be set. + + Returns: + The value of the property. + """ + return self.value + + def __eq__(self, other): + """Compares this SetProperty instance with another object for equality. + Two instances are considered equal if they have the same property and value. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same property setting; false otherwise. + """ + if not isinstance(other, SchemaChange.SetProperty): + return False + return self.property == other.property and self.value == other.value + + def __hash__(self): + """Generates a hash code for this SetProperty instance. + The hash code is based on both the property name and its value. + + Returns: + A hash code value for this property setting. + """ + return hash((self.property, self.value)) + + def __str__(self): + """Provides a string representation of the SetProperty instance. + This string format includes the class name followed by the property name and its value. + + Returns: + A string summary of the property setting. + """ + return f"SETPROPERTY {self.property} {self.value}" + + class RemoveProperty: + """SchemaChange class to remove a property from the schema.""" + def __init__(self, property): + self.property = property + + def get_property(self): + """Retrieves the name of the property to be removed. + + Returns: + The name of the property for removal. + """ + return self.property + + def __eq__(self, other): + """Compares this RemoveProperty instance with another object for equality. + Two instances are considered equal if they target the same property for removal. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the same property removal; false otherwise. + """ + if not isinstance(other, SchemaChange.RemoveProperty): + return False + return self.property == other.property + + def __hash__(self): + """Generates a hash code for this RemoveProperty instance. + This hash code is based on the property name that is to be removed. + + Returns: + A hash code value for this property removal operation. + """ + return hash(self.property) + + def __str__(self): + """Provides a string representation of the RemoveProperty instance. + This string format includes the class name followed by the property name to be removed. + + Returns: + A string summary of the property removal operation. + """ + return f"REMOVEPROPERTY {self.property}" diff --git a/clients/client-python/gravitino/api/supports_schemas.py b/clients/client-python/gravitino/api/supports_schemas.py new file mode 100644 index 00000000000..6f8b1ecc9a1 --- /dev/null +++ b/clients/client-python/gravitino/api/supports_schemas.py @@ -0,0 +1,126 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC, abstractmethod +from typing import List, Dict, Optional + +from gravitino.api.schema import Schema +from gravitino.api.schema_change import SchemaChange +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace + + +class NoSuchSchemaException(Exception): + """Exception raised if the schema does not exist.""" + pass + + +class SupportsSchemas(ABC): + """ + The Catalog interface to support schema operations. If the implemented catalog has schema + semantics, it should implement this interface. + """ + + @abstractmethod + def list_schemas(self, namespace: Namespace) -> List[NameIdentifier]: + """List schemas under a namespace. + + If an entity such as a table, view exists, its parent schemas must also exist and must be + returned by this discovery method. For example, if table a.b.t exists, this method invoked as + list_schemas(a) must return [a.b] in the result array. + + Args: + namespace: The namespace to list. + + Raises: + NoSuchCatalogException: If the catalog does not exist. + + Returns: + A list of schema identifiers under the namespace. + """ + pass + + def schema_exists(self, ident: NameIdentifier) -> bool: + """Check if a schema exists. + + If an entity such as a table, view exists, its parent namespaces must also exist. For + example, if table a.b.t exists, this method invoked as schema_exists(a.b) must return true. + + Args: + ident: The name identifier of the schema. + + Returns: + True if the schema exists, false otherwise. + """ + try: + self.load_schema(ident) + return True + except NoSuchSchemaException: + return False + + @abstractmethod + def create_schema(self, ident: NameIdentifier, comment: Optional[str], properties: Dict[str, str]) -> Schema: + """Create a schema in the catalog. + + Args: + ident: The name identifier of the schema. + comment: The comment of the schema. + properties: The properties of the schema. + + Raises: + NoSuchCatalogException: If the catalog does not exist. + SchemaAlreadyExistsException: If the schema already exists. + + Returns: + The created schema. + """ + pass + + @abstractmethod + def load_schema(self, ident: NameIdentifier) -> Schema: + """Load metadata properties for a schema. + + Args: + ident: The name identifier of the schema. + + Raises: + NoSuchSchemaException: If the schema does not exist (optional). + + Returns: + A schema. + """ + pass + + @abstractmethod + def alter_schema(self, ident: NameIdentifier, changes: List[SchemaChange]) -> Schema: + """Apply the metadata change to a schema in the catalog. + + Args: + ident: The name identifier of the schema. + changes: The metadata changes to apply. + + Raises: + NoSuchSchemaException: If the schema does not exist. + + Returns: + The altered schema. + """ + pass + + @abstractmethod + def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: + """Drop a schema from the catalog. If cascade option is true, recursively + drop all objects within the schema. + + Args: + ident: The name identifier of the schema. + cascade: If true, recursively drop all objects within the schema. + + Returns: + True if the schema exists and is dropped successfully, false otherwise. + + Raises: + NonEmptySchemaException: If the schema is not empty and cascade is false. + """ + pass diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py b/clients/client-python/gravitino/catalog/base_schema_catalog.py new file mode 100644 index 00000000000..d002c902811 --- /dev/null +++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py @@ -0,0 +1,165 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import logging +from typing import Dict + +from gravitino.api.catalog import Catalog +from gravitino.api.schema_change import SchemaChange +from gravitino.api.supports_schemas import SupportsSchemas +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.catalog_dto import CatalogDTO +from gravitino.dto.requests.schema_create_request import SchemaCreateRequest +from gravitino.dto.requests.schema_update_request import SchemaUpdateRequest +from gravitino.dto.requests.schema_updates_request import SchemaUpdatesRequest +from gravitino.dto.responses.drop_response import DropResponse +from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.dto.responses.schema_response import SchemaResponse +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace +from gravitino.utils import HTTPClient + +logger = logging.getLogger(__name__) + + +class BaseSchemaCatalog(CatalogDTO, SupportsSchemas): + """ + BaseSchemaCatalog is the base abstract class for all the catalog with schema. It provides the + common methods for managing schemas in a catalog. With BaseSchemaCatalog, users can list, + create, load, alter and drop a schema with specified identifier. + """ + + rest_client: HTTPClient + """The REST client to send the requests.""" + + def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, + comment: str = None, properties: Dict[str, str] = None, audit: AuditDTO = None, + rest_client: HTTPClient = None): + super().__init__(_name=name, _type=type, _provider=provider, _comment=comment, _properties=properties, _audit=audit) + self.rest_client = rest_client + + def as_schemas(self): + return self + + def list_schemas(self, namespace: Namespace) -> [NameIdentifier]: + """List all the schemas under the given catalog namespace. + + Args: + namespace: The namespace of the catalog. + + Raises: + NoSuchCatalogException if the catalog with specified namespace does not exist. + + Returns: + A list of {@link NameIdentifier} of the schemas under the given catalog namespace. + """ + Namespace.check_schema(namespace) + resp = self.rest_client.get(BaseSchemaCatalog.format_schema_request_path(namespace)) + entity_list_response = EntityListResponse.from_dict(resp.json()) + entity_list_response.validate() + return entity_list_response.idents + + def create_schema(self, ident: NameIdentifier = None, comment: str = None, properties: Dict[str, str] = None): + """Create a new schema with specified identifier, comment and metadata. + + Args: + ident: The name identifier of the schema. + comment: The comment of the schema. + properties: The properties of the schema. + + Raises: + NoSuchCatalogException if the catalog with specified namespace does not exist. + SchemaAlreadyExistsException if the schema with specified identifier already exists. + + Returns: + The created Schema. + """ + NameIdentifier.check_schema(ident) + req = SchemaCreateRequest(ident.name(), comment, properties) + req.validate() + + resp = self.rest_client.post(BaseSchemaCatalog.format_schema_request_path(ident.namespace()), json=req) + schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) + schema_response.validate() + + return schema_response.schema + + def load_schema(self, ident): + """Load the schema with specified identifier. + + Args: + ident: The name identifier of the schema. + + Raises: + NoSuchSchemaException if the schema with specified identifier does not exist. + + Returns: + The Schema with specified identifier. + """ + NameIdentifier.check_schema(ident) + resp = self.rest_client.get(BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) + schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) + schema_response.validate() + + return schema_response.schema + + def alter_schema(self, ident, *changes): + """Alter the schema with specified identifier by applying the changes. + + Args: + ident: The name identifier of the schema. + changes: The metadata changes to apply. + + Raises: + NoSuchSchemaException if the schema with specified identifier does not exist. + + Returns: + The altered Schema. + """ + NameIdentifier.check_schema(ident) + reqs = [BaseSchemaCatalog.to_schema_update_request(change) for change in changes] + updatesRequest = SchemaUpdatesRequest(reqs) + updatesRequest.validate() + resp = self.rest_client.put(BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) + schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) + schema_response.validate() + return schema_response.schema + + def drop_schema(self, ident, cascade: bool): + """Drop the schema with specified identifier. + + Args: + ident: The name identifier of the schema. + cascade: Whether to drop all the tables under the schema. + + Raises: + NonEmptySchemaException if the schema is not empty and cascade is false. + + Returns: + true if the schema is dropped successfully, false otherwise. + """ + NameIdentifier.check_schema(ident) + try: + params = {"cascade": str(cascade)} + resp = self.rest_client.delete( + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name(), params=params) + drop_resp = DropResponse.from_json(resp.body, infer_missing=True) + drop_resp.validate() + return drop_resp.dropped() + except Exception as e: + logger.warning("Failed to drop schema {}", ident, e) + return False + + @staticmethod + def format_schema_request_path(ns: Namespace): + return "api/metalakes/" + ns.level(0) + "/catalogs/" + ns.level(1) + "/schemas" + + @staticmethod + def to_schema_update_request(change: SchemaChange): + if isinstance(change, SchemaChange.SetProperty): + return SchemaUpdateRequest.SetSchemaPropertyRequest(change.property, change.value) + elif isinstance(change, SchemaChange.RemoveProperty): + return SchemaUpdateRequest.RemoveSchemaPropertyRequest(change.property) + else: + raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py b/clients/client-python/gravitino/catalog/fileset_catalog.py new file mode 100644 index 00000000000..c7f2c730e34 --- /dev/null +++ b/clients/client-python/gravitino/catalog/fileset_catalog.py @@ -0,0 +1,184 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import logging +from typing import List, Dict + +from gravitino.api.catalog import Catalog +from gravitino.api.fileset import Fileset +from gravitino.api.fileset_change import FilesetChange +from gravitino.catalog.base_schema_catalog import BaseSchemaCatalog +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.requests.fileset_create_request import FilesetCreateRequest +from gravitino.dto.requests.fileset_update_request import FilesetUpdateRequest +from gravitino.dto.requests.fileset_updates_request import FilesetUpdatesRequest +from gravitino.dto.responses.drop_response import DropResponse +from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.dto.responses.fileset_response import FilesetResponse +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace +from gravitino.utils import HTTPClient + +logger = logging.getLogger(__name__) + + +class FilesetCatalog(BaseSchemaCatalog): + """Fileset catalog is a catalog implementation that supports fileset like metadata operations, for + example, schemas and filesets list, creation, update and deletion. A Fileset catalog is under the metalake. + """ + + def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, comment: str = None, properties: Dict[str, str] = None, + audit: AuditDTO = None, rest_client: HTTPClient = None): + + super().__init__(name, type, provider, comment, properties, audit, rest_client) + + def as_fileset_catalog(self): + return self + + def list_filesets(self, namespace: Namespace) -> List[NameIdentifier]: + """List the filesets in a schema namespace from the catalog. + + Args: + namespace A schema namespace. + + Raises: + NoSuchSchemaException If the schema does not exist. + + Returns: + An array of fileset identifiers in the namespace. + """ + Namespace.check_fileset(namespace) + + resp = self.rest_client.get( + self.format_fileset_request_path(namespace) + ) + entity_list_resp = EntityListResponse.from_json(resp.body, infer_missing=True) + entity_list_resp.validate() + + return entity_list_resp.idents + + def load_fileset(self, ident) -> Fileset: + """Load fileset metadata by {@link NameIdentifier} from the catalog. + + Args: + ident: A fileset identifier. + + Raises: + NoSuchFilesetException If the fileset does not exist. + + Returns: + The fileset metadata. + """ + NameIdentifier.check_fileset(ident) + + resp = self.rest_client.get(f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}") + fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) + fileset_resp.validate() + + return fileset_resp.fileset + + def create_fileset(self, ident: NameIdentifier, comment: str, type: Catalog.Type, + storage_location: str, properties: Dict[str, str]) -> Fileset: + """Create a fileset metadata in the catalog. + + If the type of the fileset object is "MANAGED", the underlying storageLocation can be null, + and Gravitino will manage the storage location based on the location of the schema. + + If the type of the fileset object is "EXTERNAL", the underlying storageLocation must be set. + + Args: + ident: A fileset identifier. + comment: The comment of the fileset. + type: The type of the fileset. + storage_location: The storage location of the fileset. + properties: The properties of the fileset. + + Raises: + NoSuchSchemaException If the schema does not exist. + FilesetAlreadyExistsException If the fileset already exists. + + Returns: + The created fileset metadata + """ + NameIdentifier.check_fileset(ident) + + req = FilesetCreateRequest(name=ident.name(), comment=comment, type=type, + storage_location=storage_location, properties=properties) + + resp = self.rest_client.post(self.format_fileset_request_path(ident.namespace()), req) + fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) + fileset_resp.validate() + + return fileset_resp.fileset + + def alter_fileset(self, ident, *changes) -> Fileset: + """Update a fileset metadata in the catalog. + + Args: + ident: A fileset identifier. + changes: The changes to apply to the fileset. + + Args: + IllegalArgumentException If the changes are invalid. + NoSuchFilesetException If the fileset does not exist. + + Returns: + The updated fileset metadata. + """ + NameIdentifier.check_fileset(ident) + + updates = [FilesetCatalog.to_fileset_update_request(change) for change in changes] + req = FilesetUpdatesRequest(updates) + req.validate() + + resp = self.rest_client.put(f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", req) + fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) + fileset_resp.validate() + + return fileset_resp.fileset + + def drop_fileset(self, ident: NameIdentifier) -> bool: + """Drop a fileset from the catalog. + + The underlying files will be deleted if this fileset type is managed, otherwise, only the + metadata will be dropped. + + Args: + ident: A fileset identifier. + + Returns: + true If the fileset is dropped, false the fileset did not exist. + """ + try: + NameIdentifier.check_fileset(ident) + + resp = self.rest_client.delete( + f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", + ) + drop_resp = DropResponse.from_json(resp.body, infer_missing=True) + drop_resp.validate() + + return drop_resp.dropped() + except Exception as e: + logger.warning(f"Failed to drop fileset {ident}: {e}") + return False + + @staticmethod + def format_fileset_request_path(namespace: Namespace) -> str: + schema_ns = Namespace.of(namespace.level(0), namespace.level(1)) + return f"{BaseSchemaCatalog.format_schema_request_path(schema_ns)}/{namespace.level(2)}/filesets" + + @staticmethod + def to_fileset_update_request(change: FilesetChange): + if isinstance(change, FilesetChange.RenameFileset): + return FilesetUpdateRequest.RenameFilesetRequest(change.new_name) + elif isinstance(change, FilesetChange.UpdateFilesetComment): + return FilesetUpdateRequest.UpdateFilesetCommentRequest(change.new_comment) + elif isinstance(change, FilesetChange.SetProperty): + return FilesetUpdateRequest.SetFilesetPropertyRequest(change.property, change.value) + elif isinstance(change, FilesetChange.RemoveProperty): + return FilesetUpdateRequest.RemoveFilesetPropertyRequest(change.property) + else: + raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index ba54f32bc34..4c5853209d6 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -13,7 +13,7 @@ from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.metalake_list_response import MetalakeListResponse from gravitino.dto.responses.metalake_response import MetalakeResponse -from gravitino.meta_change import MetalakeChange +from gravitino.api.metalake_change import MetalakeChange from gravitino.name_identifier import NameIdentifier logger = logging.getLogger(__name__) @@ -29,9 +29,9 @@ def __init__(self, uri): # TODO: AuthDataProvider authDataProvider super().__init__(uri) def list_metalakes(self) -> List[GravitinoMetalake]: - """ - Retrieves a list of Metalakes from the Gravitino API. - Return: + """Retrieves a list of Metalakes from the Gravitino API. + + Returns: An array of GravitinoMetalake objects representing the Metalakes. """ resp = self.rest_client.get(self.API_METALAKES_LIST_PATH) @@ -41,19 +41,20 @@ def list_metalakes(self) -> List[GravitinoMetalake]: return [GravitinoMetalake.build(o, self.rest_client) for o in metalake_list_resp.metalakes] def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[str, str]) -> GravitinoMetalake: - """ - Creates a new Metalake using the Gravitino API. + """Creates a new Metalake using the Gravitino API. + Args: ident: The identifier of the new Metalake. comment: The comment for the new Metalake. properties: The properties of the new Metalake. - Return: + + Returns: A GravitinoMetalake instance representing the newly created Metalake. - TODO: @throws MetalakeAlreadyExistsException If a Metalake with the specified identifier already exists. + TODO: @throws MetalakeAlreadyExistsException If a Metalake with the specified identifier already exists. """ NameIdentifier.check_metalake(ident) - req = MetalakeCreateRequest(ident.name, comment, properties) + req = MetalakeCreateRequest(ident.name(), comment, properties) req.validate() resp = self.rest_client.post(self.API_METALAKES_LIST_PATH, req) @@ -63,12 +64,13 @@ def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[ return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> GravitinoMetalake: - """ - Alters a specific Metalake using the Gravitino API. + """Alters a specific Metalake using the Gravitino API. + Args: ident: The identifier of the Metalake to be altered. changes: The changes to be applied to the Metalake. - Return: + + Returns: A GravitinoMetalake instance representing the updated Metalake. TODO: @throws NoSuchMetalakeException If the specified Metalake does not exist. TODO: @throws IllegalArgumentException If the provided changes are invalid or not applicable. @@ -79,29 +81,28 @@ def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> Gra updates_request = MetalakeUpdatesRequest(reqs) updates_request.validate() - resp = self.rest_client.put(self.API_METALAKES_IDENTIFIER_PATH + ident.name, - updates_request) # , MetalakeResponse, {}, ErrorHandlers.metalake_error_handler()) - metalake_response = MetalakeResponse.from_json(resp.body) + resp = self.rest_client.put(self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request) + metalake_response = MetalakeResponse.from_json(resp.body, infer_missing=True) metalake_response.validate() return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) def drop_metalake(self, ident: NameIdentifier) -> bool: - """ - Drops a specific Metalake using the Gravitino API. + """Drops a specific Metalake using the Gravitino API. + Args: ident: The identifier of the Metalake to be dropped. - Return: + + Returns: True if the Metalake was successfully dropped, false otherwise. """ NameIdentifier.check_metalake(ident) try: - resp = self.rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + ident.name) - dropResponse = DropResponse.from_json(resp.body) + resp = self.rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + ident.name()) + dropResponse = DropResponse.from_json(resp.body, infer_missing=True) return dropResponse.dropped() - except Exception as e: - logger.warning(f"Failed to drop metadata ", e) + logger.warning(f"Failed to drop metalake {ident.name()}", e) return False diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py new file mode 100644 index 00000000000..11f764dc2a7 --- /dev/null +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -0,0 +1,76 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from typing import List, Dict + +from gravitino.api.catalog import Catalog +from gravitino.api.catalog_change import CatalogChange +from gravitino.client.gravitino_client_base import GravitinoClientBase +from gravitino.client.gravitino_metalake import GravitinoMetalake +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace + + +class NoSuchMetalakeException(Exception): + pass + + +class NoSuchCatalogException(Exception): + pass + + +class CatalogAlreadyExistsException(Exception): + pass + + +class GravitinoClient(GravitinoClientBase): + """Gravitino Client for an user to interact with the Gravitino API, allowing the client to list, + load, create, and alter Catalog. + + It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. + """ + metalake: GravitinoMetalake + + def __init__(self, uri: str, metalake_name: str): + """Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. + + Args: + uri: The base URI for the Gravitino API. + metalake_name: The specified metalake name. + TODO: authDataProvider: The provider of the data which is used for authentication. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + super().__init__(uri) + self.metalake = super().load_metalake(NameIdentifier.of(metalake_name)) + + def get_metalake(self) -> GravitinoMetalake: + """Get the current metalake object + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + + Returns: + the GravitinoMetalake object + """ + return self.metalake + + def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: + return self.get_metalake().list_catalogs(namespace) + + def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: + return self.get_metalake().list_catalogs_info(namespace) + + def load_catalog(self, ident: NameIdentifier) -> Catalog: + return self.get_metalake().load_catalog(ident) + + def create_catalog(self, ident: NameIdentifier, type: Catalog.Type, provider: str, comment: str, properties: Dict[str, str]) -> Catalog: + return self.get_metalake().create_catalog(ident, type, provider, comment, properties) + + def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange): + return self.get_metalake().alter_catalog(ident, *changes) + + def drop_catalog(self, ident: NameIdentifier): + return self.get_metalake().drop_catalog(ident) diff --git a/clients/client-python/gravitino/client/gravitino_client_base.py b/clients/client-python/gravitino/client/gravitino_client_base.py index a1f640709c8..cba3ed6d643 100644 --- a/clients/client-python/gravitino/client/gravitino_client_base.py +++ b/clients/client-python/gravitino/client/gravitino_client_base.py @@ -8,18 +8,25 @@ from gravitino.client.gravitino_version import GravitinoVersion from gravitino.dto.responses.metalake_response import MetalakeResponse from gravitino.name_identifier import NameIdentifier -from gravitino.utils import HTTPClient +from gravitino.utils import HTTPClient, Response logger = logging.getLogger(__name__) + class GravitinoClientBase: """ Base class for Gravitino Java client; It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. """ - rest_client: HTTPClient # The REST client to communicate with the REST server - API_METALAKES_LIST_PATH = "api/metalakes" # The REST API path for listing metalakes - API_METALAKES_IDENTIFIER_PATH = f"{API_METALAKES_LIST_PATH}/" # The REST API path prefix for load a specific metalake + rest_client: HTTPClient + """The REST client to communicate with the REST server""" + + API_METALAKES_LIST_PATH = "api/metalakes" + """The REST API path for listing metalakes""" + + + API_METALAKES_IDENTIFIER_PATH = f"{API_METALAKES_LIST_PATH}/" + """The REST API path prefix for load a specific metalake""" def __init__(self, uri: str): self.rest_client = HTTPClient(uri) @@ -30,16 +37,17 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: Args: ident The identifier of the Metalake to be loaded. - Return: + Returns: A GravitinoMetalake instance representing the loaded Metalake. Raises: NoSuchMetalakeException If the specified Metalake does not exist. """ + NameIdentifier.check_metalake(ident) - resp = self.rest_client.get(GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name) - metalake_response = MetalakeResponse.from_json(resp.body) + response = self.rest_client.get(GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name()) + metalake_response = MetalakeResponse.from_json(response.body, infer_missing=True) metalake_response.validate() return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) @@ -47,7 +55,7 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: def get_version(self) -> GravitinoVersion: """Retrieves the version of the Gravitino API. - Return: + Returns: A GravitinoVersion instance representing the version of the Gravitino API. """ resp = self.rest_client.get("api/version") diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index aef7e565d21..4d043863e7a 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -2,27 +2,200 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from typing import Dict +import logging +from gravitino.api.catalog import Catalog +from gravitino.api.catalog_change import CatalogChange from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.metalake_dto import MetalakeDTO +from gravitino.dto.requests.catalog_create_request import CatalogCreateRequest +from gravitino.dto.requests.catalog_updates_request import CatalogUpdatesRequest +from gravitino.dto.responses.catalog_list_response import CatalogListResponse +from gravitino.dto.responses.catalog_response import CatalogResponse +from gravitino.dto.responses.drop_response import DropResponse +from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace from gravitino.utils import HTTPClient +from typing import List, Dict + +logger = logging.getLogger(__name__) + + +class NoSuchMetalakeException(Exception): + pass + + +class NoSuchCatalogException(Exception): + pass + + +class CatalogAlreadyExistsException(Exception): + pass + + class GravitinoMetalake(MetalakeDTO): """ Gravitino Metalake is the top-level metadata repository for users. It contains a list of catalogs - as sub-level metadata collections. With {@link GravitinoMetalake}, users can list, create, load, + as sub-level metadata collections. With GravitinoMetalake, users can list, create, load, alter and drop a catalog with specified identifier. """ - restClient: HTTPClient + + rest_client: HTTPClient + + API_METALAKES_CATALOGS_PATH = "api/metalakes/{}/catalogs/{}" def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None, audit: AuditDTO = None, rest_client: HTTPClient = None): super().__init__(name=name, comment=comment, properties=properties, audit=audit) - self.restClient = rest_client + self.rest_client = rest_client @classmethod def build(cls, metalake: MetalakeDTO = None, client: HTTPClient = None): return cls(name=metalake.name, comment=metalake.comment, properties=metalake.properties, audit=metalake.audit, rest_client=client) + + def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: + """List all the catalogs under this metalake with specified namespace. + + Args: + namespace The namespace to list the catalogs under it. + + Raises: + NoSuchMetalakeException if the metalake with specified namespace does not exist. + + Returns: + A list of {@link NameIdentifier} of the catalogs under the specified namespace. + """ + Namespace.check_catalog(namespace) + url = f"api/metalakes/{namespace.level(0)}/catalogs" + response = self.rest_client.get(url) + entityList = EntityListResponse.from_json(response.body, infer_missing=True) + entityList.validate() + return entityList.idents + + def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: + """List all the catalogs with their information under this metalake with specified namespace. + + Args: + namespace The namespace to list the catalogs under it. + + Raises: + NoSuchMetalakeException if the metalake with specified namespace does not exist. + + Returns: + A list of Catalog under the specified namespace. + """ + Namespace.check_catalog(namespace) + params = {"details": "true"} + url = f"api/metalakes/{namespace.level(0)}/catalogs" + response = self.rest_client.get(url, params=params) + catalog_list = CatalogListResponse.from_json(response.body, infer_missing=True) + + return [DTOConverters.to_catalog(catalog, self.rest_client) for catalog in catalog_list.catalogs()] + + def load_catalog(self, ident: NameIdentifier) -> Catalog: + """Load the catalog with specified identifier. + + Args: + ident: The identifier of the catalog to load. + + Raises: + NoSuchCatalogException if the catalog with specified identifier does not exist. + + Returns: + The Catalog with specified identifier. + """ + NameIdentifier.check_catalog(ident) + url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + response = self.rest_client.get(url) + catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) + + return DTOConverters.to_catalog(catalog_resp.catalog(), self.rest_client) + + def create_catalog(self, ident: NameIdentifier, + type: Catalog.Type, + provider: str, + comment: str, + properties: Dict[str, str]) -> Catalog: + """Create a new catalog with specified identifier, type, comment and properties. + + Args: + ident: The identifier of the catalog. + type: The type of the catalog. + provider: The provider of the catalog. + comment: The comment of the catalog. + properties: The properties of the catalog. + + Raises: + NoSuchMetalakeException if the metalake with specified namespace does not exist. + CatalogAlreadyExistsException if the catalog with specified identifier already exists. + + Returns: + The created Catalog. + """ + NameIdentifier.check_catalog(ident) + + catalog_create_request = CatalogCreateRequest(name=ident.name(), + type=type, + provider=provider, + comment=comment, + properties=properties) + catalog_create_request.validate() + + url = f"api/metalakes/{ident.namespace().level(0)}/catalogs" + response = self.rest_client.post(url, json=catalog_create_request) + catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) + + return DTOConverters.to_catalog(catalog_resp.catalog(), self.rest_client) + + def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange) -> Catalog: + """Alter the catalog with specified identifier by applying the changes. + + Args: + ident: the identifier of the catalog. + changes: the changes to apply to the catalog. + + Raises: + NoSuchCatalogException if the catalog with specified identifier does not exist. + IllegalArgumentException if the changes are invalid. + + Returns: + the altered Catalog. + """ + NameIdentifier.check_catalog(ident) + + reqs = [DTOConverters.to_catalog_update_request(change) for change in changes] + updates_request = CatalogUpdatesRequest(reqs) + updates_request.validate() + + url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + response = self.rest_client.put(url, json=updates_request) + catalog_response = CatalogResponse.from_json(response.body, infer_missing=True) + catalog_response.validate() + + return DTOConverters.to_catalog(catalog_response.catalog(), self.rest_client) + + def drop_catalog(self, ident: NameIdentifier) -> bool: + """Drop the catalog with specified identifier. + + Args: + ident the identifier of the catalog. + + Returns: + true if the catalog is dropped successfully, false otherwise. + """ + try: + url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + response = self.rest_client.delete(url) + + drop_response = DropResponse.from_json(response.body, infer_missing=True) + drop_response.validate() + + return drop_response.dropped() + except Exception as e: + logger.warning(f"Failed to drop catalog {ident}: {e}") + return False diff --git a/clients/client-python/gravitino/dto/audit_dto.py b/clients/client-python/gravitino/dto/audit_dto.py index b1d2a8ba94e..0a05903f0fd 100644 --- a/clients/client-python/gravitino/dto/audit_dto.py +++ b/clients/client-python/gravitino/dto/audit_dto.py @@ -7,27 +7,53 @@ from dataclasses_json import DataClassJsonMixin, config +from gravitino.api.audit import Audit + @dataclass -class AuditDTO(DataClassJsonMixin): +class AuditDTO(Audit, DataClassJsonMixin): """Data transfer object representing audit information.""" - creator: Optional[str] + _creator: Optional[str] = field(default=None, metadata=config(field_name='creator')) """The creator of the audit.""" - create_time: Optional[str] = field(metadata=config(field_name='createTime')) # TODO: Can't deserialized datetime from JSON + _create_time: Optional[str] = field(default=None, metadata=config( + field_name='createTime')) # TODO: Can't deserialized datetime from JSON """The create time of the audit.""" - last_modifier: Optional[str] = field(metadata=config(field_name='lastModifier')) + _last_modifier: Optional[str] = field(default=None, metadata=config(field_name='lastModifier')) """The last modifier of the audit.""" - last_modified_time: Optional[str] = field( - metadata=config(field_name='lastModifiedTime')) # TODO: Can't deserialized datetime from JSON + _last_modified_time: Optional[str] = field(default=None, metadata=config( + field_name='lastModifiedTime')) # TODO: Can't deserialized datetime from JSON """The last modified time of the audit.""" - def __init__(self, creator: str = None, create_time: str = None, last_modifier: str = None, - last_modified_time: str = None): - self.creator: str = creator - self.create_time: str = create_time - self.last_modifier: str = last_modifier - self.last_modified_time: str = last_modified_time + def creator(self) -> str: + """The creator of the entity. + + Returns: + the creator of the entity. + """ + return self._creator + + def create_time(self) -> str: + """The creation time of the entity. + + Returns: + The creation time of the entity. + """ + return self._create_time + + def last_modifier(self) -> str: + """ + Returns: + The last modifier of the entity. + """ + return self._last_modifier + + def last_modified_time(self) -> str: + """ + Returns: + The last modified time of the entity. + """ + return self._last_modified_time diff --git a/clients/client-python/gravitino/dto/catalog_dto.py b/clients/client-python/gravitino/dto/catalog_dto.py new file mode 100644 index 00000000000..012c224dfae --- /dev/null +++ b/clients/client-python/gravitino/dto/catalog_dto.py @@ -0,0 +1,51 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from typing import Dict +from dataclasses import dataclass, field + +from dataclasses_json import config + +from .audit_dto import AuditDTO +from ..api.catalog import Catalog + + +@dataclass +class CatalogDTO(Catalog): + """Data transfer object representing catalog information.""" + + _name: str = field(metadata=config(field_name='name')) + _type: Catalog.Type = field(metadata=config(field_name='type')) + _provider: str = field(metadata=config(field_name='provider')) + _comment: str = field(metadata=config(field_name='comment')) + _properties: Dict[str, str] = field(metadata=config(field_name='properties')) + _audit: AuditDTO = field(default=None, metadata=config(field_name='audit')) + + def builder(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, comment: str = None, properties: Dict[str, str] = None, + audit: AuditDTO = None): + self._name = name + self._type = type + self._provider = provider + self._comment = comment + self._properties = properties + self._audit = audit + + def name(self) -> str: + return self._name + + def type(self) -> Catalog.Type: + return self._type + + def provider(self) -> str: + return self._provider + + def comment(self) -> str: + return self._comment + + def properties(self) -> Dict[str, str]: + return self._properties + + def audit_info(self) -> AuditDTO: + return self._audit diff --git a/clients/client-python/gravitino/dto/dto_converters.py b/clients/client-python/gravitino/dto/dto_converters.py index 67c745239e8..c0e66165b2b 100644 --- a/clients/client-python/gravitino/dto/dto_converters.py +++ b/clients/client-python/gravitino/dto/dto_converters.py @@ -2,8 +2,14 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ +from gravitino.api.catalog import Catalog +from gravitino.api.catalog_change import CatalogChange +from gravitino.catalog.fileset_catalog import FilesetCatalog +from gravitino.dto.catalog_dto import CatalogDTO +from gravitino.dto.requests.catalog_update_request import CatalogUpdateRequest from gravitino.dto.requests.metalake_update_request import MetalakeUpdateRequest -from gravitino.meta_change import MetalakeChange +from gravitino.api.metalake_change import MetalakeChange +from gravitino.utils import HTTPClient class DTOConverters: @@ -22,3 +28,29 @@ def to_metalake_update_request(change: MetalakeChange) -> object: return MetalakeUpdateRequest.RemoveMetalakePropertyRequest(change.property) else: raise ValueError(f"Unknown change type: {type(change).__name__}") + + @staticmethod + def to_catalog(catalog: CatalogDTO, client: HTTPClient): + if catalog.type() == Catalog.Type.FILESET: + return FilesetCatalog(name=catalog.name(), + type=catalog.type(), + provider=catalog.provider(), + comment=catalog.comment(), + properties=catalog.properties(), + audit=catalog.audit_info(), + rest_client=client) + else: + raise NotImplementedError("Unsupported catalog type: " + str(catalog.type())) + + @staticmethod + def to_catalog_update_request(change: CatalogChange): + if isinstance(change, CatalogChange.RenameCatalog): + return CatalogUpdateRequest.RenameCatalogRequest(change.new_name) + elif isinstance(change, CatalogChange.UpdateCatalogComment): + return CatalogUpdateRequest.UpdateCatalogCommentRequest(change.new_comment) + elif isinstance(change, CatalogChange.SetProperty): + return CatalogUpdateRequest.SetCatalogPropertyRequest(change.property, change.value) + elif isinstance(change, CatalogChange.RemoveProperty): + return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change.property) + else: + raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/fileset_dto.py b/clients/client-python/gravitino/dto/fileset_dto.py new file mode 100644 index 00000000000..f5540f4efe1 --- /dev/null +++ b/clients/client-python/gravitino/dto/fileset_dto.py @@ -0,0 +1,41 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import Optional, Dict + +from dataclasses_json import config, DataClassJsonMixin + +from gravitino.api.fileset import Fileset +from gravitino.dto.audit_dto import AuditDTO + + +@dataclass +class FilesetDTO(Fileset, DataClassJsonMixin): + """Represents a Fileset DTO (Data Transfer Object).""" + + _name: str = field(metadata=config(field_name='name')) + _comment: Optional[str] = field(metadata=config(field_name='comment')) + _type: Fileset.Type = field(metadata=config(field_name='type')) + _properties: Dict[str, str] = field(metadata=config(field_name='properties')) + _storage_location: str = field(default=None, metadata=config(field_name='storageLocation')) + _audit: AuditDTO = field(default=None) + + def name(self) -> str: + return self._name + + def type(self) -> Fileset.Type: + return self._type + + def storage_location(self) -> str: + return self._storage_location + + def comment(self) -> Optional[str]: + return self._comment + + def properties(self) -> Dict[str, str]: + return self._properties + + def audit_info(self) -> AuditDTO: + return self._audit diff --git a/clients/client-python/gravitino/dto/metalake_dto.py b/clients/client-python/gravitino/dto/metalake_dto.py index e8c00b28348..627d772ca68 100644 --- a/clients/client-python/gravitino/dto/metalake_dto.py +++ b/clients/client-python/gravitino/dto/metalake_dto.py @@ -26,13 +26,6 @@ class MetalakeDTO(DataClassJsonMixin): audit: AuditDTO = None """The audit information of the Metalake DTO.""" - def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None, - audit: AuditDTO = None): - self.name = name - self.comment = comment - self.properties = properties - self.audit = audit - def equals(self, other): if self == other: return True diff --git a/clients/client-python/gravitino/dto/requests/catalog_create_request.py b/clients/client-python/gravitino/dto/requests/catalog_create_request.py new file mode 100644 index 00000000000..78b00ef0f43 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py @@ -0,0 +1,38 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass +from typing import Optional, Dict + +from dataclasses_json import DataClassJsonMixin + +from gravitino.api.catalog import Catalog + + +@dataclass +class CatalogCreateRequest(DataClassJsonMixin): + """Represents a request to create a catalog.""" + name: str + type: Catalog.Type + provider: str + comment: Optional[str] + properties: Optional[Dict[str, str]] + + def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, + comment: str = None, properties: Dict[str, str] = None): + self.name = name + self.type = type + self.provider = provider + self.comment = comment + self.properties = properties + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if name or type are not set. + """ + assert self.name is not None, "\"name\" field is required and cannot be empty" + assert self.type is not None, "\"type\" field is required and cannot be empty" + assert self.provider is not None, "\"provider\" field is required and cannot be empty" diff --git a/clients/client-python/gravitino/dto/requests/catalog_update_request.py b/clients/client-python/gravitino/dto/requests/catalog_update_request.py new file mode 100644 index 00000000000..668c9410f17 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/catalog_update_request.py @@ -0,0 +1,75 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from dataclasses import field, dataclass +from typing import Optional + +from dataclasses_json import config + +from gravitino.api.catalog_change import CatalogChange +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class CatalogUpdateRequestBase(RESTRequest): + type: str = field(metadata=config(field_name='@type')) + + def __init__(self, type: str): + self.type = type + + @abstractmethod + def catalog_change(self): + pass + + +class CatalogUpdateRequest: + """Represents an interface for catalog update requests.""" + + class RenameCatalogRequest(CatalogUpdateRequestBase): + new_name: Optional[str] = field(metadata=config(field_name='newName')) + + def catalog_change(self): + return CatalogChange.rename(self.new_name) + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if the new name is not set. + """ + assert self.new_name is None, '"newName" field is required and cannot be empty' + + class UpdateCatalogCommentRequest(CatalogUpdateRequestBase): + """Request to update the comment of a catalog.""" + + new_comment: Optional[str] = field(metadata=config(field_name='newComment')) + + def catalog_change(self): + return CatalogChange.update_comment(self.new_comment) + + def validate(self): + assert self.new_comment is None, '"newComment" field is required and cannot be empty' + + class SetCatalogPropertyRequest(CatalogUpdateRequestBase): + """Request to set a property on a catalog.""" + property: Optional[str] = None + value: Optional[str] = None + + def catalog_change(self): + return CatalogChange.set_property(self.property, self.value) + + def validate(self): + assert self.property is None, "\"property\" field is required and cannot be empty" + assert self.value is None, "\"value\" field is required and cannot be empty" + + class RemoveCatalogPropertyRequest(CatalogUpdateRequestBase): + """Request to remove a property from a catalog.""" + property: Optional[str] = None + + def catalog_change(self): + return CatalogChange.remove_property(self.property) + + def validate(self): + assert self.property is None, "\"property\" field is required and cannot be empty" diff --git a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py new file mode 100644 index 00000000000..d61c31d5aa7 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py @@ -0,0 +1,31 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass +from typing import Optional, List + +from dataclasses_json import DataClassJsonMixin + +from gravitino.dto.requests.catalog_update_request import CatalogUpdateRequest + + +@dataclass +class CatalogUpdatesRequest(DataClassJsonMixin): + """Represents a request containing multiple catalog updates.""" + updates: Optional[List[CatalogUpdateRequest]] + + def __init__(self, updates: List[CatalogUpdateRequest] = None): + self.updates = updates + + def validate(self): + """Validates each request in the list. + + Raises: + IllegalArgumentException if validation of any request fails. + """ + if self.updates is not None: + for update_request in self.updates: + update_request.validate() + else: + raise ValueError("Updates cannot be null") diff --git a/clients/client-python/gravitino/dto/requests/fileset_create_request.py b/clients/client-python/gravitino/dto/requests/fileset_create_request.py new file mode 100644 index 00000000000..a21ffe78929 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/fileset_create_request.py @@ -0,0 +1,29 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import Optional, Dict + +from dataclasses_json import DataClassJsonMixin, config + +from gravitino.api.fileset import Fileset + + +@dataclass +class FilesetCreateRequest(DataClassJsonMixin): + """Represents a request to create a fileset.""" + name: str + comment: Optional[str] + type: Fileset.Type + storage_location: str = field(metadata=config(field_name='storageLocation')) + properties: Dict[str, str] + + def validate(self): + """Validates the request. + + Raises: + IllegalArgumentException if the request is invalid. + """ + if not self.name: + raise ValueError('"name" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/requests/fileset_update_request.py b/clients/client-python/gravitino/dto/requests/fileset_update_request.py new file mode 100644 index 00000000000..4e294cae40f --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/fileset_update_request.py @@ -0,0 +1,131 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.api.fileset_change import FilesetChange +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class FilesetUpdateRequestBase(RESTRequest): + _type: str = field(metadata=config(field_name='@type')) + + def __init__(self, type: str): + self._type = type + + @abstractmethod + def fileset_change(self): + pass + + +class FilesetUpdateRequest: + """Request to update a fileset.""" + + @dataclass + class RenameFilesetRequest(FilesetUpdateRequestBase): + """The fileset update request for renaming a fileset.""" + + new_name: str = field(metadata=config(field_name='newName')) + """The new name for the Fileset.""" + + def __init__(self, new_name: str): + super().__init__("rename") + self.new_name = new_name + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if the new name is not set. + """ + if not self.new_name: + raise ValueError('"new_name" field is required and cannot be empty') + + def fileset_change(self): + """Returns the fileset change. + + Returns: + the fileset change. + """ + return FilesetChange.rename(self.new_name) + + @dataclass + class UpdateFilesetCommentRequest(FilesetUpdateRequestBase): + """Represents a request to update the comment on a Fileset.""" + + new_comment: str = field(metadata=config(field_name='newComment')) + """The new comment for the Fileset.""" + + def __init__(self, new_comment: str): + super().__init__("updateComment") + self.new_comment = new_comment + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if the new comment is not set. + """ + if not self.new_comment: + raise ValueError('"new_comment" field is required and cannot be empty') + + def fileset_change(self): + """Returns the fileset change""" + return FilesetChange.update_comment(self.new_comment) + + @dataclass + class SetFilesetPropertyRequest(FilesetUpdateRequestBase): + """Represents a request to set a property on a Fileset.""" + + property: str = None + """The property to set.""" + + value: str = None + """The value of the property.""" + + def __init__(self, property: str, value: str): + super().__init__("setProperty") + self.property = property + self.value = value + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if property or value are not set. + """ + if not self.property: + raise ValueError('"property" field is required and cannot be empty') + if not self.value: + raise ValueError('"value" field is required and cannot be empty') + + def fileset_change(self): + return FilesetChange.set_property(self.property, self.value) + + @dataclass + class RemoveFilesetPropertyRequest(FilesetUpdateRequestBase): + """Represents a request to remove a property from a Fileset.""" + + property: str = None + """The property to remove.""" + + def __init__(self, property: str): + super().__init__("removeProperty") + self.property = property + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if property is not set. + """ + if not self.property: + raise ValueError('"property" field is required and cannot be empty') + + def fileset_change(self): + return FilesetChange.remove_property(self.property) diff --git a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py new file mode 100644 index 00000000000..12ba1dadca2 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py @@ -0,0 +1,21 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import Optional, List + +from gravitino.dto.requests.fileset_update_request import FilesetUpdateRequest +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class FilesetUpdatesRequest(RESTRequest): + """Request to represent updates to a fileset.""" + updates: List[FilesetUpdateRequest] = field(default_factory=list) + + def validate(self): + if not self.updates: + raise ValueError("Updates cannot be empty") + for update_request in self.updates: + update_request.validate() \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/requests/metalake_update_request.py b/clients/client-python/gravitino/dto/requests/metalake_update_request.py index d2e3c455b0d..dfa639b6cff 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_update_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_update_request.py @@ -2,43 +2,40 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from abc import abstractmethod, ABC +from abc import abstractmethod from dataclasses import dataclass, field -from dataclasses_json import config, DataClassJsonMixin +from dataclasses_json import config -from gravitino.meta_change import MetalakeChange +from gravitino.api.metalake_change import MetalakeChange +from gravitino.rest.rest_message import RESTRequest @dataclass -class MetalakeUpdateRequestType(DataClassJsonMixin): +class MetalakeUpdateRequestBase(RESTRequest): type: str = field(metadata=config(field_name='@type')) def __init__(self, type: str): self.type = type - -class MetalakeUpdateRequest: - """Represents an interface for Metalake update requests.""" - - @abstractmethod - def validate(self): - pass - @abstractmethod def metalake_change(self): pass + +class MetalakeUpdateRequest: + """Represents an interface for Metalake update requests.""" + @dataclass - class RenameMetalakeRequest(MetalakeUpdateRequestType): + class RenameMetalakeRequest(MetalakeUpdateRequestBase): """Represents a request to rename a Metalake.""" - newName: str = None + new_name: str = field(metadata=config(field_name='newName')) """The new name for the Metalake.""" - def __init__(self, newName: str): + def __init__(self, new_name: str): super().__init__("rename") - self.newName = newName + self.new_name = new_name def validate(self): """Validates the fields of the request. @@ -46,22 +43,22 @@ def validate(self): Raises: IllegalArgumentException if the new name is not set. """ - if not self.newName: + if not self.new_name: raise ValueError('"newName" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.rename(self.newName) + return MetalakeChange.rename(self.new_name) @dataclass - class UpdateMetalakeCommentRequest(MetalakeUpdateRequestType): + class UpdateMetalakeCommentRequest(MetalakeUpdateRequestBase): """Represents a request to update the comment on a Metalake.""" - newComment: str = None + new_comment: str = field(metadata=config(field_name='newComment')) """The new comment for the Metalake.""" - def __init__(self, newComment: str): + def __init__(self, new_comment: str): super().__init__("updateComment") - self.newComment = newComment + self.new_comment = new_comment def validate(self): """Validates the fields of the request. @@ -69,14 +66,14 @@ def validate(self): Raises: IllegalArgumentException if the new comment is not set. """ - if not self.newComment: + if not self.new_comment: raise ValueError('"newComment" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.update_comment(self.newComment) + return MetalakeChange.update_comment(self.new_comment) @dataclass - class SetMetalakePropertyRequest(MetalakeUpdateRequestType): + class SetMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to set a property on a Metalake.""" property: str = None @@ -105,7 +102,7 @@ def metalake_change(self): return MetalakeChange.set_property(self.property, self.value) @dataclass - class RemoveMetalakePropertyRequest(MetalakeUpdateRequestType): + class RemoveMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to remove a property from a Metalake.""" property: str = None diff --git a/clients/client-python/gravitino/dto/requests/schema_create_request.py b/clients/client-python/gravitino/dto/requests/schema_create_request.py new file mode 100644 index 00000000000..d77c668ab94 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/schema_create_request.py @@ -0,0 +1,20 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass +from typing import Optional, Dict + +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class SchemaCreateRequest(RESTRequest): + """Represents a request to create a schema.""" + + name: str + comment: Optional[str] + properties: Optional[Dict[str, str]] + + def validate(self): + assert self.name is not None, "\"name\" field is required and cannot be empty" diff --git a/clients/client-python/gravitino/dto/requests/schema_update_request.py b/clients/client-python/gravitino/dto/requests/schema_update_request.py new file mode 100644 index 00000000000..b11c7929273 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/schema_update_request.py @@ -0,0 +1,79 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.api.schema_change import SchemaChange +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class SchemaUpdateRequestBase(RESTRequest): + type: str = field(metadata=config(field_name='@type')) + + def __init__(self, type: str): + self.type = type + + @abstractmethod + def schema_change(self): + pass + +@dataclass +class SchemaUpdateRequest: + """Represents an interface for Schema update requests.""" + + @dataclass + class SetSchemaPropertyRequest(SchemaUpdateRequestBase): + """Represents a request to set a property on a Schema.""" + + property: str = None + """The property to set.""" + + value: str = None + """The value of the property.""" + + def __init__(self, property: str, value: str): + super().__init__("setProperty") + self.property = property + self.value = value + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if property or value are not set. + """ + if not self.property: + raise ValueError('"property" field is required and cannot be empty') + if not self.value: + raise ValueError('"value" field is required and cannot be empty') + + def schema_change(self): + return SchemaChange.set_property(self.property, self.value) + + @dataclass + class RemoveSchemaPropertyRequest(SchemaUpdateRequestBase): + """Represents a request to remove a property from a Schema.""" + + property: str = None + """The property to remove.""" + + def __init__(self, property: str): + super().__init__("removeProperty") + self.property = property + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if property is not set. + """ + if not self.property: + raise ValueError('"property" field is required and cannot be empty') + + def schema_change(self): + return SchemaChange.remove_property(self.property) diff --git a/clients/client-python/gravitino/dto/requests/schema_updates_request.py b/clients/client-python/gravitino/dto/requests/schema_updates_request.py new file mode 100644 index 00000000000..084a76753a4 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/schema_updates_request.py @@ -0,0 +1,27 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import Optional, List + +from dataclasses_json import DataClassJsonMixin + +from gravitino.dto.requests.schema_update_request import SchemaUpdateRequest + + +@dataclass +class SchemaUpdatesRequest(DataClassJsonMixin): + """Represents a request to update a schema.""" + updates: Optional[List[SchemaUpdateRequest]] = field(default_factory=list) + + def validate(self): + """Validates the request. + + Raises: + IllegalArgumentException If the request is invalid, this exception is thrown. + """ + if not self.updates: + raise ValueError("Updates cannot be empty") + for update_request in self.updates: + update_request.validate() \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/responses/base_response.py b/clients/client-python/gravitino/dto/responses/base_response.py index dca8f36b8d3..a2eae0e398b 100644 --- a/clients/client-python/gravitino/dto/responses/base_response.py +++ b/clients/client-python/gravitino/dto/responses/base_response.py @@ -4,19 +4,15 @@ """ from dataclasses import dataclass -from dataclasses_json import DataClassJsonMixin +from gravitino.rest.rest_message import RESTResponse @dataclass -class BaseResponse(DataClassJsonMixin): +class BaseResponse(RESTResponse): """Represents a base response for REST API calls.""" code: int - @classmethod - def default(cls): - return cls(code=0) - def validate(self): """Validates the response code. TODO: @throws IllegalArgumentException if code value is negative. diff --git a/clients/client-python/gravitino/dto/responses/catalog_list_response.py b/clients/client-python/gravitino/dto/responses/catalog_list_response.py new file mode 100644 index 00000000000..78b637bc8f3 --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/catalog_list_response.py @@ -0,0 +1,24 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import List + +from dataclasses_json import config + +from .base_response import BaseResponse +from ..catalog_dto import CatalogDTO + + +@dataclass +class CatalogListResponse(BaseResponse): + """Represents a response for a list of catalogs with their information.""" + _catalogs: List[CatalogDTO] = field(metadata=config(field_name='catalogs')) + + def __init__(self, catalogs: List[CatalogDTO]): + super().__init__(0) + self._catalogs = catalogs + + def catalogs(self) -> List[CatalogDTO]: + return self._catalogs diff --git a/clients/client-python/gravitino/dto/responses/catalog_response.py b/clients/client-python/gravitino/dto/responses/catalog_response.py new file mode 100644 index 00000000000..79c286747c3 --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/catalog_response.py @@ -0,0 +1,32 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field + +from dataclasses_json import config + +from .base_response import BaseResponse +from ..catalog_dto import CatalogDTO + + +@dataclass +class CatalogResponse(BaseResponse): + """Represents a response containing catalog information.""" + _catalog: CatalogDTO = field(metadata=config(field_name='catalog')) + + def validate(self): + """Validates the response data. + + Raises: + IllegalArgumentException if the catalog name, type or audit is not set. + """ + super().validate() + + assert self.catalog is not None, "catalog must not be null" + assert self.catalog.name() is not None, "catalog 'name' must not be null and empty" + assert self.catalog.type() is not None, "catalog 'type' must not be null" + assert self.catalog.audit_info() is not None, "catalog 'audit' must not be null" + + def catalog(self) -> CatalogDTO: + return self._catalog diff --git a/clients/client-python/gravitino/dto/responses/drop_response.py b/clients/client-python/gravitino/dto/responses/drop_response.py index de1a6908bfb..fb2c548b2f0 100644 --- a/clients/client-python/gravitino/dto/responses/drop_response.py +++ b/clients/client-python/gravitino/dto/responses/drop_response.py @@ -2,13 +2,18 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ +from dataclasses import dataclass, field + +from dataclasses_json import config + from gravitino.dto.responses.base_response import BaseResponse +@dataclass class DropResponse(BaseResponse): """Represents a response for a drop operation.""" - dropped : bool + _dropped: bool = field(metadata=config(field_name='dropped')) def dropped(self) -> bool: - return self.dropped + return self._dropped diff --git a/clients/client-python/gravitino/dto/responses/entity_list_response.py b/clients/client-python/gravitino/dto/responses/entity_list_response.py new file mode 100644 index 00000000000..489ab1568a3 --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/entity_list_response.py @@ -0,0 +1,27 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass, field +from typing import Optional, List + +from dataclasses_json import config + +from gravitino.dto.responses.base_response import BaseResponse +from gravitino.name_identifier import NameIdentifier + + +@dataclass +class EntityListResponse(BaseResponse): + """Represents a response containing a list of catalogs.""" + idents: Optional[List[NameIdentifier]] = field(metadata=config(field_name='identifiers')) + + def validate(self): + """Validates the response data. + + Raises: + IllegalArgumentException if catalog identifiers are not set. + """ + super().validate() + + assert self.idents is not None, "identifiers must not be null" \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/responses/fileset_response.py b/clients/client-python/gravitino/dto/responses/fileset_response.py new file mode 100644 index 00000000000..c925ffbe5a8 --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/fileset_response.py @@ -0,0 +1,26 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass + +from gravitino.dto.fileset_dto import FilesetDTO +from gravitino.dto.responses.base_response import BaseResponse + + +@dataclass +class FilesetResponse(BaseResponse): + """Response for fileset creation.""" + fileset: FilesetDTO + + def validate(self): + """Validates the response data. + + Raises: + IllegalArgumentException if catalog identifiers are not set. + """ + super().validate() + assert self.fileset is not None, "fileset must not be null" + assert self.fileset.name, "fileset 'name' must not be null and empty" + assert self.fileset.storage_location, "fileset 'storageLocation' must not be null and empty" + assert self.fileset.type is not None, "fileset 'type' must not be null and empty" diff --git a/clients/client-python/gravitino/dto/responses/metalake_list_response.py b/clients/client-python/gravitino/dto/responses/metalake_list_response.py index a7bfcddaa4c..afa01520713 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_list_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_list_response.py @@ -16,6 +16,11 @@ class MetalakeListResponse(BaseResponse): metalakes: List[MetalakeDTO] def validate(self): + """Validates the response data. + + Raises: + IllegalArgumentException if catalog identifiers are not set. + """ super().validate() if self.metalakes is None: diff --git a/clients/client-python/gravitino/dto/responses/schema_response.py b/clients/client-python/gravitino/dto/responses/schema_response.py new file mode 100644 index 00000000000..f825c39bcce --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/schema_response.py @@ -0,0 +1,29 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass +from typing import Optional + +from dataclasses_json import DataClassJsonMixin + +from gravitino.dto.responses.base_response import BaseResponse +from gravitino.dto.schema_dto import SchemaDTO + + +@dataclass +class SchemaResponse(BaseResponse, DataClassJsonMixin): + """Represents a response for a schema.""" + schema: Optional[SchemaDTO] + + def validate(self): + """Validates the response data. + + Raises: + IllegalArgumentException if catalog identifiers are not set. + """ + super().validate() + + assert self.schema is not None, "schema must be non-null" + assert self.schema.name is not None, "schema 'name' must not be null and empty" + assert self.schema.audit is not None, "schema 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/schema_dto.py b/clients/client-python/gravitino/dto/schema_dto.py new file mode 100644 index 00000000000..f1638153a1c --- /dev/null +++ b/clients/client-python/gravitino/dto/schema_dto.py @@ -0,0 +1,34 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass +from typing import Optional, Dict + +from dataclasses_json import DataClassJsonMixin + +from gravitino.dto.audit_dto import AuditDTO + + +@dataclass +class SchemaDTO(DataClassJsonMixin): + """Represents a Schema DTO (Data Transfer Object).""" + + name: str + """The name of the Metalake DTO.""" + + comment: Optional[str] + """The comment of the Metalake DTO.""" + + properties: Optional[Dict[str, str]] = None + """The properties of the Metalake DTO.""" + + audit: AuditDTO = None + """The audit information of the Metalake DTO.""" + + def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None, + audit: AuditDTO = None): + self.name = name + self.comment = comment + self.properties = properties + self.audit = audit diff --git a/clients/client-python/gravitino/name_identifier.py b/clients/client-python/gravitino/name_identifier.py index 8c0c93b6062..d40ebd2cd67 100644 --- a/clients/client-python/gravitino/name_identifier.py +++ b/clients/client-python/gravitino/name_identifier.py @@ -13,62 +13,73 @@ class NameIdentifier: schema. """ - DOT = '.' + DOT: str = '.' + + _namespace: Namespace = None + _name: str = None def __init__(self, namespace: Namespace, name: str): - self.namespace = namespace - self.name = name + self._namespace = namespace + self._name = name - def NameIdentifier(self, namespace, name): - self.check(namespace is not None, "Cannot create a NameIdentifier with null namespace") - self.check(name is not None and name != "", "Cannot create a NameIdentifier with null or empty name") + def namespace(self): + return self._namespace - self.namespace = namespace - self.name = name + def name(self): + return self._name @staticmethod def of(*names: str) -> 'NameIdentifier': - """Create the NameIdentifier with the given {@link Namespace} and name. + """Create the NameIdentifier with the given levels of names. Args: - namespace: The namespace of the identifier - name: The name of the identifier + names The names of the identifier - Return: + Returns: The created NameIdentifier """ + NameIdentifier.check(names is not None, "Cannot create a NameIdentifier with null names") - NameIdentifier.check(len(names) != 0, "Cannot create a NameIdentifier with no names") + NameIdentifier.check(len(names) > 0, "Cannot create a NameIdentifier with no names") - return NameIdentifier(Namespace.of(names[:-1]), names[-1]) + return NameIdentifier(Namespace.of(*names[:-1]), names[-1]) @staticmethod def of_namespace(namespace: Namespace, name: str) -> 'NameIdentifier': - """Create the metalake NameIdentifier with the given name. + """Create the NameIdentifier with the given Namespace and name. Args: - metalake: The metalake name + namespace: The namespace of the identifier + name: The name of the identifier - Return: - The created metalake NameIdentifier + Returns: + The created NameIdentifier """ return NameIdentifier(namespace, name) @staticmethod def of_metalake(metalake: str) -> 'NameIdentifier': - """Create the catalog NameIdentifier with the given metalake and catalog name. + """Create the metalake NameIdentifier with the given name. Args: metalake: The metalake name - catalog: The catalog name - Return: - The created catalog NameIdentifier + Returns: + The created metalake NameIdentifier """ return NameIdentifier.of(metalake) @staticmethod def of_catalog(metalake: str, catalog: str) -> 'NameIdentifier': + """Create the catalog NameIdentifier with the given metalake and catalog name. + + Args: + metalake: The metalake name + catalog: The catalog name + + Returns: + The created catalog NameIdentifier + """ return NameIdentifier.of(metalake, catalog) @staticmethod @@ -80,7 +91,7 @@ def of_schema(metalake: str, catalog: str, schema: str) -> 'NameIdentifier': catalog: The catalog name schema: The schema name - Return: + Returns: The created schema NameIdentifier """ return NameIdentifier.of(metalake, catalog, schema) @@ -95,7 +106,7 @@ def of_table(metalake: str, catalog: str, schema: str, table: str) -> 'NameIdent schema: The schema name table: The table name - Return: + Returns: The created table NameIdentifier """ return NameIdentifier.of(metalake, catalog, schema, table) @@ -110,7 +121,7 @@ def of_fileset(metalake: str, catalog: str, schema: str, fileset: str) -> 'NameI schema: The schema name fileset: The fileset name - Return: + Returns: The created fileset NameIdentifier """ return NameIdentifier.of(metalake, catalog, schema, fileset) @@ -126,7 +137,7 @@ def of_topic(metalake: str, catalog: str, schema: str, topic: str) -> 'NameIdent schema: The schema name topic: The topic name - Return: + Returns: The created topic NameIdentifier """ return NameIdentifier.of(metalake, catalog, schema, topic) @@ -140,7 +151,7 @@ def check_metalake(ident: 'NameIdentifier') -> None: ident: The metalake NameIdentifier to check. """ NameIdentifier.check(ident is not None, "Metalake identifier must not be null") - Namespace.check_metalake(ident.namespace) + Namespace.check_metalake(ident.namespace()) @staticmethod def check_catalog(ident: 'NameIdentifier') -> None: @@ -150,8 +161,8 @@ def check_catalog(ident: 'NameIdentifier') -> None: Args: ident: The catalog NameIdentifier to check. """ - NameIdentifier.check(ident is None, "Catalog identifier must not be null") - Namespace.check_catalog(ident.namespace) + NameIdentifier.check(ident is not None, "Catalog identifier must not be null") + Namespace.check_catalog(ident.namespace()) @staticmethod def check_schema(ident: 'NameIdentifier') -> None: @@ -162,7 +173,7 @@ def check_schema(ident: 'NameIdentifier') -> None: ident: The schema NameIdentifier to check. """ NameIdentifier.check(ident is not None, "Schema identifier must not be null") - Namespace.check_schema(ident.namespace) + Namespace.check_schema(ident.namespace()) @staticmethod def check_table(ident: 'NameIdentifier') -> None: @@ -173,7 +184,7 @@ def check_table(ident: 'NameIdentifier') -> None: ident: The table NameIdentifier to check. """ NameIdentifier.check(ident is not None, "Table identifier must not be null") - Namespace.check_table(ident.namespace) + Namespace.check_table(ident.namespace()) @staticmethod def check_fileset(ident: 'NameIdentifier') -> None: @@ -184,7 +195,7 @@ def check_fileset(ident: 'NameIdentifier') -> None: ident: The fileset NameIdentifier to check. """ NameIdentifier.check(ident is not None, "Fileset identifier must not be null") - Namespace.check_fileset(ident.namespace) + Namespace.check_fileset(ident.namespace()) @staticmethod def check_topic(ident: 'NameIdentifier') -> None: @@ -195,7 +206,7 @@ def check_topic(ident: 'NameIdentifier') -> None: ident: The topic NameIdentifier to check. """ NameIdentifier.check(ident is not None, "Topic identifier must not be null") - Namespace.check_topic(ident.namespace) + Namespace.check_topic(ident.namespace()) @staticmethod def parse(identifier: str) -> 'NameIdentifier': @@ -204,7 +215,7 @@ def parse(identifier: str) -> 'NameIdentifier': Args: identifier: The identifier string - Return: + Returns: The created NameIdentifier """ NameIdentifier.check(identifier is not None and identifier != '', "Cannot parse a null or empty identifier") @@ -215,15 +226,15 @@ def parse(identifier: str) -> 'NameIdentifier': def has_namespace(self): """Check if the NameIdentifier has a namespace. - Return: + Returns: True if the NameIdentifier has a namespace, false otherwise. """ - return not self.namespace.is_empty() + return not self.namespace().is_empty() def get_namespace(self): """Get the namespace of the NameIdentifier. - Return: + Returns: The namespace of the NameIdentifier. """ return self.namespace @@ -231,7 +242,7 @@ def get_namespace(self): def get_name(self): """Get the name of the NameIdentifier. - Return: + Returns: The name of the NameIdentifier. """ return self.name diff --git a/clients/client-python/gravitino/namespace.py b/clients/client-python/gravitino/namespace.py index f3f0396bb16..281439f1cda 100644 --- a/clients/client-python/gravitino/namespace.py +++ b/clients/client-python/gravitino/namespace.py @@ -11,11 +11,12 @@ class Namespace: "metalake1.catalog1.schema1" are all valid namespaces. """ - EMPTY = None - DOT = "." + _DOT: str = "." + + _levels: List[str] = [] def __init__(self, levels: List[str]): - self.levels = levels + self._levels = levels @staticmethod def empty() -> 'Namespace': @@ -36,14 +37,12 @@ def of(*levels: str) -> 'Namespace': Returns: A namespace with the given levels """ - if levels is None: - raise ValueError("Cannot create a namespace with null levels") + Namespace.check(levels is not None, "Cannot create a namespace with null levels") if len(levels) == 0: return Namespace.empty() for level in levels: - if level is None or level == "": - raise ValueError("Cannot create a namespace with null or empty level") + Namespace.check(level is not None and level != "", "Cannot create a namespace with null or empty level") return Namespace(list(levels)) @@ -90,7 +89,7 @@ def of_table(metalake: str, catalog: str, schema: str) -> 'Namespace': catalog: The catalog name schema: The schema name - Return: + Returns: A namespace for table """ return Namespace.of(metalake, catalog, schema) @@ -104,7 +103,7 @@ def of_fileset(metalake: str, catalog: str, schema: str) -> 'Namespace': catalog: The catalog name schema: The schema name - Return: + Returns: A namespace for fileset """ return Namespace.of(metalake, catalog, schema) @@ -118,7 +117,7 @@ def of_topic(metalake: str, catalog: str, schema: str) -> 'Namespace': catalog: The catalog name schema: The schema name - Return: + Returns: A namespace for topic """ return Namespace.of(metalake, catalog, schema) @@ -131,8 +130,8 @@ def check_metalake(namespace: 'Namespace') -> None: Args: namespace: The metalake namespace """ - if not namespace and not namespace.is_empty(): - raise ValueError(f"Metalake namespace must be non-null and empty, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.is_empty(), + f"Metalake namespace must be non-null and empty, the input namespace is {namespace}") @staticmethod def check_catalog(namespace: 'Namespace') -> None: @@ -142,8 +141,8 @@ def check_catalog(namespace: 'Namespace') -> None: Args: namespace: The catalog namespace """ - if not namespace and namespace.length() != 1: - raise ValueError(f"Catalog namespace must be non-null and have 1 level, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.length() == 1, + f"Catalog namespace must be non-null and have 1 level, the input namespace is {namespace}") @staticmethod def check_schema(namespace: 'Namespace') -> None: @@ -153,8 +152,8 @@ def check_schema(namespace: 'Namespace') -> None: Args: namespace: The schema namespace """ - if not namespace and namespace.length() != 2: - raise ValueError(f"Schema namespace must be non-null and have 2 levels, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.length() == 2, + f"Schema namespace must be non-null and have 2 levels, the input namespace is {namespace}") @staticmethod def check_table(namespace: 'Namespace') -> None: @@ -164,8 +163,8 @@ def check_table(namespace: 'Namespace') -> None: Args: namespace: The table namespace """ - if not namespace and namespace.length() != 3: - raise ValueError(f"Table namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.length() == 3, + f"Table namespace must be non-null and have 3 levels, the input namespace is {namespace}") @staticmethod def check_fileset(namespace: 'Namespace') -> None: @@ -175,9 +174,8 @@ def check_fileset(namespace: 'Namespace') -> None: Args: namespace: The fileset namespace """ - if not namespace and namespace.length() != 3: - raise ValueError( - f"Fileset namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.length() == 3, + f"Fileset namespace must be non-null and have 3 levels, the input namespace is {namespace}") @staticmethod def check_topic(namespace: 'Namespace') -> None: @@ -187,16 +185,16 @@ def check_topic(namespace: 'Namespace') -> None: Args: namespace: The topic namespace """ - if not namespace and namespace.length() != 3: - raise ValueError(f"Topic namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check(namespace is not None and namespace.length() == 3, + f"Topic namespace must be non-null and have 3 levels, the input namespace is {namespace}") def levels(self) -> List[str]: """Get the levels of the namespace. - Return: + Returns: The levels of the namespace """ - return self.levels + return self._levels def level(self, pos: int) -> str: """Get the level at the given position. @@ -204,39 +202,39 @@ def level(self, pos: int) -> str: Args: pos: The position of the level - Return: + Returns: The level at the given position """ - if pos < 0 or pos >= len(self.levels): + if pos < 0 or pos >= len(self._levels): raise ValueError("Invalid level position") - return self.levels[pos] + return self._levels[pos] def length(self) -> int: """Get the length of the namespace. - Return: + Returns: The length of the namespace. """ - return len(self.levels) + return len(self._levels) def is_empty(self) -> bool: """Check if the namespace is empty. - Return: + Returns: True if the namespace is empty, false otherwise. """ - return len(self.levels) == 0 + return len(self._levels) == 0 def __eq__(self, other: 'Namespace') -> bool: if not isinstance(other, Namespace): return False - return self.levels == other.levels + return self._levels == other._levels def __hash__(self) -> int: - return hash(tuple(self.levels)) + return hash(tuple(self._levels)) def __str__(self) -> str: - return Namespace.DOT.join(self.levels) + return Namespace._DOT.join(self._levels) @staticmethod def check(expression: bool, message: str, *args) -> None: diff --git a/clients/client-python/gravitino/rest/rest_message.py b/clients/client-python/gravitino/rest/rest_message.py new file mode 100644 index 00000000000..409b3802cff --- /dev/null +++ b/clients/client-python/gravitino/rest/rest_message.py @@ -0,0 +1,42 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import ABC, abstractmethod + +from dataclasses_json import DataClassJsonMixin + + +class RESTMessage(DataClassJsonMixin, ABC): + """ + Interface for REST messages. + + REST messages are objects that are sent to and received from REST endpoints. They are + typically used to represent the request and response bodies of REST API calls. + """ + + @abstractmethod + def validate(self): + """ + Ensures that a constructed instance of a REST message is valid according to the REST spec. + + This is needed when parsing data that comes from external sources and the object might have + been constructed without all the required fields present. + + Raises: + IllegalArgumentException: If the message is not valid. + """ + pass + + +class IllegalArgumentException(Exception): + """Exception raised if a REST message is not valid according to the REST spec.""" + pass + + +class RESTRequest(RESTMessage, ABC): + """Interface to mark a REST request.""" + + +class RESTResponse(RESTMessage, ABC): + """Interface to mark a REST response""" diff --git a/clients/client-python/gravitino/utils/exceptions.py b/clients/client-python/gravitino/utils/exceptions.py index 147a1698362..28afc373452 100644 --- a/clients/client-python/gravitino/utils/exceptions.py +++ b/clients/client-python/gravitino/utils/exceptions.py @@ -17,7 +17,7 @@ def __init__(self, error): def json(self): """ - :return: object of response error from the API + :Returns: object of response error from the API """ try: return json.loads(self.body.decode("utf-8")) diff --git a/clients/client-python/tests/integration/__init__.py b/clients/client-python/tests/integration/__init__.py new file mode 100644 index 00000000000..5779a3ad252 --- /dev/null +++ b/clients/client-python/tests/integration/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" diff --git a/clients/client-python/tests/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py similarity index 95% rename from clients/client-python/tests/integration_test_env.py rename to clients/client-python/tests/integration/integration_test_env.py index 7036b6f0477..73cbaed5e01 100644 --- a/clients/client-python/tests/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -7,6 +7,7 @@ import unittest import subprocess import time + import requests logger = logging.getLogger(__name__) @@ -19,7 +20,7 @@ def get_gravitino_server_version(): response.close() return True except requests.exceptions.RequestException as e: - logger.error("Failed to access the server: {}", e) + logger.warning("Failed to access the server: {}", e) return False @@ -44,8 +45,8 @@ def _init_logging(): logger.addHandler(console_handler) -# Provide real test environment for the Gravitino Server class IntegrationTestEnv(unittest.TestCase): + """Provide real test environment for the Gravitino Server""" gravitino_startup_script = None @classmethod @@ -77,6 +78,8 @@ def setUpClass(cls): logger.error("ERROR: Can't start Gravitino server!") quit(0) + cls.clean_test_date() + @classmethod def tearDownClass(cls): if os.environ.get('GRADLE_START_GRAVITINO') is not None: diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py new file mode 100644 index 00000000000..20d20970392 --- /dev/null +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -0,0 +1,127 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import logging + +from gravitino.api.catalog import Catalog +from gravitino.api.fileset import Fileset +from gravitino.api.fileset_change import FilesetChange +from gravitino.client.gravitino_admin_client import GravitinoAdminClient +from gravitino.client.gravitino_client import GravitinoClient +from gravitino.client.gravitino_metalake import GravitinoMetalake +from gravitino.dto.catalog_dto import CatalogDTO +from gravitino.name_identifier import NameIdentifier +from tests.integration.integration_test_env import IntegrationTestEnv + +logger = logging.getLogger(__name__) + + +class TestFilesetCatalog(IntegrationTestEnv): + catalog: Catalog = None + metalake: GravitinoMetalake = None + metalake_name: str = "testMetalake" + catalog_name: str = "testCatalog" + schema_name: str = "testSchema" + fileset_name: str = "testFileset1" + fileset_alter_name: str = "testFilesetAlter" + provider: str = "hadoop" + + metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) + catalog_ident: NameIdentifier = NameIdentifier.of_catalog(metalake_name, catalog_name) + schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name) + fileset_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, fileset_name) + fileset_alter_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, + fileset_alter_name) + + gravitino_admin_client: GravitinoAdminClient = None + gravitino_client: GravitinoClient = None + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.clean_test_data() + + cls.gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") + cls.metalake = cls.gravitino_admin_client.create_metalake(ident=cls.metalake_ident, + comment="test comment", properties={}) + cls.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=cls.metalake_name) + + cls.catalog = cls.gravitino_client.create_catalog( + ident=cls.catalog_ident, + type=CatalogDTO.Type.FILESET, + provider=cls.provider, + comment="comment", + properties={"k1": "v1"} + ) + + cls.catalog.as_schemas().create_schema(ident=cls.schema_ident, comment="comment", properties={"k1": "v1"}) + + @classmethod + def tearDownClass(cls): + """Clean test data""" + cls.clean_test_data() + super().tearDownClass() + + @classmethod + def clean_test_data(cls): + try: + cls.gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") + gravitino_metalake = cls.gravitino_admin_client.load_metalake(ident=cls.metalake_ident) + cls.catalog = gravitino_metalake.load_catalog(ident=cls.catalog_ident) + cls.catalog.as_fileset_catalog().drop_fileset(ident=cls.fileset_ident) + cls.catalog.as_fileset_catalog().drop_fileset(ident=cls.fileset_alter_ident) + cls.catalog.as_schemas().drop_schema(ident=cls.schema_ident, cascade=True) + gravitino_metalake.drop_catalog(ident=cls.catalog_ident) + cls.gravitino_admin_client.drop_metalake(cls.metalake_ident) + except Exception as e: + logger.debug(e) + + def create_catalog(self): + self.catalog = self.gravitino_client.create_catalog( + ident=self.catalog_ident, + type=CatalogDTO.Type.FILESET, + provider=self.provider, + comment="comment", + properties={"k1": "v1"}) + + assert self.catalog.name == self.catalog_name + assert self.catalog.type == CatalogDTO.Type.FILESET + assert self.catalog.provider == self.provider + + def create_schema(self): + self.catalog.as_schemas().create_schema( + ident=self.schema_ident, + comment="comment", + properties={"k1": "v1"}) + + def test_create_fileset(self): + fileset = self.catalog.as_fileset_catalog().create_fileset(ident=self.fileset_ident, + type=Fileset.Type.MANAGED, + comment="mock comment", + storage_location="mock location", + properties={"k1": "v1"}) + assert fileset is not None + + fileset_list = self.catalog.as_fileset_catalog().list_filesets(self.fileset_ident.namespace()) + assert fileset_list is not None and len(fileset_list) == 1 + + fileset = self.catalog.as_fileset_catalog().load_fileset(self.fileset_ident) + assert fileset is not None + assert fileset.name() == self.fileset_ident.name() + + # Alter fileset + changes = ( + FilesetChange.rename(self.fileset_alter_name), + FilesetChange.update_comment("new fileset comment"), + FilesetChange.set_property("key1", "value1"), + FilesetChange.remove_property("k1"), + ) + fileset_alter = self.catalog.as_fileset_catalog().alter_fileset(self.fileset_ident, *changes) + assert fileset_alter is not None + assert fileset_alter.name() == self.fileset_alter_name + assert fileset_alter.comment() == "new fileset comment" + assert fileset_alter.properties().get("key1") == "value1" + + # Clean test data + self.catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident) diff --git a/clients/client-python/tests/test_gravitino_admin_client.py b/clients/client-python/tests/integration/test_gravitino_admin_client.py similarity index 83% rename from clients/client-python/tests/test_gravitino_admin_client.py rename to clients/client-python/tests/integration/test_gravitino_admin_client.py index 0490e16ab55..e54f4a1ac13 100644 --- a/clients/client-python/tests/test_gravitino_admin_client.py +++ b/clients/client-python/tests/integration/test_gravitino_admin_client.py @@ -2,34 +2,38 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ +import logging + import gravitino from gravitino.client.gravitino_admin_client import GravitinoAdminClient from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest from gravitino.dto.responses.metalake_response import MetalakeResponse -from gravitino.meta_change import MetalakeChange +from gravitino.api.metalake_change import MetalakeChange from gravitino.name_identifier import NameIdentifier from gravitino.utils.exceptions import NotFoundError -from tests.integration_test_env import IntegrationTestEnv +from tests.integration.integration_test_env import IntegrationTestEnv + +logger = logging.getLogger(__name__) -class TestGravitinoClient(IntegrationTestEnv): +class TestGravitinoAdminClient(IntegrationTestEnv): def setUp(self): self._gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") def test_create_metalake(self): metalake_name = "metalake00" - try: - self.create_metalake(metalake_name) - except gravitino.utils.exceptions.HTTPError: - self.drop_metalake(metalake_name) # Clean test data self.drop_metalake(metalake_name) + self.create_metalake(metalake_name) + # Clean test data + self.drop_metalake(metalake_name) + def create_metalake(self, metalake_name): - comment = "This is a sample comment" ident = NameIdentifier.of(metalake_name) + comment = "This is a sample comment" properties = {"key1": "value1", "key2": "value2"} gravitinoMetalake = self._gravitino_admin_client.create_metalake(ident, comment, properties) @@ -37,16 +41,17 @@ def create_metalake(self, metalake_name): self.assertEqual(gravitinoMetalake.name, metalake_name) self.assertEqual(gravitinoMetalake.comment, comment) self.assertEqual(gravitinoMetalake.properties.get("key1"), "value1") - self.assertEqual(gravitinoMetalake.audit.creator, "anonymous") + self.assertEqual(gravitinoMetalake.audit._creator, "anonymous") def test_alter_metalake(self): metalake_name = "metalake02" metalake_new_name = metalake_name + "_new" - try: - self.create_metalake(metalake_name) - except gravitino.utils.exceptions.HTTPError: - self.drop_metalake(metalake_name) + # Clean test data + self.drop_metalake(metalake_name) + self.drop_metalake(metalake_new_name) + + self.create_metalake(metalake_name) changes = ( MetalakeChange.rename(metalake_new_name), MetalakeChange.update_comment("new metalake comment"), @@ -55,7 +60,7 @@ def test_alter_metalake(self): metalake = self._gravitino_admin_client.alter_metalake(NameIdentifier.of(metalake_name), *changes) self.assertEqual(metalake_new_name, metalake.name) self.assertEqual("new metalake comment", metalake.comment) - self.assertEqual("anonymous", metalake.audit.creator) # Assuming a constant or similar attribute + self.assertEqual("anonymous", metalake.audit._creator) # Assuming a constant or similar attribute # Reload metadata via new name to check if the changes are applied new_metalake = self._gravitino_admin_client.load_metalake(NameIdentifier.of(metalake_new_name)) @@ -67,13 +72,9 @@ def test_alter_metalake(self): with self.assertRaises(NotFoundError): # TODO: NoSuchMetalakeException self._gravitino_admin_client.load_metalake(old) - # Clean test data - self.drop_metalake(metalake_name) - self.drop_metalake(metalake_new_name) - - def drop_metalake(self, metalake_name): + def drop_metalake(self, metalake_name) -> bool: ident = NameIdentifier.of(metalake_name) - self.assertTrue(self._gravitino_admin_client.drop_metalake(ident)) + return self._gravitino_admin_client.drop_metalake(ident) def test_drop_metalake(self): metalake_name = "metalake03" @@ -82,7 +83,7 @@ def test_drop_metalake(self): except gravitino.utils.exceptions.HTTPError: self.drop_metalake(metalake_name) - self.drop_metalake(metalake_name) + assert self.drop_metalake(metalake_name) == True def test_metalake_update_request_to_json(self): changes = ( @@ -103,7 +104,7 @@ def test_from_json_metalake_response(self): self.assertEqual(metalake_response.code, 0) self.assertIsNotNone(metalake_response.metalake) self.assertEqual(metalake_response.metalake.name, "example_name18") - self.assertEqual(metalake_response.metalake.audit.creator, "anonymous") + self.assertEqual(metalake_response.metalake.audit._creator, "anonymous") def test_list_metalakes(self): metalake_name = "metalake05" @@ -113,4 +114,3 @@ def test_list_metalakes(self): # Clean test data self.drop_metalake(metalake_name) - From 1ba77e6fead1bd60f9d78edcd01eb6016a1627d0 Mon Sep 17 00:00:00 2001 From: Kang Date: Sun, 14 Apr 2024 09:26:11 +0800 Subject: [PATCH 016/106] [MINOR] improvement(catalog-doris): Upgrade Doris CI image version to 0.1.3 (#2922) ### What changes were proposed in this pull request? Change the Doris CI image version from 0.1.2 to 0.1.3 ### Why are the changes needed? 0.1.3 remove log from container stdout, which is better for CI framework ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- catalogs/catalog-jdbc-doris/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index a19523406fa..bd78ec34822 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -79,7 +79,7 @@ tasks.test { dependsOn(tasks.jar) doFirst { - environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.2") + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit From 3bccc652b46f7b1f3ec4813c979eae33b272482e Mon Sep 17 00:00:00 2001 From: Kang Date: Sun, 14 Apr 2024 09:28:15 +0800 Subject: [PATCH 017/106] [MINOR] docs: correct json body in create_table (#2923) ### What changes were proposed in this pull request? correct json body in create_table ### Why are the changes needed? HTTP request body is illegal, miss a dot in body use this command may get an exception message ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Mannal --- docs/manage-relational-metadata-using-gravitino.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manage-relational-metadata-using-gravitino.md b/docs/manage-relational-metadata-using-gravitino.md index 572d94aa3ec..2e44f98ffc2 100644 --- a/docs/manage-relational-metadata-using-gravitino.md +++ b/docs/manage-relational-metadata-using-gravitino.md @@ -477,7 +477,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ }, { "name": "name", - "type": "varchar(500)" + "type": "varchar(500)", "comment": "name column comment", "nullable": true, "autoIncrement": false, From 9128db0d2f4bc57dbfb410b51db17887e335cba4 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Sun, 14 Apr 2024 12:30:27 +0800 Subject: [PATCH 018/106] [#2227] improvement(jdbc-backend): Improve the judgment of exception information in JDBC backend (#2862) ### What changes were proposed in this pull request? Determine exceptions more accurately based on SQL Exception error codes. ### Why are the changes needed? Fix: #2227 ### How was this patch tested? Add the unit tests. --------- Co-authored-by: xiaojiebao --- .../storage/relational/JDBCBackend.java | 2 + .../converters/H2ExceptionConverter.java | 31 ++ .../converters/MySQLExceptionConverter.java | 32 ++ .../converters/SQLExceptionConverter.java | 24 + .../SQLExceptionConverterFactory.java | 43 ++ .../service/CatalogMetaService.java | 4 +- .../service/FilesetMetaService.java | 4 +- .../service/MetalakeMetaService.java | 4 +- .../relational/service/SchemaMetaService.java | 4 +- .../relational/service/TableMetaService.java | 4 +- .../relational/service/TopicMetaService.java | 4 +- .../relational/utils/ExceptionUtils.java | 16 +- .../storage/relational/TestJDBCBackend.java | 469 ++++++++++++++++++ 13 files changed, 619 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 1a90fa9fcb1..14a0df4fcac 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -21,6 +21,7 @@ import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.storage.relational.converters.SQLExceptionConverterFactory; import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; @@ -44,6 +45,7 @@ public class JDBCBackend implements RelationalBackend { @Override public void initialize(Config config) { SqlSessionFactoryHelper.getInstance().init(config); + SQLExceptionConverterFactory.initConverter(config); } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java new file mode 100644 index 00000000000..b47f8ce208d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.converters; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.sql.SQLException; + +/** + * Exception converter to Gravitino exception for H2. The definition of error codes can be found in + * the document: + */ +public class H2ExceptionConverter implements SQLExceptionConverter { + /** It means found a duplicated primary key or unique key entry in H2. */ + private static final int DUPLICATED_ENTRY_ERROR_CODE = 23505; + + @SuppressWarnings("FormatStringAnnotation") + @Override + public GravitinoRuntimeException toGravitinoException( + SQLException se, Entity.EntityType type, String name) { + switch (se.getErrorCode()) { + case DUPLICATED_ENTRY_ERROR_CODE: + return new AlreadyExistsException(se, se.getMessage()); + default: + return new GravitinoRuntimeException(se, se.getMessage()); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java new file mode 100644 index 00000000000..de8136629cc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.converters; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.sql.SQLException; + +/** + * Exception converter to Gravitino exception for MySQL. The definition of error codes can be found + * in the document: + */ +public class MySQLExceptionConverter implements SQLExceptionConverter { + /** It means found a duplicated primary key or unique key entry in MySQL. */ + private static final int DUPLICATED_ENTRY_ERROR_CODE = 1062; + + @SuppressWarnings("FormatStringAnnotation") + @Override + public GravitinoRuntimeException toGravitinoException( + SQLException se, Entity.EntityType type, String name) { + switch (se.getErrorCode()) { + case DUPLICATED_ENTRY_ERROR_CODE: + return new AlreadyExistsException(se, se.getMessage()); + default: + return new GravitinoRuntimeException(se, se.getMessage()); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java new file mode 100644 index 00000000000..47fe684cf93 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.converters; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.sql.SQLException; + +/** Interface for converter JDBC SQL exceptions to Gravitino exceptions. */ +public interface SQLExceptionConverter { + /** + * Convert JDBC exception to GravitinoException. + * + * @param sqlException The sql exception to map + * @param type The type of the entity + * @param name The name of the entity + * @return A best attempt at a corresponding jdbc connector exception or generic with the + * SQLException as the cause + */ + GravitinoRuntimeException toGravitinoException( + SQLException sqlException, Entity.EntityType type, String name); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java new file mode 100644 index 00000000000..3623b3dc3e6 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.converters; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.google.common.base.Preconditions; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SQLExceptionConverterFactory { + private static final Pattern TYPE_PATTERN = Pattern.compile("jdbc:(\\w+):"); + private static SQLExceptionConverter converter; + + private SQLExceptionConverterFactory() {} + + public static synchronized void initConverter(Config config) { + if (converter == null) { + String jdbcUrl = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL); + Matcher typeMatcher = TYPE_PATTERN.matcher(jdbcUrl); + if (typeMatcher.find()) { + String jdbcType = typeMatcher.group(1); + if (jdbcType.equalsIgnoreCase("mysql")) { + converter = new MySQLExceptionConverter(); + } else if (jdbcType.equalsIgnoreCase("h2")) { + converter = new H2ExceptionConverter(); + } else { + throw new IllegalArgumentException(String.format("Unsupported jdbc type: %s", jdbcType)); + } + } else { + throw new IllegalArgumentException( + String.format("Cannot find jdbc type in jdbc url: %s", jdbcUrl)); + } + } + } + + public static SQLExceptionConverter getConverter() { + Preconditions.checkState(converter != null, "Exception converter is not initialized."); + return converter; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 2da33ea27df..a075dbf8d39 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -112,7 +112,7 @@ public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { } }); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.CATALOG, catalogEntity.nameIdentifier().toString()); throw re; } @@ -147,7 +147,7 @@ public CatalogEntity updateCatalog( POConverters.updateCatalogPOWithVersion(oldCatalogPO, newEntity, metalakeId), oldCatalogPO)); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.CATALOG, newEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java index c30a738c016..eb8b1924ac2 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java @@ -122,7 +122,7 @@ public void insertFileset(FilesetEntity filesetEntity, boolean overwrite) { } })); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.FILESET, filesetEntity.nameIdentifier().toString()); throw re; } @@ -177,7 +177,7 @@ public FilesetEntity updateFileset( mapper -> mapper.updateFilesetMeta(newFilesetPO, oldFilesetPO)); } } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.FILESET, newEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 505762ba285..7c76ab4ebb5 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -90,7 +90,7 @@ public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { } }); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.METALAKE, baseMetalake.nameIdentifier().toString()); throw re; } @@ -125,7 +125,7 @@ public BaseMetalake updateMetalake( MetalakeMetaMapper.class, mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.METALAKE, newMetalakeEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java index fddbd4a6029..c1b8ba490d2 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java @@ -109,7 +109,7 @@ public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) { } }); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.SCHEMA, schemaEntity.nameIdentifier().toString()); throw re; } @@ -142,7 +142,7 @@ public SchemaEntity updateSchema( mapper.updateSchemaMeta( POConverters.updateSchemaPOWithVersion(oldSchemaPO, newEntity), oldSchemaPO)); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.SCHEMA, newEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java index afb857f4e79..fc6a03db757 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java @@ -102,7 +102,7 @@ public void insertTable(TableEntity tableEntity, boolean overwrite) { } }); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.TABLE, tableEntity.nameIdentifier().toString()); throw re; } @@ -135,7 +135,7 @@ public TableEntity updateTable( mapper.updateTableMeta( POConverters.updateTablePOWithVersion(oldTablePO, newEntity), oldTablePO)); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.TABLE, newEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java index 76d76d30d3f..cc60e266f2d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java @@ -53,7 +53,7 @@ public void insertTopic(TopicEntity topicEntity, boolean overwrite) { }); // TODO: insert topic dataLayout version after supporting it } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.TOPIC, topicEntity.nameIdentifier().toString()); throw re; } @@ -97,7 +97,7 @@ public TopicEntity updateTopic( mapper.updateTopicMeta( POConverters.updateTopicPOWithVersion(oldTopicPO, newEntity), oldTopicPO)); } catch (RuntimeException re) { - ExceptionUtils.checkSQLConstraintException( + ExceptionUtils.checkSQLException( re, Entity.EntityType.TOPIC, newEntity.nameIdentifier().toString()); throw re; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/ExceptionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/ExceptionUtils.java index bf37465a249..f11b3c2fc77 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/ExceptionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/ExceptionUtils.java @@ -5,21 +5,17 @@ package com.datastrato.gravitino.storage.relational.utils; import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; -import java.sql.SQLIntegrityConstraintViolationException; +import com.datastrato.gravitino.storage.relational.converters.SQLExceptionConverterFactory; +import java.sql.SQLException; public class ExceptionUtils { private ExceptionUtils() {} - public static void checkSQLConstraintException( + public static void checkSQLException( RuntimeException re, Entity.EntityType type, String entityName) { - if (re.getCause() != null - && re.getCause() instanceof SQLIntegrityConstraintViolationException) { - // TODO We should make more fine-grained exception judgments - // Usually throwing `SQLIntegrityConstraintViolationException` means that - // SQL violates the constraints of `primary key` and `unique key`. - // We simply think that the entity already exists at this time. - throw new AlreadyExistsException("%s entity: %s already exists", type.name(), entityName); + if (re.getCause() instanceof SQLException) { + throw SQLExceptionConverterFactory.getConverter() + .toGravitinoException((SQLException) re.getCause(), type, entityName); } } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java new file mode 100644 index 00000000000..965e6043678 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -0,0 +1,469 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.SchemaEntity; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.meta.TableEntity; +import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import org.apache.commons.io.IOUtils; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestJDBCBackend { + private static final String JDBC_STORE_PATH = + "/tmp/gravitino_jdbc_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = JDBC_STORE_PATH + "/testdb"; + private static final Config config = Mockito.mock(Config.class); + public static final ImmutableMap RELATIONAL_BACKENDS = + ImmutableMap.of( + Configs.DEFAULT_ENTITY_RELATIONAL_STORE, JDBCBackend.class.getCanonicalName()); + public static RelationalBackend backend; + + @BeforeAll + public static void setup() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("123"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + + String backendName = config.get(ENTITY_RELATIONAL_STORE); + String className = + RELATIONAL_BACKENDS.getOrDefault(backendName, Configs.DEFAULT_ENTITY_RELATIONAL_STORE); + + try { + backend = (RelationalBackend) Class.forName(className).getDeclaredConstructor().newInstance(); + backend.initialize(config); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create and initialize RelationalBackend by name: " + backendName, e); + } + + prepareJdbcTable(); + } + + @AfterAll + public static void tearDown() throws IOException { + dropAllTables(); + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + backend.close(); + } + + @BeforeEach + public void init() { + truncateAllTables(); + } + + private static void prepareJdbcTable() { + // Read the ddl sql to create table + String scriptPath = "h2/schema-h2.sql"; + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement()) { + StringBuilder ddlBuilder = new StringBuilder(); + IOUtils.readLines( + Objects.requireNonNull( + TestJDBCBackend.class.getClassLoader().getResourceAsStream(scriptPath)), + StandardCharsets.UTF_8) + .forEach(line -> ddlBuilder.append(line).append("\n")); + statement.execute(ddlBuilder.toString()); + } catch (Exception e) { + throw new IllegalStateException("Create tables failed", e); + } + } + + private static void truncateAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("TRUNCATE TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Truncate table failed", e); + } + } + + private static void dropAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("DROP TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Drop table failed", e); + } + } + + @Test + public void testInsertAlreadyExistsException() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "metalake", auditInfo); + BaseMetalake metalakeCopy = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "metalake", auditInfo); + backend.insert(metalake, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(metalakeCopy, false)); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("metalake"), + "catalog", + auditInfo); + CatalogEntity catalogCopy = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("metalake"), + "catalog", + auditInfo); + backend.insert(catalog, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(catalogCopy, false)); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("metalake", "catalog"), + "schema", + auditInfo); + SchemaEntity schemaCopy = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("metalake", "catalog"), + "schema", + auditInfo); + backend.insert(schema, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(schemaCopy, false)); + + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofTable("metalake", "catalog", "schema"), + "table", + auditInfo); + TableEntity tableCopy = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofTable("metalake", "catalog", "schema"), + "table", + auditInfo); + backend.insert(table, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(tableCopy, false)); + + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset", + auditInfo); + FilesetEntity filesetCopy = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset", + auditInfo); + backend.insert(fileset, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(filesetCopy, false)); + + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "topic", + auditInfo); + TopicEntity topicCopy = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "topic", + auditInfo); + backend.insert(topic, false); + assertThrows(AlreadyExistsException.class, () -> backend.insert(topicCopy, false)); + } + + @Test + public void testUpdateAlreadyExistsException() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "metalake", auditInfo); + BaseMetalake metalakeCopy = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "metalake1", auditInfo); + backend.insert(metalake, false); + backend.insert(metalakeCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + metalakeCopy.nameIdentifier(), + Entity.EntityType.METALAKE, + e -> createBaseMakeLake(metalakeCopy.id(), "metalake", auditInfo))); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("metalake"), + "catalog", + auditInfo); + CatalogEntity catalogCopy = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("metalake"), + "catalog1", + auditInfo); + backend.insert(catalog, false); + backend.insert(catalogCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + catalogCopy.nameIdentifier(), + Entity.EntityType.CATALOG, + e -> + createCatalog( + catalogCopy.id(), catalogCopy.namespace(), "catalog", auditInfo))); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("metalake", "catalog"), + "schema", + auditInfo); + SchemaEntity schemaCopy = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("metalake", "catalog"), + "schema1", + auditInfo); + backend.insert(schema, false); + backend.insert(schemaCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + schemaCopy.nameIdentifier(), + Entity.EntityType.SCHEMA, + e -> + createSchemaEntity( + schemaCopy.id(), schemaCopy.namespace(), "schema", auditInfo))); + + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofTable("metalake", "catalog", "schema"), + "table", + auditInfo); + TableEntity tableCopy = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofTable("metalake", "catalog", "schema"), + "table1", + auditInfo); + backend.insert(table, false); + backend.insert(tableCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + tableCopy.nameIdentifier(), + Entity.EntityType.TABLE, + e -> createTableEntity(tableCopy.id(), tableCopy.namespace(), "table", auditInfo))); + + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset", + auditInfo); + FilesetEntity filesetCopy = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset1", + auditInfo); + backend.insert(fileset, false); + backend.insert(filesetCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + filesetCopy.nameIdentifier(), + Entity.EntityType.FILESET, + e -> + createFilesetEntity( + filesetCopy.id(), filesetCopy.namespace(), "fileset", auditInfo))); + + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "topic", + auditInfo); + TopicEntity topicCopy = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "topic1", + auditInfo); + backend.insert(topic, false); + backend.insert(topicCopy, false); + assertThrows( + AlreadyExistsException.class, + () -> + backend.update( + topicCopy.nameIdentifier(), + Entity.EntityType.TOPIC, + e -> createTopicEntity(topicCopy.id(), topicCopy.namespace(), "topic", auditInfo))); + } + + public static BaseMetalake createBaseMakeLake(Long id, String name, AuditInfo auditInfo) { + return BaseMetalake.builder() + .withId(id) + .withName(name) + .withAuditInfo(auditInfo) + .withComment("") + .withProperties(null) + .withVersion(SchemaVersion.V_0_1) + .build(); + } + + public static CatalogEntity createCatalog( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return CatalogEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment("") + .withProperties(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static SchemaEntity createSchemaEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return SchemaEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withComment("") + .withProperties(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static TableEntity createTableEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return TableEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withAuditInfo(auditInfo) + .build(); + } + + public static FilesetEntity createFilesetEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return FilesetEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withFilesetType(Fileset.Type.MANAGED) + .withStorageLocation("/tmp") + .withComment("") + .withProperties(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static TopicEntity createTopicEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return TopicEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withComment("test comment") + .withProperties(ImmutableMap.of("key", "value")) + .withAuditInfo(auditInfo) + .build(); + } +} From 89002cf800b606ee5561cde0849fa2be8296b6a9 Mon Sep 17 00:00:00 2001 From: Congling Xia Date: Mon, 15 Apr 2024 11:46:04 +0800 Subject: [PATCH 019/106] [#2433] improvement(trino-connector): simplify dynamic catalog names (#2547) ### What changes were proposed in this pull request? Support omitting metalake prefix for Trino connector. ### Why are the changes needed? Gravitino register a dynamic catalog with name like `some_metalake.some_catalog`. It is long and must be quoted to use. Besides, if one wants to manage file-based catalogs with Gravitino, users need to adjust their SQL for catalog name changing. With this patch, Trino admins can add a Gravitino connector property `gravitino.simplify-catalog-names=true` to keep the catalog name as it is without `some_metalake.` prefix. Fix: #2433 ### Does this PR introduce _any_ user-facing change? No for default settings. If `gravitino.simplify-catalog-names=true` is set, the catalog names will change when using Trino with Gravitino. ### How was this patch tested? UT --------- Co-authored-by: yuhui --- docs/trino-connector/configuration.md | 11 ++- trino-connector/build.gradle.kts | 1 + .../trino/connector/GravitinoConfig.java | 13 +++ .../connector/GravitinoConnectorFactory.java | 9 +- .../catalog/CatalogConnectorManager.java | 25 +++-- .../connector/catalog/CatalogInjector.java | 6 ++ .../connector/metadata/GravitinoCatalog.java | 6 +- .../trino/connector/GravitinoMockServer.java | 70 ++++++++------ .../TestCreateGravitinoConnector.java | 54 +++++++++++ .../connector/TestGravitinoConnector.java | 28 ++---- ...avitinoConnectorWithSimpleCatalogName.java | 92 +++++++++++++++++++ .../trino/connector/TestGravitinoPlugin.java | 8 +- .../metadata/TestGravitinoCatalog.java | 2 +- 13 files changed, 254 insertions(+), 71 deletions(-) create mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java create mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 671550ebe77..5b1065abe7d 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -6,8 +6,9 @@ license: "Copyright 2023 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2." --- -| Property | Type | Default Value | Description | Required | Since Version | -|--------------------|--------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| -| connector.name | string | (none) | The `connector.name` defines the name of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | -| gravitino.metalake | string | (none) | The `gravitino.metalake` defines which metalake in Gravitino server the Trino connector uses. Trino connector should set it at start, the value of `gravitino.metalake` needs to be a valid name, Trino connector can detect and load the metalake with catalogs, schemas and tables once created and keep in sync. | Yes | 0.2.0 | -| gravitino.uri | string | http://localhost:8090 | The `gravitino.uri` defines the connection URL of the Gravitino server, the default value is `http://localhost:8090`. Trino connector can detect and connect to Gravitino server once it is ready, no need to start Gravitino server beforehand. | Yes | 0.2.0 | +| Property | Type | Default Value | Description | Required | Since Version | +|-----------------------------------|---------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| +| connector.name | string | (none) | The `connector.name` defines the name of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | +| gravitino.metalake | string | (none) | The `gravitino.metalake` defines which metalake in Gravitino server the Trino connector uses. Trino connector should set it at start, the value of `gravitino.metalake` needs to be a valid name, Trino connector can detect and load the metalake with catalogs, schemas and tables once created and keep in sync. | Yes | 0.2.0 | +| gravitino.uri | string | http://localhost:8090 | The `gravitino.uri` defines the connection URL of the Gravitino server, the default value is `http://localhost:8090`. Trino connector can detect and connect to Gravitino server once it is ready, no need to start Gravitino server beforehand. | Yes | 0.2.0 | +| gravitino.simplify-catalog-names | boolean | false | The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. If you set it to true, Trino will configure only one Graviton catalog. | NO | 0.5.0 | diff --git a/trino-connector/build.gradle.kts b/trino-connector/build.gradle.kts index 747de13b37b..38f75911c83 100644 --- a/trino-connector/build.gradle.kts +++ b/trino-connector/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { compileOnly(libs.trino.spi) { exclude("org.apache.logging.log4j") } + testImplementation(libs.awaitility) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation(libs.trino.memory) { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java index 72773214fc4..63b68bb1626 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java @@ -24,6 +24,13 @@ public class GravitinoConfig { private static final ConfigEntry GRAVITINO_METALAKE = new ConfigEntry("gravitino.metalake", "The metalake name for used", "", true); + private static final ConfigEntry GRAVITINO_SIMPLIFY_CATALOG_NAMES = + new ConfigEntry( + "gravitino.simplify-catalog-names", + "Omit metalake prefix for catalog names", + "false", + false); + public GravitinoConfig(Map requiredConfig) { config = requiredConfig; @@ -47,6 +54,12 @@ public String getMetalake() { return config.getOrDefault(GRAVITINO_METALAKE.key, GRAVITINO_METALAKE.defaultValue); } + public boolean simplifyCatalogNames() { + return Boolean.parseBoolean( + config.getOrDefault( + GRAVITINO_SIMPLIFY_CATALOG_NAMES.key, GRAVITINO_SIMPLIFY_CATALOG_NAMES.defaultValue)); + } + boolean isDynamicConnector() { // 'isDynamicConnector' indicates whether the connector is user-configured within Trino or // loaded from the Gravitino server. diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java index f59a95b3f5a..1c7a43f09fe 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.trino.connector; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_METALAKE_NOT_EXISTS; +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorContext; @@ -69,8 +70,7 @@ public Connector create( catalogConnectorManager = new CatalogConnectorManager(catalogInjector, catalogConnectorFactory); catalogConnectorManager.config(config); - catalogConnectorManager.setGravitinoClient(clientProvider().get()); - catalogConnectorManager.start(); + catalogConnectorManager.start(clientProvider().get()); new GravitinoSystemTableFactory(catalogConnectorManager); @@ -95,6 +95,11 @@ public Connector create( if (Strings.isNullOrEmpty(metalake)) { throw new TrinoException(GRAVITINO_METALAKE_NOT_EXISTS, "No gravitino metalake selected"); } + if (config.simplifyCatalogNames() && catalogConnectorManager.getCatalogs().size() > 1) { + throw new TrinoException( + GRAVITINO_MISSING_CONFIG, + "Multiple metalakes are not supported when setting gravitino.simplify-catalog-names = true"); + } catalogConnectorManager.addMetalake(metalake); GravitinoStoredProcedureFactory gravitinoStoredProcedureFactory = new GravitinoStoredProcedureFactory(catalogConnectorManager, metalake); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index edb6347ef54..9ba6fefeb77 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -7,6 +7,7 @@ import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CATALOG_ALREADY_EXISTS; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CATALOG_NOT_EXISTS; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_METALAKE_NOT_EXISTS; +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION; import com.datastrato.gravitino.Catalog; @@ -90,14 +91,11 @@ public void config(GravitinoConfig config) { this.config = Preconditions.checkNotNull(config, "config is not null"); } - @VisibleForTesting - public void setGravitinoClient(GravitinoAdminClient gravitinoClient) { - this.gravitinoClient = gravitinoClient; - } - - public void start() { - if (gravitinoClient == null) { - gravitinoClient = GravitinoAdminClient.builder(config.getURI()).build(); + public void start(GravitinoAdminClient client) { + if (client == null) { + this.gravitinoClient = GravitinoAdminClient.builder(config.getURI()).build(); + } else { + this.gravitinoClient = client; } // Schedule a task to load catalog from gravitino server. @@ -147,7 +145,9 @@ public void loadCatalogs(GravitinoMetalake metalake) { // Delete those catalogs that have been deleted in Gravitino server Set catalogNameStrings = - Arrays.stream(catalogNames).map(NameIdentifier::toString).collect(Collectors.toSet()); + Arrays.stream(catalogNames) + .map(config.simplifyCatalogNames() ? NameIdentifier::name : NameIdentifier::toString) + .collect(Collectors.toSet()); for (Map.Entry entry : catalogConnectors.entrySet()) { if (!catalogNameStrings.contains(entry.getKey()) @@ -164,7 +164,8 @@ public void loadCatalogs(GravitinoMetalake metalake) { (NameIdentifier nameIdentifier) -> { try { Catalog catalog = metalake.loadCatalog(nameIdentifier); - GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(metalake.name(), catalog); + GravitinoCatalog gravitinoCatalog = + new GravitinoCatalog(metalake.name(), catalog, config.simplifyCatalogNames()); if (catalogConnectors.containsKey(gravitinoCatalog.getFullName())) { // Reload catalogs that have been updated in Gravitino server. reloadCatalog(metalake, gravitinoCatalog); @@ -360,6 +361,10 @@ public void alterCatalog( } public void addMetalake(String metalake) { + if (config.simplifyCatalogNames() && usedMetalakes.size() > 1) + throw new TrinoException( + GRAVITINO_MISSING_CONFIG, + "Multiple metalakes are not supported when setting gravitino.simplify-catalog-names = true"); usedMetalakes.add(metalake); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java index f0a8530218d..02cac9f4be4 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java @@ -273,6 +273,12 @@ private void createInjectHandler( Object catalogConnector = createCatalogMethod.invoke(catalogFactory, catalogPropertiesObject); + if (catalogs.containsKey(catalogName)) { + String message = + String.format("Inject catalog failed. catalog %s already exists", catalogName); + LOG.error(message); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, message); + } catalogs.put(catalogName, catalogConnector); }; } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java index 7a7bf125368..19c2fc81659 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -18,10 +18,12 @@ public class GravitinoCatalog { private final String metalake; private final Catalog catalog; + private final boolean usingSimpleName; - public GravitinoCatalog(String metalake, Catalog catalog) { + public GravitinoCatalog(String metalake, Catalog catalog, boolean usingSimpleName) { this.metalake = metalake; this.catalog = catalog; + this.usingSimpleName = usingSimpleName; } public String getProvider() { @@ -33,7 +35,7 @@ public String getName() { } public String getFullName() { - return metalake + "." + catalog.name(); + return usingSimpleName ? catalog.name() : metalake + "." + catalog.name(); } public NameIdentifier geNameIdentifier() { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 9c2a7a1cc91..0b10579dcf4 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -30,6 +31,7 @@ import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.hive.HiveDataTypeTransformer; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.datastrato.gravitino.trino.connector.metadata.GravitinoColumn; import com.datastrato.gravitino.trino.connector.metadata.GravitinoSchema; import com.datastrato.gravitino.trino.connector.metadata.GravitinoTable; @@ -42,6 +44,7 @@ import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.SchemaTableName; import io.trino.testing.ResourcePresence; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -58,12 +61,18 @@ public class GravitinoMockServer implements AutoCloseable { private final Map metalakes = new HashMap<>(); private boolean start = true; + private boolean simpleCatalogName; CatalogConnectorManager catalogConnectorManager; private GeneralDataTypeTransformer dataTypeTransformer = new HiveDataTypeTransformer(); public GravitinoMockServer() { - createGravitinoMetalake(NameIdentifier.ofMetalake(testMetalake)); - createGravitinoCatalog(NameIdentifier.ofCatalog(testMetalake, testCatalog)); + this(false); + } + + public GravitinoMockServer(boolean simpleCatalogName) { + this.simpleCatalogName = simpleCatalogName; + createMetalake(NameIdentifier.ofMetalake(testMetalake)); + createCatalog(NameIdentifier.ofCatalog(testMetalake, testCatalog)); } public void setCatalogConnectorManager(CatalogConnectorManager catalogConnectorManager) { @@ -79,7 +88,7 @@ public GravitinoAdminClient createGravitinoClient() { @Override public GravitinoMetalake answer(InvocationOnMock invocation) throws Throwable { NameIdentifier metalakeName = invocation.getArgument(0); - return createGravitinoMetalake(metalakeName); + return createMetalake(metalakeName); } }); @@ -120,7 +129,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { return client; } - private GravitinoMetalake createGravitinoMetalake(NameIdentifier metalakeName) { + private GravitinoMetalake createMetalake(NameIdentifier metalakeName) { GravitinoMetalake metaLake = mock(GravitinoMetalake.class); when(metaLake.name()).thenReturn(metalakeName.name()); when(metaLake.listCatalogs(any(Namespace.class))) @@ -142,7 +151,7 @@ public NameIdentifier[] answer(InvocationOnMock invocation) throws Throwable { public Catalog answer(InvocationOnMock invocation) throws Throwable { NameIdentifier catalogName = invocation.getArgument(0); - Catalog catalog = createGravitinoCatalog(catalogName); + Catalog catalog = createCatalog(catalogName); return catalog; } @@ -197,21 +206,28 @@ void reloadCatalogs() { catalogConnectorManager.loadCatalogs(metaLake); } - private Catalog createGravitinoCatalog(NameIdentifier catalogName) { + private Catalog createCatalog(NameIdentifier catalogName) { Catalog catalog = mock(Catalog.class); when(catalog.name()).thenReturn(catalogName.name()); when(catalog.provider()).thenReturn(testCatalogProvider); when(catalog.type()).thenReturn(Catalog.Type.RELATIONAL); when(catalog.properties()).thenReturn(Map.of("max_ttl", "10")); - when(catalog.asTableCatalog()).thenAnswer(answer -> createTableCatalog(catalogName)); + Audit mockAudit = mock(Audit.class); + when(mockAudit.creator()).thenReturn("gravitino"); + when(mockAudit.createTime()).thenReturn(Instant.now()); + when(catalog.auditInfo()).thenReturn(mockAudit); + + GravitinoCatalog gravitinoCatalog = + new GravitinoCatalog(testMetalake, catalog, simpleCatalogName); + when(catalog.asTableCatalog()).thenAnswer(answer -> createTableCatalog(gravitinoCatalog)); - when(catalog.asSchemas()).thenAnswer(answer -> createSchemas(catalogName)); + when(catalog.asSchemas()).thenAnswer(answer -> createSchemas(gravitinoCatalog)); metalakes.get(catalogName.namespace().toString()).catalogs.put(catalogName.name(), catalog); return catalog; } - private SupportsSchemas createSchemas(NameIdentifier catalogName) { + private SupportsSchemas createSchemas(GravitinoCatalog catalog) { SupportsSchemas schemas = mock(SupportsSchemas.class); when(schemas.createSchema(any(NameIdentifier.class), anyString(), anyMap())) .thenAnswer( @@ -225,12 +241,12 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getMetadataAdapter(); GravitinoSchema schema = new GravitinoSchema(schemaName.name(), properties, ""); metadata.createSchema(null, schemaName.name(), emptyMap(), null); @@ -253,7 +269,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); metadata.dropSchema(null, nameIdentifier.name(), cascade); @@ -270,7 +286,7 @@ public NameIdentifier[] answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); return metadata.listSchemaNames(null).stream() @@ -291,7 +307,7 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); memoryConnector.getMetadata(null, null); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -300,7 +316,7 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getMetadataAdapter(); GravitinoSchema gravitinoSchema = @@ -320,7 +336,7 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { return schemas; } - private TableCatalog createTableCatalog(NameIdentifier catalogName) { + private TableCatalog createTableCatalog(GravitinoCatalog catalog) { TableCatalog tableCatalog = mock(TableCatalog.class); when(tableCatalog.createTable( any(NameIdentifier.class), @@ -345,7 +361,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { tableName.schema(), tableName.table(), columns, comment, properties); CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getMetadataAdapter(); ConnectorTableMetadata tableMetadata = metadataAdapter.getTableMetadata(gravitinoTable); @@ -353,7 +369,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); metadata.createTable(null, tableMetadata, false); @@ -372,7 +388,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); memoryConnector.getMetadata(null, null); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -399,7 +415,7 @@ public NameIdentifier[] answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); ArrayList tableNames = new ArrayList<>(); @@ -425,7 +441,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); return metadata.getTableHandle( @@ -447,7 +463,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -461,7 +477,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getMetadataAdapter(); GravitinoTable gravitinoTable = metadataAdapter.createTable(tableMetadata); @@ -489,7 +505,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalogName.toString()) + .getCatalogConnector(catalog.getFullName()) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); ConnectorTableHandle tableHandle = @@ -501,7 +517,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { for (int i = 1; i < invocation.getArguments().length; i++) { TableChange tableChange = invocation.getArgument(i); - doAlterTable(tableChange, tableHandle, tableName, metadata, catalogName); + doAlterTable(tableChange, tableHandle, tableName, metadata, catalog); } return null; } @@ -514,7 +530,7 @@ void doAlterTable( ConnectorTableHandle tableHandle, TableName tableName, ConnectorMetadata metadata, - NameIdentifier catalogName) { + GravitinoCatalog catalog) { if (tableChange instanceof TableChange.RenameTable) { TableChange.RenameTable renameTable = (TableChange.RenameTable) tableChange; metadata.renameTable( @@ -526,7 +542,7 @@ void doAlterTable( GravitinoColumn column = new GravitinoColumn(fieldName, addColumn.getDataType(), -1, "", true); CatalogConnectorMetadataAdapter metadataAdapter = - catalogConnectorManager.getCatalogConnector(catalogName.toString()).getMetadataAdapter(); + catalogConnectorManager.getCatalogConnector(catalog.getFullName()).getMetadataAdapter(); metadata.addColumn(null, tableHandle, metadataAdapter.getColumnMetadata(column)); } else if (tableChange instanceof TableChange.DeleteColumn) { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java new file mode 100644 index 00000000000..5b7013b9b75 --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector; + +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastrato.gravitino.client.GravitinoAdminClient; +import io.trino.Session; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import java.util.HashMap; +import org.testng.annotations.Test; + +public class TestCreateGravitinoConnector { + + GravitinoMockServer server; + + @Test + public void testCreateSimpleCatalogNameConnector() throws Exception { + server = new GravitinoMockServer(true); + Session session = testSessionBuilder().setCatalog("gravitino").build(); + QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); + + GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); + TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); + queryRunner.installPlugin(gravitinoPlugin); + + { + // create a gravitino connector named gravitino using metalake test + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "true"); + queryRunner.createCatalog("test0", "gravitino", properties); + } + + { + // Test failed to create catalog with different metalake + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test1"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + try { + queryRunner.createCatalog("test1", "gravitino", properties); + } catch (Exception e) { + assertThat(e.getMessage()).contains("Multiple metalakes are not supported"); + } + } + + server.close(); + } +} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index 16aa2005780..c6054909975 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -22,10 +22,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.testcontainers.shaded.com.google.common.base.Preconditions; -import org.testng.annotations.BeforeMethod; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.Parameters; import org.testng.annotations.Test; @@ -34,11 +35,6 @@ public class TestGravitinoConnector extends AbstractTestQueryFramework { GravitinoMockServer server; - @BeforeMethod - public void reloadCatalog() { - server.reloadCatalogs(); - } - @Override protected QueryRunner createQueryRunner() throws Exception { server = closeAfterClass(new GravitinoMockServer()); @@ -50,8 +46,7 @@ protected QueryRunner createQueryRunner() throws Exception { // queryRunner = LocalQueryRunner.builder(session).build(); queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); - TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(); - gravitinoPlugin.setGravitinoClient(gravitinoClient); + TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); queryRunner.installPlugin(gravitinoPlugin); queryRunner.installPlugin(new MemoryPlugin()); @@ -73,21 +68,14 @@ protected QueryRunner createQueryRunner() throws Exception { CatalogConnectorManager catalogConnectorManager = gravitinoPlugin.getCatalogConnectorManager(); - catalogConnectorManager.setGravitinoClient(gravitinoClient); server.setCatalogConnectorManager(catalogConnectorManager); // Wait for the catalog to be created. Wait for at least 30 seconds. - int max_tries = 35; - while (catalogConnectorManager.getCatalogs().isEmpty() && max_tries > 0) { - Thread.sleep(1000); - max_tries--; - } - - if (max_tries == 0) { - throw new RuntimeException("Failed to create catalog in about 35 seconds..."); - } - + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Create query runner failed", e); } return queryRunner; } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java new file mode 100644 index 00000000000..573816701de --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java @@ -0,0 +1,92 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector; + +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; + +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; +import io.trino.Session; +import io.trino.plugin.memory.MemoryPlugin; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedResult; +import io.trino.testing.MaterializedRow; +import io.trino.testing.QueryRunner; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.annotations.Test; + +public class TestGravitinoConnectorWithSimpleCatalogName extends AbstractTestQueryFramework { + + GravitinoMockServer server; + + @Override + protected QueryRunner createQueryRunner() throws Exception { + server = closeAfterClass(new GravitinoMockServer(true)); + GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); + + Session session = testSessionBuilder().setCatalog("gravitino").build(); + QueryRunner queryRunner = null; + try { + queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); + + TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); + queryRunner.installPlugin(gravitinoPlugin); + queryRunner.installPlugin(new MemoryPlugin()); + + // create a gravitino connector named gravitino using metalake test + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "true"); + queryRunner.createCatalog("gravitino", "gravitino", properties); + + CatalogConnectorManager catalogConnectorManager = + gravitinoPlugin.getCatalogConnectorManager(); + server.setCatalogConnectorManager(catalogConnectorManager); + // Wait for the catalog to be created. Wait for at least 30 seconds. + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); + } catch (Exception e) { + throw new RuntimeException("Create query runner failed", e); + } + return queryRunner; + } + + @Test + public void testCatalogName() { + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory"); + assertUpdate("call gravitino.system.create_catalog('memory1', 'memory', Map())"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory1"); + + String schemaName = "db1"; + String fullSchemaName = String.format("\"%s\".%s", "memory", schemaName); + assertUpdate("create schema " + fullSchemaName); + assertThat(computeActual("show schemas from \"memory\"").getOnlyColumnAsSet()) + .contains(schemaName); + + assertUpdate("drop schema " + fullSchemaName); + assertUpdate("call gravitino.system.drop_catalog('memory1')"); + } + + @Test + public void testSystemTable() throws Exception { + MaterializedResult expectedResult = computeActual("select * from gravitino.system.catalog"); + assertEquals(expectedResult.getRowCount(), 1); + List expectedRows = expectedResult.getMaterializedRows(); + MaterializedRow row = expectedRows.get(0); + assertEquals(row.getField(0), "memory"); + assertEquals(row.getField(1), "memory"); + assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); + } +} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java index 375623b00f2..c6b6923db01 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java @@ -14,6 +14,10 @@ public class TestGravitinoPlugin extends GravitinoPlugin { private GravitinoAdminClient gravitinoClient; + public TestGravitinoPlugin(GravitinoAdminClient gravitinoClient) { + this.gravitinoClient = gravitinoClient; + } + @Override public Iterable getConnectorFactories() { factory = new TestGravitinoConnectorFactory(); @@ -21,10 +25,6 @@ public Iterable getConnectorFactories() { return ImmutableList.of(factory); } - public void setGravitinoClient(GravitinoAdminClient client) { - this.gravitinoClient = client; - } - public CatalogConnectorManager getCatalogConnectorManager() { return factory.getCatalogConnectorManager(); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 58453993a63..6641943465a 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -24,7 +24,7 @@ public void testGravitinoCatalog() { Catalog mockCatalog = mockCatalog( catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, Collections.emptyMap()); - GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); + GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog, false); assertEquals(catalogName, catalog.getName()); assertEquals(provider, catalog.getProvider()); } From 1fa6300c2ddd827858de3edb61a402221972c39b Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 15 Apr 2024 16:33:17 +0800 Subject: [PATCH 020/106] [#2934] feat(catalog-mysql): use metaData.getTables instead of directly SQL (#2935) ### What changes were proposed in this pull request? use metaData.getTables instead of directly SQL ### Why are the changes needed? Fix: #2934 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? IT --- .../mysql/operation/MysqlTableOperations.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index ef7486e3e3f..09ebab3506b 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -25,11 +25,11 @@ import com.datastrato.gravitino.rel.types.Types; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -55,19 +55,17 @@ public class MysqlTableOperations extends JdbcTableOperations { @Override public List listTables(String databaseName) throws NoSuchSchemaException { - try (Connection connection = getConnection(databaseName)) { - try (Statement statement = connection.createStatement()) { - String showTablesQuery = "SHOW TABLES"; - ResultSet resultSet = statement.executeQuery(showTablesQuery); - List names = new ArrayList<>(); - while (resultSet.next()) { - String tableName = resultSet.getString(1); - names.add(tableName); + final List names = Lists.newArrayList(); + + try (Connection connection = getConnection(databaseName); + ResultSet tables = getTables(connection)) { + while (tables.next()) { + if (Objects.equals(tables.getString("TABLE_CAT"), databaseName)) { + names.add(tables.getString("TABLE_NAME")); } - LOG.info( - "Finished listing tables size {} for database name {} ", names.size(), databaseName); - return names; } + LOG.info("Finished listing tables size {} for database name {} ", names.size(), databaseName); + return names; } catch (final SQLException se) { throw this.exceptionMapper.toGravitinoException(se); } From 68b4587eac1cc669e6356dd3a756766daed83935 Mon Sep 17 00:00:00 2001 From: cai can <94670132+caican00@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:17:16 +0800 Subject: [PATCH 021/106] [#2927] Improvement(catalog-lakehouse-iceberg): Support more file formats in using clause when create iceberg tables (#2931) ### What changes were proposed in this pull request? Support more file formats in using clause when create iceberg tables, such as `using parquet`, `using orc`, `using avro`, `using iceberg`.using other provider will throw an exception. ### Why are the changes needed? Because Iceberg official supports using parquet/orc/avro/iceberg when create a new table. Fix: https://github.com/datastrato/gravitino/issues/2927 ### Does this PR introduce _any_ user-facing change? Yes, users can using `parquet/orc/avro/iceberg` keywork in `using clause` when create a new table. ### How was this patch tested? New IT. --- .../lakehouse/iceberg/IcebergTable.java | 28 ++++- .../IcebergTablePropertiesMetadata.java | 10 +- .../integration/test/CatalogIcebergIT.java | 104 ++++++++++++++++++ 3 files changed, 136 insertions(+), 6 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java index 4b909a652d8..4c2e3cf9c50 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.catalog.lakehouse.iceberg; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata.DISTRIBUTION_MODE; +import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; import com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.ConvertUtil; import com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.FromIcebergPartitionSpec; @@ -47,12 +48,33 @@ public class IcebergTable extends BaseTable { /** The default provider of the table. */ public static final String DEFAULT_ICEBERG_PROVIDER = "iceberg"; + /** The supported parquet file format for Iceberg tables. */ + public static final String ICEBERG_PARQUET_FILE_FORMAT = "parquet"; + /** The supported orc file format for Iceberg tables. */ + public static final String ICEBERG_ORC_FILE_FORMAT = "orc"; + /** The supported avro file format for Iceberg tables. */ + public static final String ICEBERG_AVRO_FILE_FORMAT = "avro"; + public static final String ICEBERG_COMMENT_FIELD_NAME = "comment"; private String location; private IcebergTable() {} + public static Map rebuildCreateProperties(Map createProperties) { + String provider = createProperties.get(PROP_PROVIDER); + if (ICEBERG_PARQUET_FILE_FORMAT.equalsIgnoreCase(provider)) { + createProperties.put(DEFAULT_FILE_FORMAT, ICEBERG_PARQUET_FILE_FORMAT); + } else if (ICEBERG_AVRO_FILE_FORMAT.equalsIgnoreCase(provider)) { + createProperties.put(DEFAULT_FILE_FORMAT, ICEBERG_AVRO_FILE_FORMAT); + } else if (ICEBERG_ORC_FILE_FORMAT.equalsIgnoreCase(provider)) { + createProperties.put(DEFAULT_FILE_FORMAT, ICEBERG_ORC_FILE_FORMAT); + } else if (provider != null && !DEFAULT_ICEBERG_PROVIDER.equalsIgnoreCase(provider)) { + throw new IllegalArgumentException("Unsupported format in USING: " + provider); + } + return createProperties; + } + public CreateTableRequest toCreateTableRequest() { Schema schema = ConvertUtil.toIcebergSchema(this); properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties); @@ -62,7 +84,7 @@ public CreateTableRequest toCreateTableRequest() { .withName(name) .withLocation(location) .withSchema(schema) - .setProperties(properties) + .setProperties(rebuildCreateProperties(properties)) .withPartitionSpec(ToIcebergPartitionSpec.toPartitionSpec(schema, partitioning)) .withWriteOrder(ToIcebergSortOrder.toSortOrder(schema, sortOrders)); return builder.build(); @@ -186,10 +208,6 @@ protected IcebergTable internalBuild() { if (null != comment) { icebergTable.properties.putIfAbsent(ICEBERG_COMMENT_FIELD_NAME, comment); } - String provider = icebergTable.properties.get(PROP_PROVIDER); - if (provider != null && !DEFAULT_ICEBERG_PROVIDER.equalsIgnoreCase(provider)) { - throw new IllegalArgumentException("Unsupported format in USING: " + provider); - } return icebergTable; } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java index ab6b557e177..e07f8ccfa8b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java @@ -23,6 +23,7 @@ public class IcebergTablePropertiesMetadata extends BasePropertiesMetadata { public static final String CHERRY_PICK_SNAPSHOT_ID = "cherry-pick-snapshot-id"; public static final String SORT_ORDER = "sort-order"; public static final String IDENTIFIER_FIELDS = "identifier-fields"; + public static final String PROVIDER = "provider"; public static final String DISTRIBUTION_MODE = TableProperties.WRITE_DISTRIBUTION_MODE; @@ -47,7 +48,14 @@ public class IcebergTablePropertiesMetadata extends BasePropertiesMetadata { SORT_ORDER, "Selecting a specific snapshot in a merge operation", false), stringReservedPropertyEntry( IDENTIFIER_FIELDS, "The identifier field(s) for defining the table", false), - stringReservedPropertyEntry(DISTRIBUTION_MODE, "Write distribution mode", false)); + stringReservedPropertyEntry(DISTRIBUTION_MODE, "Write distribution mode", false), + stringImmutablePropertyEntry( + PROVIDER, + "Iceberg provider for Iceberg table fileFormat, such as parquet, orc, avro, iceberg", + false, + null, + false, + false)); PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java index cc9512f122d..e2a330b94c4 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java @@ -4,6 +4,12 @@ */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable.DEFAULT_ICEBERG_PROVIDER; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable.ICEBERG_AVRO_FILE_FORMAT; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable.ICEBERG_ORC_FILE_FORMAT; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable.ICEBERG_PARQUET_FILE_FORMAT; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable.PROP_PROVIDER; +import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; import static org.junit.jupiter.api.Assertions.assertThrows; import com.datastrato.gravitino.Catalog; @@ -1051,6 +1057,104 @@ public void testTableDistribution() { "Iceberg's Distribution Mode.RANGE not support set expressions.")); } + @Test + void testIcebergTablePropertiesWhenCreate() { + String[] providers = + new String[] { + null, + DEFAULT_ICEBERG_PROVIDER, + ICEBERG_PARQUET_FILE_FORMAT, + ICEBERG_ORC_FILE_FORMAT, + ICEBERG_AVRO_FILE_FORMAT + }; + + // Create table from Gravitino API + Column[] columns = createColumns(); + + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + Distribution distribution = Distributions.NONE; + + final SortOrder[] sortOrders = + new SortOrder[] { + SortOrders.of( + NamedReference.field(ICEBERG_COL_NAME2), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST) + }; + + Transform[] partitioning = new Transform[] {Transforms.day(columns[1].name())}; + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Arrays.stream(providers) + .forEach( + provider -> { + if (provider != null) { + properties.put(PROP_PROVIDER, provider); + } + if (DEFAULT_ICEBERG_PROVIDER.equals(provider)) { + provider = null; + } + checkIcebergTableFileFormat( + tableCatalog, + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + sortOrders, + provider); + tableCatalog.dropTable(tableIdentifier); + }); + + properties.put(PROP_PROVIDER, "text"); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + sortOrders)); + + properties.put(PROP_PROVIDER, ICEBERG_PARQUET_FILE_FORMAT); + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + sortOrders); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + tableCatalog.alterTable( + tableIdentifier, TableChange.setProperty(PROP_PROVIDER, ICEBERG_ORC_FILE_FORMAT))); + } + + private static void checkIcebergTableFileFormat( + TableCatalog tableCatalog, + NameIdentifier tableIdentifier, + Column[] columns, + String comment, + Map properties, + Transform[] partitioning, + Distribution distribution, + SortOrder[] sortOrders, + String expectedFileFormat) { + Table createdTable = + tableCatalog.createTable( + tableIdentifier, columns, comment, properties, partitioning, distribution, sortOrders); + Assertions.assertEquals(expectedFileFormat, createdTable.properties().get(DEFAULT_FILE_FORMAT)); + Table loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(expectedFileFormat, loadTable.properties().get(DEFAULT_FILE_FORMAT)); + } + protected static void assertionsTableInfo( String tableName, String tableComment, From 0ff45cd526ee6fb9340010ef81db23557e7fc9f0 Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 15 Apr 2024 17:40:55 +0800 Subject: [PATCH 022/106] [#2920]feat(core): supports more table event (#2895) ### What changes were proposed in this pull request? supports `AlterTableEvent` `PurgeTableEvent` `LoadTableEvent` `ListTableEvent` ### Why are the changes needed? Fix: #2920 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../catalog/TableEventDispatcher.java | 52 +++++++++++++++-- .../listener/api/event/AlterTableEvent.java | 58 +++++++++++++++++++ .../api/event/AlterTableFailureEvent.java | 43 ++++++++++++++ .../listener/api/event/CreateTableEvent.java | 31 +++------- .../api/event/CreateTableFailureEvent.java | 20 ++----- .../listener/api/event/DropTableEvent.java | 10 +--- .../api/event/DropTableFailureEvent.java | 13 +---- .../gravitino/listener/api/event/Event.java | 7 ++- .../listener/api/event/FailureEvent.java | 3 +- .../listener/api/event/ListTableEvent.java | 43 ++++++++++++++ .../api/event/ListTableFailureEvent.java | 40 +++++++++++++ .../listener/api/event/LoadTableEvent.java | 37 ++++++++++++ .../api/event/LoadTableFailureEvent.java | 25 ++++++++ .../listener/api/event/PurgeTableEvent.java | 38 ++++++++++++ .../api/event/PurgeTableFailureEvent.java | 28 +++++++++ .../listener/api/event/TableEvent.java | 3 +- .../listener/api/event/TableFailureEvent.java | 7 +-- 17 files changed, 388 insertions(+), 70 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java index 987b2687b96..bb2735527d9 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java @@ -11,10 +11,18 @@ import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterTableEvent; +import com.datastrato.gravitino.listener.api.event.AlterTableFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateTableEvent; import com.datastrato.gravitino.listener.api.event.CreateTableFailureEvent; import com.datastrato.gravitino.listener.api.event.DropTableEvent; import com.datastrato.gravitino.listener.api.event.DropTableFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListTableEvent; +import com.datastrato.gravitino.listener.api.event.ListTableFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadTableEvent; +import com.datastrato.gravitino.listener.api.event.LoadTableFailureEvent; +import com.datastrato.gravitino.listener.api.event.PurgeTableEvent; +import com.datastrato.gravitino.listener.api.event.PurgeTableFailureEvent; import com.datastrato.gravitino.listener.api.info.TableInfo; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; @@ -50,12 +58,29 @@ public TableEventDispatcher(EventBus eventBus, TableDispatcher dispatcher) { @Override public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { - return dispatcher.listTables(namespace); + try { + NameIdentifier[] nameIdentifiers = dispatcher.listTables(namespace); + eventBus.dispatchEvent(new ListTableEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return nameIdentifiers; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListTableFailureEvent(PrincipalUtils.getCurrentUserName(), namespace, e)); + throw e; + } } @Override public Table loadTable(NameIdentifier ident) throws NoSuchTableException { - return dispatcher.loadTable(ident); + try { + Table table = dispatcher.loadTable(ident); + eventBus.dispatchEvent( + new LoadTableEvent(PrincipalUtils.getCurrentUserName(), ident, new TableInfo(table))); + return table; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadTableFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } } @Override @@ -98,7 +123,17 @@ public Table createTable( @Override public Table alterTable(NameIdentifier ident, TableChange... changes) throws NoSuchTableException, IllegalArgumentException { - return dispatcher.alterTable(ident, changes); + try { + Table table = dispatcher.alterTable(ident, changes); + eventBus.dispatchEvent( + new AlterTableEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new TableInfo(table))); + return table; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterTableFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } } @Override @@ -117,7 +152,16 @@ public boolean dropTable(NameIdentifier ident) { @Override public boolean purgeTable(NameIdentifier ident) { - return dispatcher.purgeTable(ident); + try { + boolean isExists = dispatcher.purgeTable(ident); + eventBus.dispatchEvent( + new PurgeTableEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new PurgeTableFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java new file mode 100644 index 00000000000..a18359c70f3 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TableInfo; +import com.datastrato.gravitino.rel.TableChange; + +/** Represents an event fired when a table is successfully altered. */ +@DeveloperApi +public final class AlterTableEvent extends TableEvent { + private final TableInfo updatedTableInfo; + private final TableChange[] tableChanges; + + /** + * Constructs an instance of {@code AlterTableEvent}, encapsulating the key details about the + * successful alteration of a table. + * + * @param user The username of the individual responsible for initiating the table alteration. + * @param identifier The unique identifier of the altered table, serving as a clear reference + * point for the table in question. + * @param tableChanges An array of {@link TableChange} objects representing the specific changes + * applied to the table during the alteration process. + * @param updatedTableInfo The post-alteration state of the table. + */ + public AlterTableEvent( + String user, + NameIdentifier identifier, + TableChange[] tableChanges, + TableInfo updatedTableInfo) { + super(user, identifier); + this.tableChanges = tableChanges.clone(); + this.updatedTableInfo = updatedTableInfo; + } + + /** + * Retrieves the updated state of the table after the successful alteration. + * + * @return A {@link TableInfo} instance encapsulating the details of the altered table. + */ + public TableInfo updatedTableInfo() { + return updatedTableInfo; + } + + /** + * Retrieves the specific changes that were made to the table during the alteration process. + * + * @return An array of {@link TableChange} objects detailing each modification applied to the + * table. + */ + public TableChange[] tableChanges() { + return tableChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java new file mode 100644 index 00000000000..0c8b1edbfbe --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.rel.TableChange; + +/** + * Represents an event that is triggered when an attempt to alter a table fails due to an exception. + */ +@DeveloperApi +public final class AlterTableFailureEvent extends TableFailureEvent { + private final TableChange[] tableChanges; + + /** + * Constructs an {@code AlterTableFailureEvent} instance, capturing detailed information about the + * failed table alteration attempt. + * + * @param user The user who initiated the table alteration operation. + * @param identifier The identifier of the table that was attempted to be altered. + * @param exception The exception that was thrown during the table alteration operation. + * @param tableChanges The changes that were attempted on the table. + */ + public AlterTableFailureEvent( + String user, NameIdentifier identifier, Exception exception, TableChange[] tableChanges) { + super(user, identifier, exception); + this.tableChanges = tableChanges.clone(); + } + + /** + * Retrieves the changes that were attempted on the table. + * + * @return An array of {@link TableChange} objects representing the attempted modifications to the + * table. + */ + public TableChange[] tableChanges() { + return tableChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java index b68dee9db0d..fe89dcc8f63 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java @@ -9,16 +9,7 @@ import com.datastrato.gravitino.annotation.DeveloperApi; import com.datastrato.gravitino.listener.api.info.TableInfo; -/** - * Represents an event triggered upon the successful creation of a table. This class extends {@link - * TableEvent} to provide detailed information specifically related to the table's creation. This - * includes the final table information as it is returned to the user once the creation process has - * successfully completed. - * - *

Such an event is instrumental for a variety of purposes including, but not limited to, - * auditing activities associated with table creation, monitoring the creation of tables within a - * system, and acquiring insights into the final configuration and state of a newly created table. - */ +/** Represents an event triggered upon the successful creation of a table. */ @DeveloperApi public final class CreateTableEvent extends TableEvent { private final TableInfo createdTableInfo; @@ -27,17 +18,9 @@ public final class CreateTableEvent extends TableEvent { * Constructs an instance of {@code CreateTableEvent}, capturing essential details about the * successful creation of a table. * - *

This constructor documents the successful culmination of the table creation process by - * encapsulating the final state of the table. This includes any adjustments or resolutions made - * to the table's properties or configuration during the creation process. - * - * @param user The username of the individual who initiated the table creation. This information - * is vital for tracking the source of changes and for audit trails. - * @param identifier The unique identifier of the table that was created, providing a precise - * reference to the affected table. - * @param createdTableInfo The final state of the table post-creation. This information is - * reflective of the table's configuration, including any default settings or properties - * applied during the creation process. + * @param user The username of the individual who initiated the table creation. + * @param identifier The unique identifier of the table that was created. + * @param createdTableInfo The final state of the table post-creation. */ public CreateTableEvent(String user, NameIdentifier identifier, TableInfo createdTableInfo) { super(user, identifier); @@ -45,11 +28,11 @@ public CreateTableEvent(String user, NameIdentifier identifier, TableInfo create } /** - * Retrieves the final state and configuration information of the table as it was returned to the - * user after successful creation. + * Retrieves the final state of the table as it was returned to the user after successful + * creation. * * @return A {@link TableInfo} instance encapsulating the comprehensive details of the newly - * created table, highlighting its configuration and any default settings applied. + * created table. */ public TableInfo createdTableInfo() { return createdTableInfo; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java index 6370378b3f6..dc60ad44c88 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java @@ -8,14 +8,10 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.annotation.DeveloperApi; import com.datastrato.gravitino.listener.api.info.TableInfo; -import com.datastrato.gravitino.rel.Table; /** * Represents an event that is generated when an attempt to create a table fails due to an - * exception. This class extends {@link TableFailureEvent} to specifically address failure scenarios - * encountered during the table creation process. It encapsulates both the exception that caused the - * failure and the original request details for the table creation, providing comprehensive context - * for the failure. + * exception. */ @DeveloperApi public final class CreateTableFailureEvent extends TableFailureEvent { @@ -25,10 +21,8 @@ public final class CreateTableFailureEvent extends TableFailureEvent { * Constructs a {@code CreateTableFailureEvent} instance, capturing detailed information about the * failed table creation attempt. * - * @param user The user who initiated the table creation operation. This information is essential - * for auditing and diagnosing the cause of the failure. - * @param identifier The identifier of the table that was attempted to be created. This helps in - * pinpointing the specific table related to the failure. + * @param user The user who initiated the table creation operation. + * @param identifier The identifier of the table that was attempted to be created. * @param exception The exception that was thrown during the table creation operation, providing * insight into what went wrong. * @param createTableRequest The original request information used to attempt to create the table. @@ -42,12 +36,10 @@ public CreateTableFailureEvent( } /** - * Retrieves the original request information for the attempted table creation. This information - * can be valuable for understanding the configuration and expectations that led to the failure, - * facilitating analysis and potential corrective actions. + * Retrieves the original request information for the attempted table creation. * - * @return The {@link Table} instance representing the request information for the failed table - * creation attempt. + * @return The {@link TableInfo} instance representing the request information for the failed + * table creation attempt. */ public TableInfo createTableRequest() { return createTableRequest; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java index a75939eaf66..6e2ba0a6dd5 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java @@ -10,9 +10,6 @@ /** * Represents an event that is generated after a table is successfully dropped from the database. - * This class extends {@link TableEvent} to capture specific details related to the dropping of a - * table, including the status of the table's existence at the time of the operation and identifying - * information about the table and the user who initiated the drop operation. */ @DeveloperApi public final class DropTableEvent extends TableEvent { @@ -22,12 +19,9 @@ public final class DropTableEvent extends TableEvent { * Constructs a new {@code DropTableEvent} instance, encapsulating information about the outcome * of a table drop operation. * - * @param user The user who initiated the drop table operation. This information is important for - * auditing purposes and understanding who is responsible for the change. - * @param identifier The identifier of the table that was attempted to be dropped. This provides a - * clear reference to the specific table affected by the operation. + * @param user The user who initiated the drop table operation. + * @param identifier The identifier of the table that was attempted to be dropped. * @param isExists A boolean flag indicating whether the table existed at the time of the drop - * operation. This can be useful to understand the state of the database prior to the * operation. */ public DropTableEvent(String user, NameIdentifier identifier, boolean isExists) { diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java index 78076fe961d..87eef3b9e22 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java @@ -10,11 +10,7 @@ /** * Represents an event that is generated when an attempt to drop a table from the database fails due - * to an exception. This class extends {@link TableFailureEvent} to provide specific context related - * to table drop failures, encapsulating details about the user who initiated the operation, the - * identifier of the table that was attempted to be dropped, and the exception that led to the - * failure. This event can be used for auditing purposes and to facilitate error handling and - * diagnostic processes. + * to an exception. */ @DeveloperApi public final class DropTableFailureEvent extends TableFailureEvent { @@ -22,11 +18,8 @@ public final class DropTableFailureEvent extends TableFailureEvent { * Constructs a new {@code DropTableFailureEvent} instance, capturing detailed information about * the failed attempt to drop a table. * - * @param user The user who initiated the drop table operation. This information is crucial for - * understanding the context of the operation and for auditing who is responsible for the - * attempted change. - * @param identifier The identifier of the table that the operation attempted to drop. This - * provides a clear reference to the specific table involved in the failure. + * @param user The user who initiated the drop table operation. + * @param identifier The identifier of the table that the operation attempted to drop. * @param exception The exception that was thrown during the drop table operation, offering * insights into what went wrong and why the operation failed. */ diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java index 484120a8bca..e4ed6d474f0 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.annotation.DeveloperApi; +import javax.annotation.Nullable; /** * The abstract base class for all events. It encapsulates common information such as the user who @@ -16,7 +17,7 @@ @DeveloperApi public abstract class Event { private final String user; - private final NameIdentifier identifier; + @Nullable private final NameIdentifier identifier; /** * Constructs an Event instance with the specified user and resource identifier details. @@ -43,9 +44,13 @@ public String user() { /** * Retrieves the resource identifier associated with this event. * + *

For list operations within a namespace, the identifier is the identifier corresponds to that + * namespace. For metalake list operation, identifier is null. + * * @return A NameIdentifier object that represents the resource, like a metalake, catalog, schema, * table, etc., associated with the event. */ + @Nullable public NameIdentifier identifier() { return identifier; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java index ca0c263c20a..471c9baad9f 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java @@ -23,8 +23,7 @@ public abstract class FailureEvent extends Event { * Constructs a new {@code FailureEvent} instance with the specified user, resource identifier, * and the exception that was thrown. * - * @param user The user associated with the operation that resulted in a failure. This information - * is important for auditing and understanding the context of the failure. + * @param user The user associated with the operation that resulted in a failure. * @param identifier The identifier of the resource involved in the operation that failed. This * provides a clear reference to what was being acted upon when the exception occurred. * @param exception The exception that was thrown during the operation. This is the primary piece diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java new file mode 100644 index 00000000000..63eb0d410cc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered upon the successful list of tables within a namespace. + * + *

To optimize memory usage and avoid the potential overhead associated with storing a large + * number of tables directly within the ListTableEvent, the actual tables listed are not maintained + * in this event. This design decision helps in managing resource efficiency, especially in + * environments with extensive table listings. + */ +@DeveloperApi +public final class ListTableEvent extends TableEvent { + private final Namespace namespace; + + /** + * Constructs an instance of {@code ListTableEvent}. + * + * @param user The username of the individual who initiated the table listing. + * @param namespace The namespace from which tables were listed. + */ + public ListTableEvent(String user, Namespace namespace) { + super(user, NameIdentifier.parse(namespace.toString())); + this.namespace = namespace; + } + + /** + * Provides the namespace associated with this event. + * + * @return A {@link Namespace} instance from which tables were listed. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java new file mode 100644 index 00000000000..706fc71d166 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list tables within a namespace fails due + * to an exception. + */ +@DeveloperApi +public final class ListTableFailureEvent extends TableFailureEvent { + private final Namespace namespace; + + /** + * Constructs a {@code ListTableFailureEvent} instance. + * + * @param user The username of the individual who initiated the operation to list tables. + * @param namespace The namespace for which the table listing was attempted. + * @param exception The exception encountered during the attempt to list tables. + */ + public ListTableFailureEvent(String user, Namespace namespace, Exception exception) { + super(user, NameIdentifier.of(namespace.toString()), exception); + this.namespace = namespace; + } + + /** + * Retrieves the namespace associated with this failure event. + * + * @return A {@link Namespace} instance for which the table listing was attempted + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java new file mode 100644 index 00000000000..f3bfb4236dd --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TableInfo; + +/** Represents an event triggered upon the successful loading of a table. */ +@DeveloperApi +public final class LoadTableEvent extends TableEvent { + private final TableInfo loadedTableInfo; + + /** + * Constructs an instance of {@code LoadTableEvent}. + * + * @param user The username of the individual who initiated the table loading. + * @param identifier The unique identifier of the table that was loaded. + * @param tableInfo The state of the table post-loading. + */ + public LoadTableEvent(String user, NameIdentifier identifier, TableInfo tableInfo) { + super(user, identifier); + this.loadedTableInfo = tableInfo; + } + + /** + * Retrieves the state of the table as it was made available to the user after successful loading. + * + * @return A {@link TableInfo} instance encapsulating the details of the table as loaded. + */ + public TableInfo loadedTableInfo() { + return loadedTableInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java new file mode 100644 index 00000000000..b01b8493207 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when an attempt to load a table fails due to an exception. */ +@DeveloperApi +public final class LoadTableFailureEvent extends TableFailureEvent { + /** + * Constructs a {@code LoadTableFailureEvent} instance. + * + * @param user The user who initiated the table loading operation. + * @param identifier The identifier of the table that the loading attempt was made for. + * @param exception The exception that was thrown during the table loading operation, offering + * insight into the issues encountered. + */ + public LoadTableFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java new file mode 100644 index 00000000000..fd3048ee460 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs after a table is successfully purged from the database. */ +@DeveloperApi +public final class PurgeTableEvent extends TableEvent { + private final boolean isExists; + + /** + * Constructs a new {@code PurgeTableEvent} instance. + * + * @param user The user who initiated the purge table operation. + * @param identifier The identifier of the table that was targeted for purging. + * @param isExists A boolean indicator reflecting whether the table was present in the database at + * the time of the purge operation. + */ + public PurgeTableEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the status of the table's existence at the time of the purge operation. + * + * @return A boolean value indicating the table's existence status. {@code true} signifies that + * the table was present before the operation, {@code false} indicates it was not. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java new file mode 100644 index 00000000000..42fddfc0f1c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event triggered when an attempt to purge a table from the database fails due to an + * exception. + */ +@DeveloperApi +public final class PurgeTableFailureEvent extends TableFailureEvent { + /** + * Constructs a new {@code PurgeTableFailureEvent} instance. + * + * @param user The user who initiated the table purge operation. + * @param identifier The identifier of the table intended to be purged. + * @param exception The exception encountered during the table purge operation, providing insights + * into the reasons behind the operation's failure. + */ + public PurgeTableFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java index 238ad0eea4c..19e10d5f229 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java @@ -22,8 +22,7 @@ public abstract class TableEvent extends Event { /** * Constructs a new {@code TableEvent} with the specified user and table identifier. * - * @param user The user responsible for triggering the table operation. This information is - * crucial for auditing and tracking purposes. + * @param user The user responsible for triggering the table operation. * @param identifier The identifier of the table involved in the operation. This encapsulates * details such as the metalake, catalog, schema, and table name. */ diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java index 8b78df964c4..e87d1281fa1 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java @@ -24,11 +24,8 @@ public abstract class TableFailureEvent extends FailureEvent { * Constructs a new {@code TableFailureEvent} instance, capturing information about the failed * table operation. * - * @param user The user associated with the failed table operation. This information helps in - * auditing and understanding the context of the operation that resulted in a failure. - * @param identifier The identifier of the table that was involved in the failed operation. This - * provides a clear reference to the specific table that the operation was attempting to - * modify or interact with. + * @param user The user associated with the failed table operation. + * @param identifier The identifier of the table that was involved in the failed operation. * @param exception The exception that was thrown during the table operation, indicating the cause * of the failure. */ From e946923fabf2133770772fe2eacebf699d6a4a7a Mon Sep 17 00:00:00 2001 From: yuhsinlai <68462169+laiyousin@users.noreply.github.com> Date: Mon, 15 Apr 2024 06:04:36 -0400 Subject: [PATCH 023/106] [#2883] refactor(ITUtils): move assertion functions from AbstractIT for clarity (#2900) ### What changes were proposed in this pull request? Move the two utility-oriented functions, assertionsTableInfo and assertColumn, from AbstractIT.java to ITUtils.java, and make them public. Update all references to these functions accordingly. ### Why are the changes needed? Currently, the AbstractIT class in the Gravitino project contains two functions, assertionsTableInfo and assertColumn, which are more utility-oriented rather than integral to the abstract test class itself. Fix: #2883 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- .../integration/test/CatalogMysqlIT.java | 50 +++++++------- .../integration/test/CatalogPostgreSqlIT.java | 42 ++++++----- .../integration/test/util/AbstractIT.java | 67 ------------------ .../integration/test/util/ITUtils.java | 69 +++++++++++++++++++ 4 files changed, 119 insertions(+), 109 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index c96e402e625..944822488ed 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -351,7 +351,7 @@ void testCreateAndLoadMysqlTable() { Assertions.assertEquals(createdTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], createdTable.columns()[i]); + ITUtils.assertColumn(columns[i], createdTable.columns()[i]); } Table loadTable = tableCatalog.loadTable(tableIdentifier); @@ -364,7 +364,7 @@ void testCreateAndLoadMysqlTable() { } Assertions.assertEquals(loadTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], loadTable.columns()[i]); + ITUtils.assertColumn(columns[i], loadTable.columns()[i]); } } @@ -821,10 +821,10 @@ void testCreateTableIndex() { Distributions.NONE, new SortOrder[0], indexes); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, createdTable); Table table = tableCatalog.loadTable(tableIdentifier); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, "test_failed"); @@ -922,10 +922,10 @@ public void testAutoIncrement() { new SortOrder[0], indexes); // Test create auto increment key success. - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, createdTable); Table table = tableCatalog.loadTable(tableIdentifier); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); // Test alter table. auto increment exist. @@ -942,7 +942,7 @@ public void testAutoIncrement() { col4, col5 }; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(alterColumns), properties, indexes, table); // UpdateColumnComment @@ -957,7 +957,7 @@ public void testAutoIncrement() { col4, col5 }; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(alterColumns), properties, indexes, table); // RenameColumn @@ -977,7 +977,7 @@ public void testAutoIncrement() { Indexes.createMysqlPrimaryKey(new String[][] {{"col_1_1"}, {"col_2"}}), Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}) }; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(alterColumns), properties, indexes, table); tableCatalog.dropTable(tableIdentifier); @@ -1158,22 +1158,26 @@ void testMySQLSpecialTableName() { Table t1 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name)); Arrays.stream(t1.columns()).anyMatch(c -> Objects.equals(c.name(), "t112")); - assertionsTableInfo(t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); + ITUtils.assertionsTableInfo( + t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); Table t2 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name)); Arrays.stream(t2.columns()).anyMatch(c -> Objects.equals(c.name(), "t212")); - assertionsTableInfo(t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); + ITUtils.assertionsTableInfo( + t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); Table t3 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name)); Arrays.stream(t3.columns()).anyMatch(c -> Objects.equals(c.name(), "t_12")); - assertionsTableInfo(t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); + ITUtils.assertionsTableInfo( + t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); Table t4 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name)); Arrays.stream(t4.columns()).anyMatch(c -> Objects.equals(c.name(), "_1__")); - assertionsTableInfo(t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); + ITUtils.assertionsTableInfo( + t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); } @Test @@ -1205,10 +1209,10 @@ void testMySQLTableNameCaseSensitive() { Distributions.NONE, new SortOrder[0], indexes); - assertionsTableInfo( + ITUtils.assertionsTableInfo( "tableName", table_comment, Arrays.asList(newColumns), properties, indexes, createdTable); Table table = tableCatalog.loadTable(tableIdentifier); - assertionsTableInfo( + ITUtils.assertionsTableInfo( "tableName", table_comment, Arrays.asList(newColumns), properties, indexes, table); // Test create table with same name but different case @@ -1230,7 +1234,7 @@ void testMySQLTableNameCaseSensitive() { Assertions.assertEquals("TABLENAME", tableAgain.name()); table = tableCatalog.loadTable(tableIdentifier2); - assertionsTableInfo( + ITUtils.assertionsTableInfo( "TABLENAME", table_comment, Arrays.asList(newColumns), properties, indexes, table); } @@ -1345,7 +1349,7 @@ void testOperationTableIndex() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), Indexes.createMysqlPrimaryKey(new String[][] {{"col_1"}}) }; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); // delete index and add new column and index. @@ -1367,7 +1371,7 @@ void testOperationTableIndex() { tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); Column col4 = Column.of("col_4", Types.VarCharType.of(255), null, true, false, null); newColumns = new Column[] {col1, col2, col3, col4}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); // Add a previously existing index @@ -1387,7 +1391,7 @@ void testOperationTableIndex() { }; table = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); } @@ -1454,7 +1458,7 @@ void testAddColumnAutoIncrement() { Column col6 = Column.of("col_6", Types.LongType.get(), "id", false, true, null); Index[] indices = new Index[] {Indexes.createMysqlPrimaryKey(new String[][] {{"col_6"}})}; newColumns = new Column[] {col1, col2, col3, col4, col5, col6}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indices, table); // Test the auto-increment property of modified fields @@ -1464,7 +1468,7 @@ void testAddColumnAutoIncrement() { col6 = Column.of("col_6", Types.LongType.get(), "id", false, false, null); indices = new Index[] {Indexes.createMysqlPrimaryKey(new String[][] {{"col_6"}})}; newColumns = new Column[] {col1, col2, col3, col4, col5, col6}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indices, table); // Add the auto-increment attribute to the field @@ -1474,7 +1478,7 @@ void testAddColumnAutoIncrement() { col6 = Column.of("col_6", Types.LongType.get(), "id", false, true, null); indices = new Index[] {Indexes.createMysqlPrimaryKey(new String[][] {{"col_6"}})}; newColumns = new Column[] {col1, col2, col3, col4, col5, col6}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indices, table); } @@ -1516,7 +1520,7 @@ void testAddColumnDefaultValue() { Table table = tableCatalog.loadTable(tableIdentifier); newColumns = new Column[] {col1, col2, col3, col4}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index fa88bb405a8..dc88d476c79 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -255,14 +255,14 @@ void testCreateTableWithArrayType() { Assertions.assertEquals(tableName, createdTable.name()); Assertions.assertEquals(columns.length, createdTable.columns().length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], createdTable.columns()[i]); + ITUtils.assertColumn(columns[i], createdTable.columns()[i]); } Table loadTable = tableCatalog.loadTable(tableIdentifier); Assertions.assertEquals(tableName, loadTable.name()); Assertions.assertEquals(columns.length, loadTable.columns().length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], loadTable.columns()[i]); + ITUtils.assertColumn(columns[i], loadTable.columns()[i]); } } @@ -406,7 +406,7 @@ void testCreateAndLoadPostgreSqlTable() { Assertions.assertEquals(createdTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], createdTable.columns()[i]); + ITUtils.assertColumn(columns[i], createdTable.columns()[i]); } Table loadTable = tableCatalog.loadTable(tableIdentifier); @@ -419,7 +419,7 @@ void testCreateAndLoadPostgreSqlTable() { } Assertions.assertEquals(loadTable.columns().length, columns.length); for (int i = 0; i < columns.length; i++) { - assertColumn(columns[i], loadTable.columns()[i]); + ITUtils.assertColumn(columns[i], loadTable.columns()[i]); } } @@ -647,10 +647,10 @@ void testCreateIndexTable() { Distributions.NONE, new SortOrder[0], indexes); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, createdTable); Table table = tableCatalog.loadTable(tableIdentifier); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); // Test create index complex fields fail. @@ -1019,22 +1019,26 @@ void testPGSpecialTableName() { Table t1 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name)); Arrays.stream(t1.columns()).anyMatch(c -> Objects.equals(c.name(), "t112")); - assertionsTableInfo(t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); + ITUtils.assertionsTableInfo( + t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); Table t2 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name)); Arrays.stream(t2.columns()).anyMatch(c -> Objects.equals(c.name(), "t212")); - assertionsTableInfo(t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); + ITUtils.assertionsTableInfo( + t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); Table t3 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name)); Arrays.stream(t3.columns()).anyMatch(c -> Objects.equals(c.name(), "t_12")); - assertionsTableInfo(t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); + ITUtils.assertionsTableInfo( + t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); Table t4 = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name)); Arrays.stream(t4.columns()).anyMatch(c -> Objects.equals(c.name(), "_1__")); - assertionsTableInfo(t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); + ITUtils.assertionsTableInfo( + t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); } @Test @@ -1061,7 +1065,7 @@ void testPGTableNameCaseSensitive() { Distributions.NONE, new SortOrder[0], indexes); - assertionsTableInfo( + ITUtils.assertionsTableInfo( "tablename", "low case table name", Arrays.asList(newColumns), @@ -1069,7 +1073,7 @@ void testPGTableNameCaseSensitive() { indexes, createdTable); Table table = tableCatalog.loadTable(tableIdentifier); - assertionsTableInfo( + ITUtils.assertionsTableInfo( "tablename", "low case table name", Arrays.asList(newColumns), properties, indexes, table); // Test create table with same name but different case @@ -1260,7 +1264,7 @@ void testOperationTableIndex() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), Indexes.primary("pk1_key", new String[][] {{"col_1"}}) }; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); // delete index and add new column and index. @@ -1282,7 +1286,7 @@ void testOperationTableIndex() { tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); Column col4 = Column.of("col_4", Types.VarCharType.of(255), null, true, false, null); newColumns = new Column[] {col1, col2, col3, col4}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); // Add a previously existing index @@ -1302,7 +1306,7 @@ void testOperationTableIndex() { }; table = tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); } @@ -1342,7 +1346,7 @@ void testAddColumnAutoIncrement() { Column col5 = Column.of("col_5", Types.LongType.get(), "id", false, true, null); newColumns = new Column[] {col1, col2, col3, col4, col5}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), @@ -1356,7 +1360,7 @@ void testAddColumnAutoIncrement() { table = tableCatalog.loadTable(tableIdentifier); col5 = Column.of("col_5", Types.LongType.get(), "id", false, false, null); newColumns = new Column[] {col1, col2, col3, col4, col5}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), @@ -1370,7 +1374,7 @@ void testAddColumnAutoIncrement() { table = tableCatalog.loadTable(tableIdentifier); col5 = Column.of("col_5", Types.LongType.get(), "id", false, true, null); newColumns = new Column[] {col1, col2, col3, col4, col5}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), @@ -1418,7 +1422,7 @@ void testAddColumnDefaultValue() { Table table = tableCatalog.loadTable(tableIdentifier); newColumns = new Column[] {col1, col2, col3}; - assertionsTableInfo( + ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 5965f17b125..898a52882fd 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -5,7 +5,6 @@ package com.datastrato.gravitino.integration.test.util; import static com.datastrato.gravitino.Configs.ENTRY_KV_ROCKSDB_BACKEND_PATH; -import static com.datastrato.gravitino.dto.util.DTOConverters.toDTO; import static com.datastrato.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX; import com.datastrato.gravitino.Config; @@ -13,13 +12,8 @@ import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.config.ConfigConstants; -import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; import com.datastrato.gravitino.integration.test.MiniGravitino; import com.datastrato.gravitino.integration.test.MiniGravitinoContext; -import com.datastrato.gravitino.rel.Column; -import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.server.GravitinoServer; import com.datastrato.gravitino.server.ServerConfig; import com.datastrato.gravitino.server.web.JettyServerConfig; @@ -31,16 +25,12 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; @@ -265,61 +255,4 @@ protected String readGitCommitIdFromGitFile() { return ""; } } - - protected static void assertionsTableInfo( - String tableName, - String tableComment, - List columns, - Map properties, - Index[] indexes, - Table table) { - Assertions.assertEquals(tableName, table.name()); - Assertions.assertEquals(tableComment, table.comment()); - Assertions.assertEquals(columns.size(), table.columns().length); - for (int i = 0; i < columns.size(); i++) { - assertColumn(columns.get(i), table.columns()[i]); - } - for (Map.Entry entry : properties.entrySet()) { - Assertions.assertEquals(entry.getValue(), table.properties().get(entry.getKey())); - } - if (ArrayUtils.isNotEmpty(indexes)) { - Assertions.assertEquals(indexes.length, table.index().length); - - Map indexByName = - Arrays.stream(indexes).collect(Collectors.toMap(Index::name, index -> index)); - - for (int i = 0; i < table.index().length; i++) { - Assertions.assertTrue(indexByName.containsKey(table.index()[i].name())); - Assertions.assertEquals( - indexByName.get(table.index()[i].name()).type(), table.index()[i].type()); - for (int j = 0; j < table.index()[i].fieldNames().length; j++) { - for (int k = 0; k < table.index()[i].fieldNames()[j].length; k++) { - Assertions.assertEquals( - indexByName.get(table.index()[i].name()).fieldNames()[j][k], - table.index()[i].fieldNames()[j][k]); - } - } - } - } - } - - protected static void assertColumn(Column expected, Column actual) { - if (!(actual instanceof ColumnDTO)) { - actual = toDTO(actual); - } - if (!(expected instanceof ColumnDTO)) { - expected = toDTO(expected); - } - - Assertions.assertEquals(expected.name(), actual.name()); - Assertions.assertEquals(expected.dataType(), actual.dataType()); - Assertions.assertEquals(expected.nullable(), actual.nullable()); - Assertions.assertEquals(expected.comment(), actual.comment()); - Assertions.assertEquals(expected.autoIncrement(), actual.autoIncrement()); - if (expected.defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET) && expected.nullable()) { - Assertions.assertEquals(LiteralDTO.NULL, actual.defaultValue()); - } else { - Assertions.assertEquals(expected.defaultValue(), actual.defaultValue()); - } - } } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/ITUtils.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/ITUtils.java index 17c219d971d..6f96c4e2a50 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/ITUtils.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/ITUtils.java @@ -4,6 +4,13 @@ */ package com.datastrato.gravitino.integration.test.util; +import static com.datastrato.gravitino.dto.util.DTOConverters.toDTO; + +import com.datastrato.gravitino.dto.rel.ColumnDTO; +import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.indexes.Index; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -11,8 +18,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; +import org.apache.commons.lang3.ArrayUtils; +import org.junit.jupiter.api.Assertions; public class ITUtils { public static final String TEST_MODE = "testMode"; @@ -53,5 +65,62 @@ public static void overwriteConfigFile(String configFileName, Properties props) } } + public static void assertionsTableInfo( + String tableName, + String tableComment, + List columns, + Map properties, + Index[] indexes, + Table table) { + Assertions.assertEquals(tableName, table.name()); + Assertions.assertEquals(tableComment, table.comment()); + Assertions.assertEquals(columns.size(), table.columns().length); + for (int i = 0; i < columns.size(); i++) { + assertColumn(columns.get(i), table.columns()[i]); + } + for (Map.Entry entry : properties.entrySet()) { + Assertions.assertEquals(entry.getValue(), table.properties().get(entry.getKey())); + } + if (ArrayUtils.isNotEmpty(indexes)) { + Assertions.assertEquals(indexes.length, table.index().length); + + Map indexByName = + Arrays.stream(indexes).collect(Collectors.toMap(Index::name, index -> index)); + + for (int i = 0; i < table.index().length; i++) { + Assertions.assertTrue(indexByName.containsKey(table.index()[i].name())); + Assertions.assertEquals( + indexByName.get(table.index()[i].name()).type(), table.index()[i].type()); + for (int j = 0; j < table.index()[i].fieldNames().length; j++) { + for (int k = 0; k < table.index()[i].fieldNames()[j].length; k++) { + Assertions.assertEquals( + indexByName.get(table.index()[i].name()).fieldNames()[j][k], + table.index()[i].fieldNames()[j][k]); + } + } + } + } + } + + public static void assertColumn(Column expected, Column actual) { + if (!(actual instanceof ColumnDTO)) { + actual = toDTO(actual); + } + if (!(expected instanceof ColumnDTO)) { + expected = toDTO(expected); + } + + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(expected.dataType(), actual.dataType()); + Assertions.assertEquals(expected.nullable(), actual.nullable()); + Assertions.assertEquals(expected.comment(), actual.comment()); + Assertions.assertEquals(expected.autoIncrement(), actual.autoIncrement()); + if (expected.defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET) && expected.nullable()) { + Assertions.assertEquals(LiteralDTO.NULL, actual.defaultValue()); + } else { + Assertions.assertEquals(expected.defaultValue(), actual.defaultValue()); + } + } + private ITUtils() {} } From 64bf3394834f3817f64a6bff33a6a3772f16315b Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:01:43 +0800 Subject: [PATCH 024/106] [#2235][part-1] feat(api,core): Add the role entity (#2772) ### What changes were proposed in this pull request? This pull request adds the role entity and one privilege. I avoid to create a large pull request. I will add more privileges in the later pull request. ### Why are the changes needed? Fix: #2235 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? new ut. --------- Co-authored-by: Heng Qin --- .../gravitino/authorization/Privilege.java | 27 ++ .../gravitino/authorization/Privileges.java | 59 ++++ .../gravitino/authorization/Role.java | 51 ++++ .../authorization/SecurableObject.java | 39 +++ .../authorization/SecurableObjects.java | 189 ++++++++++++ .../exceptions/NoSuchRoleException.java | 34 +++ .../RoleAlreadyExistsException.java | 34 +++ .../catalog/hive/HiveTableOperations.java | 5 + .../java/com/datastrato/gravitino/Entity.java | 7 + .../authorization/AccessControlManager.java | 56 +++- .../gravitino/authorization/AdminManager.java | 2 +- .../gravitino/authorization/RoleManager.java | 139 +++++++++ .../authorization/UserGroupManager.java | 2 +- .../gravitino/catalog/CatalogManager.java | 4 + .../catalog/FilesetOperationDispatcher.java | 5 + .../catalog/SchemaOperationDispatcher.java | 5 + .../catalog/TableOperationDispatcher.java | 5 + .../catalog/TopicOperationDispatcher.java | 5 + .../datastrato/gravitino/meta/RoleEntity.java | 274 ++++++++++++++++++ .../gravitino/proto/ProtoEntitySerDe.java | 7 +- .../gravitino/proto/RoleEntitySerDe.java | 84 ++++++ .../storage/kv/BinaryEntityKeyEncoder.java | 3 + .../gravitino/storage/kv/KvEntityStore.java | 4 +- .../TestAccessControlManager.java | 70 +++++ .../datastrato/gravitino/meta/TestEntity.java | 39 +++ .../gravitino/proto/TestEntityProtoSerDe.java | 31 ++ .../gravitino/storage/TestEntityStorage.java | 28 +- meta/src/main/proto/gravitino_meta.proto | 11 +- 28 files changed, 1212 insertions(+), 7 deletions(-) create mode 100644 api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java create mode 100644 api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java create mode 100644 api/src/main/java/com/datastrato/gravitino/authorization/Role.java create mode 100644 api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java create mode 100644 api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java create mode 100644 api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java create mode 100644 api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java create mode 100644 core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java create mode 100644 core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java create mode 100644 core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java new file mode 100644 index 00000000000..d27bde422e5 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import com.datastrato.gravitino.annotation.Evolving; + +/** + * The interface of a privilege. The privilege represents the ability to execute kinds of operations + * for kinds of entities + */ +@Evolving +public interface Privilege { + + /** @return The generic name of the privilege. */ + Name name(); + + /** @return A readable string representation for the privilege. */ + String simpleString(); + + /** The name of this privilege. */ + enum Name { + /** The privilege of load a catalog. */ + LOAD_CATALOG + } +} diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java new file mode 100644 index 00000000000..9eb4f3f9ef8 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +/** The helper class for {@link Privilege}. */ +public class Privileges { + + /** + * Returns the Privilege from the string representation. + * + * @param privilege The string representation of the privilege. + * @return The Privilege. + */ + public static Privilege fromString(String privilege) { + Privilege.Name name = Privilege.Name.valueOf(privilege); + return fromName(name); + } + + /** + * Returns the Privilege from the `Privilege.Name`. + * + * @param name The `Privilege.Name` of the privilege. + * @return The Privilege. + */ + public static Privilege fromName(Privilege.Name name) { + switch (name) { + case LOAD_CATALOG: + return LoadCatalog.get(); + default: + throw new IllegalArgumentException("Don't support the privilege: " + name); + } + } + + /** The privilege of load a catalog. */ + public static class LoadCatalog implements Privilege { + private static final LoadCatalog INSTANCE = new LoadCatalog(); + + /** @return The instance of the privilege. */ + public static LoadCatalog get() { + return INSTANCE; + } + + private LoadCatalog() {} + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return Name.LOAD_CATALOG; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "load catalog"; + } + } +} diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Role.java b/api/src/main/java/com/datastrato/gravitino/authorization/Role.java new file mode 100644 index 00000000000..914fb7159c6 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Role.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import com.datastrato.gravitino.Auditable; +import com.datastrato.gravitino.annotation.Evolving; +import java.util.List; +import java.util.Map; + +/** + * The interface of a role. The role is the entity which has kinds of privileges. One role can have + * multiple privileges of one securable object. Gravitino chooses to bind one securable object to + * one role to avoid granting too many privileges to one role. + */ +@Evolving +public interface Role extends Auditable { + + /** + * The name of the role. + * + * @return The name of the role. + */ + String name(); + + /** + * The properties of the role. Note, this method will return null if the properties are not set. + * + * @return The properties of the role. + */ + Map properties(); + + /** + * The privileges of the role. All privileges belong to one securable object. For example: If the + * securable object is a table, the privileges could be `READ TABLE`, `WRITE TABLE`, etc. If a + * schema has the privilege of `LOAD TABLE`. It means the role can all tables of the schema. + * + * @return The privileges of the role. + */ + List privileges(); + + /** + * The securable object represents a special kind of entity with a unique identifier. All + * securable objects are organized by tree structure. For example: If the securable object is a + * table, the identifier may be `catalog1.schema1.table1`. + * + * @return The securable object of the role. + */ + SecurableObject securableObject(); +} diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java new file mode 100644 index 00000000000..b3d8aa5ce2c --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import com.datastrato.gravitino.annotation.Evolving; +import javax.annotation.Nullable; + +/** + * The securable object is the entity which access can be granted. Unless allowed by a grant, access + * is denied. Gravitino organizes the securable objects using tree structure. The securable object + * may be a catalog, a table or a schema, etc. For example, `catalog1.schema1.table1` represents a + * table named `table1`. It's in the schema named `schema1`. The schema is in the catalog named + * `catalog1`. Similarly, `catalog1.schema1.topic1` can represent a topic. + * `catalog1.schema1.fileset1` can represent a fileset. `*` represents all the catalogs. If you want + * to use other securable objects which represents all entities," you can use their parent entity, + * For example if you want to have read table privileges of all tables of `catalog1.schema1`, " you + * can use add `read table` privilege for `catalog1.schema1` directly + */ +@Evolving +public interface SecurableObject { + + /** + * The parent securable object. If the securable object doesn't have parent, this method will + * return null. + * + * @return The parent securable object. + */ + @Nullable + SecurableObject parent(); + + /** + * The name of th securable object. + * + * @return The name of the securable object. + */ + String name(); +} diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java new file mode 100644 index 00000000000..48b05f6ef6f --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java @@ -0,0 +1,189 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import java.util.Objects; + +/** The helper class for {@link SecurableObject}. */ +public class SecurableObjects { + + /** + * Create the {@link SecurableObject} with the given names. + * + * @param names The names of the securable object. + * @return The created {@link SecurableObject} + */ + public static SecurableObject of(String... names) { + if (names == null) { + throw new IllegalArgumentException("Cannot create a securable object with null names"); + } + + if (names.length == 0) { + throw new IllegalArgumentException("Cannot create a securable object with no names"); + } + + SecurableObject parent = null; + for (String name : names) { + if (name == null) { + throw new IllegalArgumentException("Cannot create a securable object with null name"); + } + + if (name.equals("*")) { + throw new IllegalArgumentException( + "Cannot create a securable object with `*` name. If you want to use a securable object which represents all catalogs," + + " you use the method `ofAllCatalogs`." + + " If you want to create an another securable object which represents all entities," + + " you can use its parent entity, For example," + + " if you want to have read table privileges of all tables of `catalog1.schema1`," + + " you can use add `read table` privilege for `catalog1.schema1` directly"); + } + + parent = new SecurableObjectImpl(parent, name); + } + + return parent; + } + + /** + * Create the catalog {@link SecurableObject} with the given catalog name. + * + * @param catalog The catalog name + * @return The created catalog {@link SecurableObject} + */ + public static SecurableObject ofCatalog(String catalog) { + return of(catalog); + } + + /** + * Create the schema {@link SecurableObject} with the given securable catalog object and schema + * name. + * + * @param catalog The securable catalog object + * @param schema The schema name + * @return The created schema {@link SecurableObject} + */ + public static SecurableObject ofSchema(SecurableObject catalog, String schema) { + checkCatalog(catalog); + + return of(catalog.name(), schema); + } + + /** + * Create the table {@link SecurableObject} with the given securable schema object and table name. + * + * @param schema The securable schema object + * @param table The table name + * @return The created table {@link SecurableObject} + */ + public static SecurableObject ofTable(SecurableObject schema, String table) { + checkSchema(schema); + + return of(schema.parent().name(), schema.name(), table); + } + + /** + * Create the topic {@link SecurableObject} with the given securable schema object and topic name. + * + * @param schema The securable schema object + * @param topic The topic name + * @return The created topic {@link SecurableObject} + */ + public static SecurableObject ofTopic(SecurableObject schema, String topic) { + checkSchema(schema); + + return of(schema.parent().name(), schema.name(), topic); + } + + /** + * Create the table {@link SecurableObject} with the given securable schema object and fileset + * name. + * + * @param schema The securable schema object + * @param fileset The fileset name + * @return The created fileset {@link SecurableObject} + */ + public static SecurableObject ofFileset(SecurableObject schema, String fileset) { + checkSchema(schema); + + return of(schema.parent().name(), schema.name(), fileset); + } + + /** + * All catalogs is a special securable object .You can give the securable object the privileges + * `LOAD CATALOG`, `CREATE CATALOG`, etc. It means that you can load any catalog and create any + * which doesn't exist. + * + * @return The created {@link SecurableObject} + */ + public static SecurableObject ofAllCatalogs() { + return ALL_CATALOGS; + } + + private static void checkSchema(SecurableObject schema) { + if (schema == null) { + throw new IllegalArgumentException("Securable schema object can't be null"); + } + checkCatalog(schema.parent()); + } + + private static void checkCatalog(SecurableObject catalog) { + if (catalog == null) { + throw new IllegalArgumentException("Securable catalog object can't be null"); + } + + if (catalog.parent() != null) { + throw new IllegalArgumentException( + String.format("The parent of securable catalog object %s must be null", catalog.name())); + } + } + + private static final SecurableObject ALL_CATALOGS = new SecurableObjectImpl(null, "*"); + + private static class SecurableObjectImpl implements SecurableObject { + + private final SecurableObject parent; + private final String name; + + SecurableObjectImpl(SecurableObject parent, String name) { + this.parent = parent; + this.name = name; + } + + @Override + public SecurableObject parent() { + return parent; + } + + @Override + public String name() { + return name; + } + + @Override + public int hashCode() { + return Objects.hash(parent, name); + } + + @Override + public String toString() { + if (parent != null) { + return parent.toString() + "." + name; + } else { + return name; + } + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof SecurableObject)) { + return false; + } + + SecurableObject otherSecurableObject = (SecurableObject) other; + return Objects.equals(parent, otherSecurableObject.parent()) + && Objects.equals(name, otherSecurableObject.name()); + } + } +} diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java new file mode 100644 index 00000000000..0ca9decfb65 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a role with specified name is not existed. */ +public class NoSuchRoleException extends NotFoundException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchRoleException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NoSuchRoleException(Throwable cause, String message, Object... args) { + super(cause, message, args); + } +} diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java new file mode 100644 index 00000000000..d64dc7ff3ce --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a role with specified name already exists. */ +public class RoleAlreadyExistsException extends AlreadyExistsException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public RoleAlreadyExistsException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public RoleAlreadyExistsException(Throwable cause, @FormatString String message, Object... args) { + super(cause, message, args); + } +} diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java index 2cdc8155553..8f2388753f6 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.catalog.hive; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.connector.TableOperations; import com.datastrato.gravitino.exceptions.NoSuchPartitionException; import com.datastrato.gravitino.exceptions.NoSuchTableException; @@ -132,6 +133,10 @@ private String[][] getFieldNames(String partitionName) { @Override public Partition addPartition(Partition partition) throws PartitionAlreadyExistsException { + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(partition.name())) { + throw new IllegalArgumentException("Can't create a catalog with with reserved partition `*`"); + } + Preconditions.checkArgument( partition instanceof IdentityPartition, "Hive only supports identity partition"); IdentityPartition identityPartition = (IdentityPartition) partition; diff --git a/core/src/main/java/com/datastrato/gravitino/Entity.java b/core/src/main/java/com/datastrato/gravitino/Entity.java index af2c1872000..d8d6ab03cd7 100644 --- a/core/src/main/java/com/datastrato/gravitino/Entity.java +++ b/core/src/main/java/com/datastrato/gravitino/Entity.java @@ -23,6 +23,9 @@ public interface Entity extends Serializable { /** The system reserved catalog name. */ String SYSTEM_CATALOG_RESERVED_NAME = "system"; + /** The securable object reserved entity name. */ + String SECURABLE_ENTITY_RESERVED_NAME = "*"; + /** The authorization catalog name in the system metalake. */ String AUTHORIZATION_CATALOG_NAME = "authorization"; @@ -31,6 +34,8 @@ public interface Entity extends Serializable { /** The group schema name in the system catalog. */ String GROUP_SCHEMA_NAME = "group"; + /** The role schema name in the system catalog. */ + String ROLE_SCHEMA_NAME = "role"; /** The admin schema name in the authorization catalog of the system metalake. */ String ADMIN_SCHEMA_NAME = "admin"; @@ -47,6 +52,7 @@ enum EntityType { TOPIC("to", 6), USER("us", 7), GROUP("gr", 8), + ROLE("ro", 9), AUDIT("au", 65534); @@ -89,6 +95,7 @@ public static List getParentEntityTypes(EntityType entityType) { case TOPIC: case USER: case GROUP: + case ROLE: return ImmutableList.of(METALAKE, CATALOG, SCHEMA); case COLUMN: return ImmutableList.of(METALAKE, CATALOG, SCHEMA, TABLE); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index 44882f3ce82..fcfc911d883 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -8,10 +8,14 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.Executable; +import java.util.List; +import java.util.Map; /** * AccessControlManager is used for manage users, roles, admin, grant information, this class is an @@ -25,12 +29,14 @@ public class AccessControlManager { private final UserGroupManager userGroupManager; private final AdminManager adminManager; + private final RoleManager roleManager; private final Object adminOperationLock = new Object(); private final Object nonAdminOperationLock = new Object(); public AccessControlManager(EntityStore store, IdGenerator idGenerator, Config config) { this.userGroupManager = new UserGroupManager(store, idGenerator); this.adminManager = new AdminManager(store, idGenerator, config); + this.roleManager = new RoleManager(store, idGenerator); } /** @@ -50,7 +56,7 @@ public User addUser(String metalake, String name) throws UserAlreadyExistsExcept * Removes a User. * * @param metalake The Metalake of the User. - * @param user THe name of the User. + * @param user The name of the User. * @return `true` if the User was successfully removed, `false` otherwise. * @throws RuntimeException If removing the User encounters storage issues. */ @@ -152,6 +158,54 @@ public boolean isMetalakeAdmin(String user) { return doWithAdminLock(() -> adminManager.isMetalakeAdmin(user)); } + /** + * Creates a new Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @param properties The properties of the Role. + * @param securableObject The securable object of the Role. + * @param privileges The privileges of the Role. + * @return The created Role instance. + * @throws RoleAlreadyExistsException If a Role with the same identifier already exists. + * @throws RuntimeException If creating the Role encounters storage issues. + */ + public Role createRole( + String metalake, + String role, + Map properties, + SecurableObject securableObject, + List privileges) + throws RoleAlreadyExistsException { + return doWithNonAdminLock( + () -> roleManager.createRole(metalake, role, properties, securableObject, privileges)); + } + + /** + * Loads a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return The loading Role instance. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RuntimeException If loading the Role encounters storage issues. + */ + public Role loadRole(String metalake, String role) throws NoSuchRoleException { + return doWithNonAdminLock(() -> roleManager.loadRole(metalake, role)); + } + + /** + * Drops a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return `true` if the Role was successfully dropped, `false` otherwise. + * @throws RuntimeException If dropping the User encounters storage issues. + */ + public boolean dropRole(String metalake, String role) { + return doWithNonAdminLock(() -> roleManager.dropRole(metalake, role)); + } + private R doWithNonAdminLock(Executable executable) throws E { synchronized (nonAdminOperationLock) { return executable.execute(); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java index 55a3940da59..edbe18cc2c0 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java @@ -30,7 +30,7 @@ * metalake or drops its metalake. The metalake admin will be responsible for managing the access * control. AdminManager operates underlying store using the lock because kv storage needs the lock. */ -public class AdminManager { +class AdminManager { private static final Logger LOG = LoggerFactory.getLogger(AdminManager.class); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java new file mode 100644 index 00000000000..5813744af18 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -0,0 +1,139 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.authorization; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.storage.IdGenerator; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * RoleManager is responsible for managing the roles. Role contains the privileges of one privilege + * entity. If one Role is created and the privilege entity is an external system, the role will be + * created in the underlying entity, too. + */ +class RoleManager { + + private static final Logger LOG = LoggerFactory.getLogger(RoleManager.class); + private static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; + private final EntityStore store; + private final IdGenerator idGenerator; + + public RoleManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + /** + * Creates a new Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @param properties The properties of the Role. + * @param securableObject The securable object of the Role. + * @param privileges The privileges of the Role. + * @return The created Role instance. + * @throws RoleAlreadyExistsException If a Role with the same identifier already exists. + * @throws RuntimeException If creating the Role encounters storage issues. + */ + public Role createRole( + String metalake, + String role, + Map properties, + SecurableObject securableObject, + List privileges) + throws RoleAlreadyExistsException { + AuthorizationUtils.checkMetalakeExists(store, metalake); + RoleEntity roleEntity = + RoleEntity.builder() + .withId(idGenerator.nextId()) + .withName(role) + .withProperties(properties) + .securableObject(securableObject) + .withPrivileges(privileges) + .withNamespace( + Namespace.of( + metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withAuditInfo( + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .build()) + .build(); + try { + store.put(roleEntity, false /* overwritten */); + return roleEntity; + } catch (EntityAlreadyExistsException e) { + LOG.warn("Role {} in the metalake {} already exists", role, metalake, e); + throw new RoleAlreadyExistsException( + "Role %s in the metalake %s already exists", role, metalake); + } catch (IOException ioe) { + LOG.error( + "Creating role {} failed in the metalake {} due to storage issues", role, metalake, ioe); + throw new RuntimeException(ioe); + } + } + + /** + * Loads a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return The loading Role instance. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RuntimeException If loading the Role encounters storage issues. + */ + public Role loadRole(String metalake, String role) throws NoSuchRoleException { + try { + AuthorizationUtils.checkMetalakeExists(store, metalake); + return store.get(ofRole(metalake, role), Entity.EntityType.ROLE, RoleEntity.class); + } catch (NoSuchEntityException e) { + LOG.warn("Role {} does not exist in the metalake {}", role, metalake, e); + throw new NoSuchRoleException(ROLE_DOES_NOT_EXIST_MSG, role, metalake); + } catch (IOException ioe) { + LOG.error("Loading role {} failed due to storage issues", role, ioe); + throw new RuntimeException(ioe); + } + } + + /** + * Drops a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return `true` if the Role was successfully dropped, `false` otherwise. + * @throws RuntimeException If dropping the User encounters storage issues. + */ + public boolean dropRole(String metalake, String role) { + try { + AuthorizationUtils.checkMetalakeExists(store, metalake); + return store.delete(ofRole(metalake, role), Entity.EntityType.ROLE); + } catch (IOException ioe) { + LOG.error( + "Deleting role {} in the metalake {} failed due to storage issues", role, metalake, ioe); + throw new RuntimeException(ioe); + } + } + + private NameIdentifier ofRole(String metalake, String role) { + return NameIdentifier.of( + metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME, role); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java index 967c55a9697..411c640a84e 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java @@ -32,7 +32,7 @@ * metalake and the user or group. Metalake is like a concept of the organization. `AddUser` or * `AddGroup` means that a role or user enters an organization. */ -public class UserGroupManager { +class UserGroupManager { private static final Logger LOG = LoggerFactory.getLogger(UserGroupManager.class); private static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index f374ce6e204..9c4419206e5 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -293,6 +293,10 @@ public Catalog createCatalog( throw new IllegalArgumentException("Can't create a catalog with with reserved name `system`"); } + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { + throw new IllegalArgumentException("Can't create a catalog with with reserved name `*`"); + } + // load catalog-related configuration from catalog-specific configuration file Map catalogSpecificConfig = loadCatalogSpecificConfig(properties, provider); Map mergedConfig = mergeConf(properties, catalogSpecificConfig); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java index 4d9166c8109..892b444e1fc 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java @@ -6,6 +6,7 @@ import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -100,6 +101,10 @@ public Fileset createFileset( Map properties) throws NoSuchSchemaException, FilesetAlreadyExistsException { NameIdentifier catalogIdent = getCatalogIdentifier(ident); + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { + throw new IllegalArgumentException("Can't create a fileset with with reserved name `*`"); + } + doWithCatalog( catalogIdent, c -> diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index ed750ce976f..20a952f0ae4 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -7,6 +7,7 @@ import static com.datastrato.gravitino.Entity.EntityType.SCHEMA; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -74,6 +75,10 @@ public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogExc @Override public Schema createSchema(NameIdentifier ident, String comment, Map properties) throws NoSuchCatalogException, SchemaAlreadyExistsException { + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { + throw new IllegalArgumentException("Can't create a schema with with reserved name `*`"); + } + NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index 1d58fb0d9ab..4939ec801cf 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -8,6 +8,7 @@ import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -133,6 +134,10 @@ public Table createTable( SortOrder[] sortOrders, Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException { + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { + throw new IllegalArgumentException("Can't create a table with with reserved name `*`"); + } + NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java index 172c11b86c1..26c59627196 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java @@ -8,6 +8,7 @@ import static com.datastrato.gravitino.StringIdentifier.fromProperties; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -118,6 +119,10 @@ public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { public Topic createTopic( NameIdentifier ident, String comment, DataLayout dataLayout, Map properties) throws NoSuchSchemaException, TopicAlreadyExistsException { + if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { + throw new IllegalArgumentException("Can't create a topic with with reserved name `*`"); + } + NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java new file mode 100644 index 00000000000..5e4948fa0b8 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -0,0 +1,274 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.meta; + +import com.datastrato.gravitino.Auditable; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.Field; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.authorization.Privilege; +import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObject; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class RoleEntity implements Role, Entity, Auditable, HasIdentifier { + + public static final Field ID = + Field.required("id", Long.class, " The unique id of the role entity."); + + public static final Field NAME = + Field.required("name", String.class, "The name of the role entity."); + + public static final Field PROPERTIES = + Field.optional("properties", Map.class, "The properties of the role entity."); + + public static final Field AUDIT_INFO = + Field.required("audit_info", AuditInfo.class, "The audit details of the role entity."); + + public static final Field SECURABLE_OBJECT = + Field.required( + "securable_object", SecurableObject.class, "The securable object of the role entity."); + + public static final Field PRIVILEGES = + Field.required("privileges", List.class, "The privileges of the role entity."); + + private Long id; + private String name; + private Map properties; + private AuditInfo auditInfo; + private List privileges; + private Namespace namespace; + private SecurableObject securableObject; + + /** + * The name of the role. + * + * @return The name of the role. + */ + @Override + public String name() { + return name; + } + + /** @return The audit information of the entity. */ + @Override + public AuditInfo auditInfo() { + return auditInfo; + } + + /** + * The properties of the role. Note, this method will return null if the properties are not set. + * + * @return The properties of the role. + */ + @Override + public Map properties() { + return properties; + } + + /** + * The securable object represents a special kind of entity with a unique identifier. All + * securable objects are organized by tree structure. For example: If the securable object is a + * table, the identifier may be `catalog1.schema1.table1`. + * + * @return The securable object of the role. + */ + @Override + public SecurableObject securableObject() { + // The securable object is a special kind of entities. Some entity types aren't the securable + // object, such as + // User, Role, etc. + // The securable object identifier must be unique. + // Gravitino assumes that the identifiers of the entities may be the same if they have different + // types. + // So one type of them can't be the securable object at least if there are the two same + // identifier + // entities . + return securableObject; + } + + /** + * The privileges of the role. All privileges belong to one securable object. For example: If the + * securable object is a table, the privileges could be `READ TABLE`, `WRITE TABLE`, etc. If a + * schema has the privilege of `LOAD TABLE`. It means the role can all tables of the schema. + * + * @return The privileges of the role. + */ + @Override + public List privileges() { + return privileges; + } + + /** + * Retrieves the fields and their associated values of the entity. + * + * @return A map of Field to Object representing the entity's schema with values. + */ + @Override + public Map fields() { + Map fields = Maps.newHashMap(); + fields.put(ID, id); + fields.put(NAME, name); + fields.put(AUDIT_INFO, auditInfo); + fields.put(PROPERTIES, properties); + fields.put(SECURABLE_OBJECT, securableObject); + fields.put(PRIVILEGES, privileges); + + return Collections.unmodifiableMap(fields); + } + + /** + * Retrieves the type of the entity. + * + * @return The type of the entity as defined by {@link EntityType}. + */ + @Override + public EntityType type() { + return EntityType.ROLE; + } + + /** + * Get the unique id of the entity. + * + * @return The unique id of the entity. + */ + @Override + public Long id() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof RoleEntity)) return false; + + RoleEntity that = (RoleEntity) o; + return Objects.equals(id, that.id) + && Objects.equals(name, that.name) + && Objects.equals(auditInfo, that.auditInfo) + && Objects.equals(properties, that.properties) + && Objects.equals(securableObject, that.securableObject) + && Objects.equals(privileges, that.privileges); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, properties, auditInfo, securableObject, privileges); + } + + /** + * Get the namespace of the entity. + * + * @return The namespace of the entity. + */ + @Override + public Namespace namespace() { + return namespace; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final RoleEntity roleEntity; + + private Builder() { + roleEntity = new RoleEntity(); + } + + /** + * Sets the unique id of the role entity. + * + * @param id The unique id of the role entity. + * @return The builder instance. + */ + public Builder withId(Long id) { + roleEntity.id = id; + return this; + } + + /** + * Sets the name of the role entity. + * + * @param name The name of the role entity. + * @return The builder instance. + */ + public Builder withName(String name) { + roleEntity.name = name; + return this; + } + + /** + * Sets the properties of the role entity. + * + * @param properties The properties of the role entity. + * @return The builder instance. + */ + public Builder withProperties(Map properties) { + roleEntity.properties = properties; + return this; + } + + /** + * Sets the audit details of the role entity. + * + * @param auditInfo The audit details of the role entity. + * @return The builder instance. + */ + public Builder withAuditInfo(AuditInfo auditInfo) { + roleEntity.auditInfo = auditInfo; + return this; + } + + /** + * Sets the securable object of the role entity. + * + * @param securableObject The securable object of the role entity. + * @return The builder instance. + */ + public Builder securableObject(SecurableObject securableObject) { + roleEntity.securableObject = securableObject; + return this; + } + + /** + * Sets the privileges of the role entity. + * + * @param privileges The privileges of the role entity. + * @return The builder instance. + */ + public Builder withPrivileges(List privileges) { + roleEntity.privileges = privileges; + return this; + } + + /** + * Sets the namespace of the role entity. + * + * @param namespace The namespace of the role entity. + * @return The builder instance. + */ + public Builder withNamespace(Namespace namespace) { + roleEntity.namespace = namespace; + return this; + } + + /** + * Builds the role entity. + * + * @return The built role entity. + */ + public RoleEntity build() { + roleEntity.validate(); + return roleEntity; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index 7da49a85da4..b826b4cedb3 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -46,6 +46,9 @@ public class ProtoEntitySerDe implements EntitySerDe { .put( "com.datastrato.gravitino.meta.GroupEntity", "com.datastrato.gravitino.proto.GroupEntitySerDe") + .put( + "com.datastrato.gravitino.meta.RoleEntity", + "com.datastrato.gravitino.proto.RoleEntitySerDe") .build(); private static final Map ENTITY_TO_PROTO = @@ -67,7 +70,9 @@ public class ProtoEntitySerDe implements EntitySerDe { "com.datastrato.gravitino.meta.UserEntity", "com.datastrato.gravitino.proto.User", "com.datastrato.gravitino.meta.GroupEntity", - "com.datastrato.gravitino.proto.Group"); + "com.datastrato.gravitino.proto.Group", + "com.datastrato.gravitino.meta.RoleEntity", + "com.datastrato.gravitino.proto.Role"); private final Map, ProtoSerDe> entityToSerDe; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java new file mode 100644 index 00000000000..ae65c3a117c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.proto; + +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObject; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.meta.RoleEntity; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +public class RoleEntitySerDe implements ProtoSerDe { + + private static final Splitter DOT = Splitter.on('.'); + + /** + * Serializes the provided entity into its corresponding Protocol Buffer message representation. + * + * @param roleEntity The entity to be serialized. + * @return The Protocol Buffer message representing the serialized entity. + */ + @Override + public Role serialize(RoleEntity roleEntity) { + Role.Builder builder = + Role.newBuilder() + .setId(roleEntity.id()) + .setName(roleEntity.name()) + .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())) + .addAllPrivileges( + roleEntity.privileges().stream() + .map(privilege -> privilege.name().toString()) + .collect(Collectors.toList())) + .setSecurableObject(roleEntity.securableObject().toString()); + + if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { + builder.putAllProperties(roleEntity.properties()); + } + + return builder.build(); + } + + /** + * Deserializes the provided Protocol Buffer message into its corresponding entity representation. + * + * @param role The Protocol Buffer message to be deserialized. + * @return The entity representing the deserialized Protocol Buffer message. + */ + @Override + public RoleEntity deserialize(Role role) { + RoleEntity.Builder builder = + RoleEntity.builder() + .withId(role.getId()) + .withName(role.getName()) + .withPrivileges( + role.getPrivilegesList().stream() + .map(Privileges::fromString) + .collect(Collectors.toList())) + .securableObject(parseSecurableObject(role.getSecurableObject())) + .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo())); + + if (!role.getPropertiesMap().isEmpty()) { + builder.withProperties(role.getPropertiesMap()); + } + + return builder.build(); + } + + private static SecurableObject parseSecurableObject(String securableObjectIdentifier) { + if ("*".equals(securableObjectIdentifier)) { + return SecurableObjects.ofAllCatalogs(); + } + + if (StringUtils.isBlank(securableObjectIdentifier)) { + throw new IllegalArgumentException("securable object identifier can't be blank"); + } + + Iterable parts = DOT.split(securableObjectIdentifier); + return SecurableObjects.of(Iterables.toArray(parts, String.class)); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java index 773b3769486..49fea351a69 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java @@ -8,6 +8,7 @@ import static com.datastrato.gravitino.Entity.EntityType.FILESET; import static com.datastrato.gravitino.Entity.EntityType.GROUP; import static com.datastrato.gravitino.Entity.EntityType.METALAKE; +import static com.datastrato.gravitino.Entity.EntityType.ROLE; import static com.datastrato.gravitino.Entity.EntityType.SCHEMA; import static com.datastrato.gravitino.Entity.EntityType.TABLE; import static com.datastrato.gravitino.Entity.EntityType.TOPIC; @@ -82,6 +83,8 @@ public class BinaryEntityKeyEncoder implements EntityKeyEncoder { new String[] {USER.getShortName() + "/", "/", "/", "/"}, GROUP, new String[] {GROUP.getShortName() + "/", "/", "/", "/"}, + ROLE, + new String[] {ROLE.getShortName() + "/", "/", "/", "/"}, TOPIC, new String[] {TOPIC.getShortName() + "/", "/", "/", "/"}); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 6ede9954145..6222c9d60c9 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -8,6 +8,7 @@ import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE; import static com.datastrato.gravitino.Entity.EntityType.GROUP; import static com.datastrato.gravitino.Entity.EntityType.METALAKE; +import static com.datastrato.gravitino.Entity.EntityType.ROLE; import static com.datastrato.gravitino.Entity.EntityType.USER; import static com.datastrato.gravitino.storage.kv.BinaryEntityEncoderUtil.generateKeyForMapping; import static com.datastrato.gravitino.storage.kv.BinaryEntityEncoderUtil.getSubEntitiesPrefix; @@ -228,7 +229,8 @@ void deleteAuthorizationEntitiesIfNecessary(NameIdentifier ident, EntityType typ } byte[] encode = entityKeyEncoder.encode(ident, type, true); - String[] entityShortNames = new String[] {USER.getShortName(), GROUP.getShortName()}; + String[] entityShortNames = + new String[] {USER.getShortName(), GROUP.getShortName(), ROLE.getShortName()}; for (String name : entityShortNames) { byte[] prefix = replacePrefixTypeInfo(encode, name); transactionalKvBackend.deleteRange( diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java index 5c708c99094..c53ada3a35f 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java @@ -8,19 +8,24 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.storage.RandomIdGenerator; import com.datastrato.gravitino.storage.memory.TestMemoryEntityStore; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; +import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -207,4 +212,69 @@ public void testServiceAdmin() { Assertions.assertTrue(accessControlManager.isServiceAdmin("admin2")); Assertions.assertFalse(accessControlManager.isServiceAdmin("admin3")); } + + @Test + public void testCreateRole() { + Map props = ImmutableMap.of("key1", "value1"); + + Role role = + accessControlManager.createRole( + "metalake", "create", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + Assertions.assertEquals("create", role.name()); + testProperties(props, role.properties()); + + // Test with RoleAlreadyExistsException + Assertions.assertThrows( + RoleAlreadyExistsException.class, + () -> + accessControlManager.createRole( + "metalake", + "create", + props, + SecurableObjects.ofAllCatalogs(), + Lists.newArrayList())); + } + + @Test + public void testLoadRole() { + Map props = ImmutableMap.of("k1", "v1"); + + accessControlManager.createRole( + "metalake", "loadRole", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + Role role = accessControlManager.loadRole("metalake", "loadRole"); + Assertions.assertEquals("loadRole", role.name()); + testProperties(props, role.properties()); + + // Test load non-existed group + Throwable exception = + Assertions.assertThrows( + NoSuchRoleException.class, + () -> accessControlManager.loadRole("metalake", "not-exist")); + Assertions.assertTrue(exception.getMessage().contains("Role not-exist does not exist")); + } + + @Test + public void testDropRole() { + Map props = ImmutableMap.of("k1", "v1"); + + accessControlManager.createRole( + "metalake", "testDrop", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + + // Test drop role + boolean dropped = accessControlManager.dropRole("metalake", "testDrop"); + Assertions.assertTrue(dropped); + + // Test drop non-existed role + boolean dropped1 = accessControlManager.dropRole("metalake", "no-exist"); + Assertions.assertFalse(dropped1); + } + + private void testProperties(Map expectedProps, Map testProps) { + expectedProps.forEach( + (k, v) -> { + Assertions.assertEquals(v, testProps.get(k)); + }); + + Assertions.assertFalse(testProps.containsKey(StringIdentifier.ID_KEY)); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java index 6158574a57d..5d1f689a6ae 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java @@ -6,6 +6,8 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Field; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.file.Fileset; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -56,6 +58,10 @@ public class TestEntity { private final Long groupId = 1L; private final String groupName = "testGroup"; + // Role test data + private final Long roleId = 1L; + private final String roleName = "testRole"; + @Test public void testMetalake() { BaseMetalake metalake = @@ -252,4 +258,37 @@ public void testGroup() { Assertions.assertNull(groupWithoutFields.roles()); } + + @Test + public void testRole() { + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(roleName) + .withAuditInfo(auditInfo) + .securableObject(SecurableObjects.of(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withProperties(map) + .build(); + + Map fields = role.fields(); + Assertions.assertEquals(roleId, fields.get(RoleEntity.ID)); + Assertions.assertEquals(roleName, fields.get(RoleEntity.NAME)); + Assertions.assertEquals(auditInfo, fields.get(RoleEntity.AUDIT_INFO)); + Assertions.assertEquals(map, fields.get(RoleEntity.PROPERTIES)); + Assertions.assertEquals( + Lists.newArrayList(Privileges.LoadCatalog.get()), fields.get(RoleEntity.PRIVILEGES)); + Assertions.assertEquals( + SecurableObjects.of(catalogName), fields.get(RoleEntity.SECURABLE_OBJECT)); + + RoleEntity roleWithoutFields = + RoleEntity.builder() + .withId(1L) + .withName(roleName) + .withAuditInfo(auditInfo) + .securableObject(SecurableObjects.of(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .build(); + Assertions.assertNull(roleWithoutFields.properties()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index 83e0f41ca48..9b63d32167e 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -6,7 +6,10 @@ import com.datastrato.gravitino.EntitySerDe; import com.datastrato.gravitino.EntitySerDeFactory; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.UserEntity; import com.google.common.collect.ImmutableMap; @@ -320,5 +323,33 @@ public void testEntitiesSerDe() throws IOException { groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class); Assertions.assertEquals(groupWithoutFields, groupFromBytes); Assertions.assertNull(groupWithoutFields.roles()); + + // Test RoleEntity + Long roleId = 1L; + String roleName = "testRole"; + RoleEntity roleEntity = + RoleEntity.builder() + .withId(roleId) + .withName(roleName) + .withAuditInfo(auditInfo) + .securableObject(SecurableObjects.of(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withProperties(props) + .build(); + byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); + RoleEntity roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + Assertions.assertEquals(roleEntity, roleFromBytes); + + RoleEntity roleWithoutFields = + RoleEntity.builder() + .withId(1L) + .withName(roleName) + .withAuditInfo(auditInfo) + .securableObject(SecurableObjects.of(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .build(); + roleBytes = protoEntitySerDe.serialize(roleWithoutFields); + roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + Assertions.assertEquals(roleWithoutFields, roleFromBytes); } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 914013f176a..b7272c413a0 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -27,6 +27,8 @@ import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NonEmptyEntityException; @@ -36,6 +38,7 @@ import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; @@ -55,6 +58,7 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -421,7 +425,7 @@ void testEntityUpdate(String type) throws Exception { @ParameterizedTest @MethodSource("storageProvider") public void testAuthorizationEntityDelete(String type) throws IOException { - // User and Group entity only support kv store. + // User, Group and Role entity only support kv store. Assumptions.assumeTrue(Configs.DEFAULT_ENTITY_STORE.equals(type)); Config config = Mockito.mock(Config.class); init(type, config); @@ -442,15 +446,23 @@ public void testAuthorizationEntityDelete(String type) throws IOException { store.put(oneGroup); GroupEntity anotherGroup = createGroup("metalake", "anotherGroup", auditInfo); store.put(anotherGroup); + RoleEntity oneRole = createRole("metalake", "oneRole", auditInfo); + store.put(oneRole); + RoleEntity anotherRole = createRole("metalake", "anotherRole", auditInfo); + store.put(anotherRole); Assertions.assertTrue(store.exists(oneUser.nameIdentifier(), Entity.EntityType.USER)); Assertions.assertTrue(store.exists(anotherUser.nameIdentifier(), Entity.EntityType.USER)); Assertions.assertTrue(store.exists(oneGroup.nameIdentifier(), Entity.EntityType.GROUP)); Assertions.assertTrue(store.exists(anotherGroup.nameIdentifier(), Entity.EntityType.GROUP)); + Assertions.assertTrue(store.exists(oneRole.nameIdentifier(), Entity.EntityType.ROLE)); + Assertions.assertTrue(store.exists(anotherRole.nameIdentifier(), Entity.EntityType.ROLE)); store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE); Assertions.assertFalse(store.exists(oneUser.nameIdentifier(), Entity.EntityType.USER)); Assertions.assertFalse(store.exists(anotherUser.nameIdentifier(), Entity.EntityType.USER)); Assertions.assertFalse(store.exists(oneGroup.nameIdentifier(), Entity.EntityType.GROUP)); Assertions.assertFalse(store.exists(anotherGroup.nameIdentifier(), Entity.EntityType.GROUP)); + Assertions.assertFalse(store.exists(oneRole.nameIdentifier(), Entity.EntityType.ROLE)); + Assertions.assertFalse(store.exists(anotherRole.nameIdentifier(), Entity.EntityType.ROLE)); } } @@ -1162,6 +1174,20 @@ private static GroupEntity createGroup(String metalake, String name, AuditInfo a .build(); } + private static RoleEntity createRole(String metalake, String name, AuditInfo auditInfo) { + return RoleEntity.builder() + .withId(1L) + .withNamespace( + Namespace.of( + metalake, CatalogEntity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withName(name) + .withAuditInfo(auditInfo) + .securableObject(SecurableObjects.of("catalog")) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withProperties(Collections.emptyMap()) + .build(); + } + private void validateDeleteTopicCascade(EntityStore store, TopicEntity topic1) throws IOException { // Delete the topic 'metalake.catalog.schema1.topic1' diff --git a/meta/src/main/proto/gravitino_meta.proto b/meta/src/main/proto/gravitino_meta.proto index c040023826a..94cccc9185f 100644 --- a/meta/src/main/proto/gravitino_meta.proto +++ b/meta/src/main/proto/gravitino_meta.proto @@ -114,4 +114,13 @@ message Group { string name = 2; repeated string roles = 3; AuditInfo audit_info = 4; -} \ No newline at end of file +} + +message Role { + uint64 id = 1; + string name = 2; + repeated string privileges = 3; + string securable_object = 4; + map properties = 5; + AuditInfo audit_info = 6; +} From c19f9cf34fdc14e5585c0beb1efa41ea035928f6 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Mon, 15 Apr 2024 19:30:05 +0800 Subject: [PATCH 025/106] [#2910] improvement(filesystem-hadoop3): Make `getWorkingDirectory` method synchronized (#2945) ### What changes were proposed in this pull request? Make `getWorkingDirectory` method synchronized to avoid some concurrent issues. ### Why are the changes needed? Fix: #2910 Co-authored-by: xiaojiebao --- .../filesystem/hadoop/GravitinoVirtualFileSystem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 9b185f7ba5d..c4ed6379ce6 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -118,7 +118,7 @@ private void initializeCache(int maxCapacity, long expireAfterAccess) { .removalListener( (key, value, cause) -> { try { - Pair pair = ((Pair) value); + Pair pair = (Pair) value; if (pair != null && pair.getRight() != null) pair.getRight().close(); } catch (IOException e) { Logger.error("Cannot close the file system for fileset: {}", key, e); @@ -360,7 +360,7 @@ public URI getUri() { } @Override - public Path getWorkingDirectory() { + public synchronized Path getWorkingDirectory() { return this.workingDirectory; } From 8fd412c0e6c3eb2fcc19623e572da7ff7676ab0d Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 15 Apr 2024 19:48:28 +0800 Subject: [PATCH 026/106] [#1661] feat(core): introduce catalog capability framework (#2819) ### What changes were proposed in this pull request? 1. Introduce catalog capability framework 2. Support `column not null` capability to show how the framework works ### Why are the changes needed? Improving code quality Fix: #1662 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../gravitino/catalog/hive/HiveCatalog.java | 6 + .../catalog/hive/HiveCatalogCapability.java | 19 +++ .../catalog/hive/HiveCatalogOperations.java | 18 --- .../gravitino/catalog/hive/TestHiveTable.java | 50 ------- .../hive/integration/test/CatalogHiveIT.java | 36 +++++ .../gravitino/catalog/CapabilityHelpers.java | 131 ++++++++++++++++++ .../gravitino/catalog/CatalogManager.java | 6 + .../catalog/TableOperationDispatcher.java | 7 +- .../gravitino/connector/BaseCatalog.java | 27 ++++ .../connector/capability/Capability.java | 76 ++++++++++ .../capability/CapabilityResult.java | 64 +++++++++ .../catalog/TestTableOperationDispatcher.java | 3 +- 12 files changed, 371 insertions(+), 72 deletions(-) create mode 100644 catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java create mode 100644 core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java create mode 100644 core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java index 9145d5b79f7..a84f582d2b0 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.ProxyPlugin; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.rel.SupportsSchemas; import com.datastrato.gravitino.rel.TableCatalog; import java.util.Map; @@ -37,6 +38,11 @@ protected CatalogOperations newOps(Map config) { return ops; } + @Override + public Capability newCapability() { + return new HiveCatalogCapability(); + } + /** * Returns the Hive catalog operations as a {@link SupportsSchemas}. * diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java new file mode 100644 index 00000000000..3a162b22bb7 --- /dev/null +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java @@ -0,0 +1,19 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.hive; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class HiveCatalogCapability implements Capability { + @Override + public CapabilityResult columnNotNull() { + // The NOT NULL constraint for column is supported since Hive3.0, see + // https://issues.apache.org/jira/browse/HIVE-16575 + return CapabilityResult.unsupported( + "The NOT NULL constraint for column is only supported since Hive 3.0, " + + "but the current Gravitino Hive catalog only supports Hive 2.x."); + } +} diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java index a44519ae3f0..e820376d00c 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java @@ -591,11 +591,6 @@ private void validateColumnChangeForAlter( || !partitionFields.contains(fieldToAdd), "Cannot alter partition column: " + fieldToAdd); - if (c instanceof TableChange.UpdateColumnNullability) { - throw new IllegalArgumentException( - "Hive does not support altering column nullability"); - } - if (c instanceof TableChange.UpdateColumnDefaultValue) { throw new IllegalArgumentException( "Hive does not support altering column default value"); @@ -690,7 +685,6 @@ public Table createTable( Arrays.stream(columns) .forEach( c -> { - validateNullable(c.name(), c.nullable()); validateColumnDefaultValue(c.name(), c.defaultValue()); }); @@ -791,7 +785,6 @@ public Table alterTable(NameIdentifier tableIdent, TableChange... changes) if (change instanceof TableChange.AddColumn) { TableChange.AddColumn addColumn = (TableChange.AddColumn) change; String fieldName = String.join(".", addColumn.fieldName()); - validateNullable(fieldName, addColumn.isNullable()); validateColumnDefaultValue(fieldName, addColumn.getDefaultValue()); doAddColumn(cols, addColumn); @@ -872,17 +865,6 @@ private void validateColumnDefaultValue(String fieldName, Expression defaultValu } } - private void validateNullable(String fieldName, boolean nullable) { - // The NOT NULL constraint for column is supported since Hive3.0, see - // https://issues.apache.org/jira/browse/HIVE-16575 - if (!nullable) { - throw new IllegalArgumentException( - "The NOT NULL constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x. Illegal column: " - + fieldName); - } - } - private int columnPosition(List columns, TableChange.ColumnPosition position) { Preconditions.checkArgument(position != null, "Column position cannot be null"); if (position instanceof TableChange.After) { diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java index 6555bed0d10..c5a98cf5a78 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java @@ -202,33 +202,6 @@ public void testCreateHiveTable() { sortOrders)); Assertions.assertTrue(exception.getMessage().contains("Table already exists")); - HiveColumn illegalColumn = - HiveColumn.builder() - .withName("col_3") - .withType(Types.ByteType.get()) - .withComment(HIVE_COMMENT) - .withNullable(false) - .build(); - - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier, - new Column[] {illegalColumn}, - HIVE_COMMENT, - properties, - new Transform[0], - distribution, - sortOrders)); - Assertions.assertTrue( - exception - .getMessage() - .contains( - "The NOT NULL constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x")); - HiveColumn withDefault = HiveColumn.builder() .withName("col_3") @@ -455,21 +428,6 @@ public void testAlterHiveTable() { () -> tableCatalog.alterTable(tableIdentifier, tableChange3)); Assertions.assertTrue(exception.getMessage().contains("Column position cannot be null")); - TableChange.ColumnPosition first = TableChange.ColumnPosition.first(); - TableChange tableChange4 = - TableChange.addColumn(new String[] {"col_3"}, Types.ByteType.get(), null, first, false); - - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> tableCatalog.alterTable(tableIdentifier, tableChange4)); - Assertions.assertTrue( - exception - .getMessage() - .contains( - "The NOT NULL constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x")); - TableChange.ColumnPosition pos = TableChange.ColumnPosition.after(col2.name()); TableChange tableChange5 = TableChange.addColumn(new String[] {"col_3"}, Types.ByteType.get(), pos); @@ -489,14 +447,6 @@ public void testAlterHiveTable() { () -> tableCatalog.alterTable(tableIdentifier, tableChange6)); Assertions.assertTrue(exception.getMessage().contains("Cannot add column with duplicate name")); - TableChange tableChange7 = TableChange.updateColumnNullability(new String[] {"col_1"}, false); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> tableCatalog.alterTable(tableIdentifier, tableChange7)); - Assertions.assertEquals( - "Hive does not support altering column nullability", exception.getMessage()); - TableChange tableChange8 = TableChange.addColumn( new String[] {"col_3"}, Types.ByteType.get(), "comment", Literals.NULL); diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 52884aba9d7..8207b3bebae 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -485,6 +485,28 @@ public void testCreateHiveTable() throws TException, InterruptedException { Assertions.assertEquals(properties.get(key), hiveTable1.getParameters().get(key))); assertTableEquals(createdTable1, hiveTable1); checkTableReadWrite(hiveTable1); + + // test column not null + Column illegalColumn = + Column.of("not_null_column", Types.StringType.get(), "not null column", false, false, null); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + catalog + .asTableCatalog() + .createTable( + nameIdentifier, + new Column[] {illegalColumn}, + TABLE_COMMENT, + properties, + Transforms.EMPTY_TRANSFORM)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "The NOT NULL constraint for column is only supported since Hive 3.0, " + + "but the current Gravitino Hive catalog only supports Hive 2.x")); } @Test @@ -1098,6 +1120,20 @@ public void testAlterHiveTable() throws TException, InterruptedException { + "but the current Gravitino Hive catalog only supports Hive 2.x"), "The exception message is: " + exception.getMessage()); + // test alter column nullability exception + TableChange alterColumnNullability = + TableChange.updateColumnNullability(new String[] {HIVE_COL_NAME1}, false); + exception = + assertThrows( + IllegalArgumentException.class, + () -> tableCatalog.alterTable(id, alterColumnNullability)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "The NOT NULL constraint for column is only supported since Hive 3.0," + + " but the current Gravitino Hive catalog only supports Hive 2.x. Illegal column: hive_col_name1")); + // test updateColumnPosition exception Column col1 = Column.of("name", Types.StringType.get(), "comment"); Column col2 = Column.of("address", Types.StringType.get(), "comment"); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java new file mode 100644 index 00000000000..c53de1003bf --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.TableChange; +import com.google.common.base.Preconditions; +import java.util.Arrays; + +public class CapabilityHelpers { + + public static Column[] applyCapabilities(Column[] columns, Capability capabilities) { + return Arrays.stream(columns) + .map(c -> applyCapabilities(c, capabilities)) + .toArray(Column[]::new); + } + + public static TableChange[] applyCapabilities(Capability capabilities, TableChange... changes) { + return Arrays.stream(changes) + .map( + change -> { + if (change instanceof TableChange.AddColumn) { + return applyCapabilities((TableChange.AddColumn) change, capabilities); + + } else if (change instanceof TableChange.UpdateColumnNullability) { + return applyCapabilities( + (TableChange.UpdateColumnNullability) change, capabilities); + } + return change; + }) + .toArray(TableChange[]::new); + } + + private static TableChange applyCapabilities( + TableChange.AddColumn addColumn, Capability capabilities) { + Column appliedColumn = + applyCapabilities( + Column.of( + addColumn.fieldName()[0], + addColumn.getDataType(), + addColumn.getComment(), + addColumn.isNullable(), + addColumn.isAutoIncrement(), + addColumn.getDefaultValue()), + capabilities); + + return TableChange.addColumn( + applyCaseSensitiveOnColumnName(addColumn.fieldName(), capabilities), + appliedColumn.dataType(), + appliedColumn.comment(), + addColumn.getPosition(), + appliedColumn.nullable(), + appliedColumn.autoIncrement(), + appliedColumn.defaultValue()); + } + + private static TableChange applyCapabilities( + TableChange.UpdateColumnNullability updateColumnNullability, Capability capabilities) { + + applyColumnNotNull( + String.join(".", updateColumnNullability.fieldName()), + updateColumnNullability.nullable(), + capabilities); + + return TableChange.updateColumnNullability( + applyCaseSensitiveOnColumnName(updateColumnNullability.fieldName(), capabilities), + updateColumnNullability.nullable()); + } + + private static Column applyCapabilities(Column column, Capability capabilities) { + applyColumnNotNull(column, capabilities); + applyColumnDefaultValue(column, capabilities); + applyNameSpecification(Capability.Scope.COLUMN, column.name(), capabilities); + + return Column.of( + applyCaseSensitiveOnName(Capability.Scope.COLUMN, column.name(), capabilities), + column.dataType(), + column.comment(), + column.nullable(), + column.autoIncrement(), + column.defaultValue()); + } + + private static String applyCaseSensitiveOnName( + Capability.Scope scope, String name, Capability capabilities) { + return capabilities.caseSensitiveOnName(scope).supported() ? name : name.toLowerCase(); + } + + private static String[] applyCaseSensitiveOnColumnName(String[] name, Capability capabilities) { + if (!capabilities.caseSensitiveOnName(Capability.Scope.COLUMN).supported()) { + String[] standardizeColumnName = Arrays.copyOf(name, name.length); + standardizeColumnName[0] = name[0].toLowerCase(); + return standardizeColumnName; + } + return name; + } + + private static void applyColumnNotNull(Column column, Capability capabilities) { + applyColumnNotNull(column.name(), column.nullable(), capabilities); + } + + private static void applyColumnNotNull( + String columnName, boolean nullable, Capability capabilities) { + Preconditions.checkArgument( + capabilities.columnNotNull().supported() || nullable, + capabilities.columnNotNull().unsupportedMessage() + " Illegal column: " + columnName); + } + + private static void applyColumnDefaultValue(Column column, Capability capabilities) { + Preconditions.checkArgument( + capabilities.columnDefaultValue().supported() + || DEFAULT_VALUE_NOT_SET.equals(column.defaultValue()), + capabilities.columnDefaultValue().unsupportedMessage() + + " Illegal column: " + + column.name()); + } + + private static void applyNameSpecification( + Capability.Scope scope, String name, Capability capabilities) { + Preconditions.checkArgument( + capabilities.specificationOnName(scope, name).supported(), + capabilities.specificationOnName(scope, name).unsupportedMessage() + + " Illegal name: " + + name); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 9c4419206e5..93857febf1c 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -25,6 +25,7 @@ import com.datastrato.gravitino.SupportsCatalogs; import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; @@ -135,6 +136,10 @@ public R doWithPropertiesMeta(ThrowableFunction fn) return classLoader.withClassLoader(cl -> fn.apply(catalog.ops())); } + public Capability capabilities() throws Exception { + return classLoader.withClassLoader(cl -> catalog.capability()); + } + public void close() { try { classLoader.withClassLoader( @@ -570,6 +575,7 @@ private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { // so. For simply, We will preload the value of properties and thus AppClassLoader can get // the value of properties. wrapper.catalog.properties(); + wrapper.catalog.capability(); return null; }, IllegalArgumentException.class); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index 4939ec801cf..d388979582f 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.catalog; import static com.datastrato.gravitino.Entity.EntityType.TABLE; +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCapabilities; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; @@ -163,7 +164,7 @@ public Table createTable( t -> t.createTable( ident, - columns, + applyCapabilities(columns, c.capabilities()), comment, updatedProperties, partitions == null ? EMPTY_TRANSFORM : partitions, @@ -227,7 +228,9 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) Table tempAlteredTable = doWithCatalog( catalogIdent, - c -> c.doWithTableOps(t -> t.alterTable(ident, changes)), + c -> + c.doWithTableOps( + t -> t.alterTable(ident, applyCapabilities(c.capabilities(), changes))), NoSuchTableException.class, IllegalArgumentException.class); diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java index 46d760774a5..dd0cf713ea1 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.CatalogProvider; import com.datastrato.gravitino.annotation.Evolving; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.meta.CatalogEntity; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -44,6 +45,8 @@ public abstract class BaseCatalog private volatile CatalogOperations ops; + private volatile Capability capability; + private volatile Map properties; private static String ENTITY_IS_NOT_SET = "entity is not set"; @@ -65,6 +68,18 @@ public abstract class BaseCatalog @Evolving protected abstract CatalogOperations newOps(Map config); + /** + * Create a new instance of {@link Capability}, if the child class has special capabilities, it + * should implement this method to provide a specific {@link Capability} instance regarding that + * catalog. + * + * @return A new instance of {@link Capability}. + */ + @Evolving + protected Capability newCapability() { + return new Capability() {}; + } + /** * Create a new instance of ProxyPlugin, it is optional. If the child class needs to support the * specific proxy logic, it should implement this method to provide a specific ProxyPlugin. @@ -131,6 +146,18 @@ public CatalogOperations ops() { return ops; } + public Capability capability() { + if (capability == null) { + synchronized (this) { + if (capability == null) { + capability = newCapability(); + } + } + } + + return capability; + } + private CatalogOperations createOps(Map conf) { String customCatalogOperationClass = conf.get(CATALOG_OPERATION_IMPL); return Optional.ofNullable(customCatalogOperationClass) diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java new file mode 100644 index 00000000000..326f96867f2 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.connector.capability; + +import com.datastrato.gravitino.annotation.Evolving; + +/** + * The Catalog interface to provide the capabilities of the catalog. If the implemented catalog has + * some special capabilities, it should override the default implementation of the capabilities. + */ +@Evolving +public interface Capability { + + /** The scope of the capability. */ + enum Scope { + CATALOG, + SCHEMA, + TABLE, + COLUMN, + FILESET, + TOPIC, + PARTITION + } + + /** + * Check if the catalog supports not null constraint on column. + * + * @return The check result of the not null constraint. + */ + default CapabilityResult columnNotNull() { + return CapabilityResult.SUPPORTED; + } + + /** + * Check if the catalog supports default value on column. + * + * @return The check result of the default value. + */ + default CapabilityResult columnDefaultValue() { + return CapabilityResult.SUPPORTED; + } + + /** + * Check if the name is case-sensitive in the scope. + * + * @param scope The scope of the capability. + * @return The capability of the case-sensitive on name. + */ + default CapabilityResult caseSensitiveOnName(Scope scope) { + return CapabilityResult.SUPPORTED; + } + + /** + * Check if the name is illegal in the scope, such as special characters, reserved words, etc. + * + * @param scope The scope of the capability. + * @param name The name to be checked. + * @return The capability of the specification on name. + */ + default CapabilityResult specificationOnName(Scope scope, String name) { + return CapabilityResult.SUPPORTED; + } + + /** + * Check if the entity is fully managed by Gravitino in the scope. + * + * @param scope The scope of the capability. + * @return The capability of the managed storage. + */ + default CapabilityResult managedStorage(Scope scope) { + return CapabilityResult.unsupported( + String.format("The %s entity is not fully managed by Gravitino.", scope)); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java new file mode 100644 index 00000000000..629c31caf4e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.connector.capability; + +import com.datastrato.gravitino.annotation.Evolving; +import com.google.common.base.Preconditions; + +/** The CapabilityResult class is responsible for managing the capability result. */ +@Evolving +public interface CapabilityResult { + + /** The supported capability result. */ + CapabilityResult SUPPORTED = new ResultImpl(true, null); + + /** + * The unsupported capability result. + * + * @param unsupportedMessage The unsupported message. + * @return The unsupported capability result. + */ + static CapabilityResult unsupported(String unsupportedMessage) { + return new ResultImpl(false, unsupportedMessage); + } + + /** + * Check if the capability is supported. + * + * @return true if the capability is supported, false otherwise. + */ + boolean supported(); + + /** + * Get the unsupported message. + * + * @return The unsupported message. + */ + String unsupportedMessage(); + + /** The CapabilityResult implementation. */ + class ResultImpl implements CapabilityResult { + private final boolean supported; + private final String unsupportedMessage; + + private ResultImpl(boolean supported, String unsupportedMessage) { + Preconditions.checkArgument( + supported || unsupportedMessage != null, + "unsupportedReason is required when supportsNotNull is false"); + this.supported = supported; + this.unsupportedMessage = unsupportedMessage; + } + + @Override + public boolean supported() { + return supported; + } + + @Override + public String unsupportedMessage() { + return unsupportedMessage; + } + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java index 4dc1edab7e8..e87af83d485 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java @@ -57,8 +57,7 @@ public void testCreateAndListTables() throws IOException { NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, "table1"); Column[] columns = new Column[] { - TestColumn.builder().withName("col1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("col2").withType(Types.StringType.get()).build() + Column.of("col1", Types.StringType.get()), Column.of("col2", Types.StringType.get()) }; Table table1 = From 802e5395c5aa04cf06dcb87117d268c2b885545d Mon Sep 17 00:00:00 2001 From: Peidian li <38486782+coolderli@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:54:37 +0800 Subject: [PATCH 027/106] improve(filesystem-hadoop): support path without scheme for gvfs api (#2779) ### What changes were proposed in this pull request? It will use the path without scheme in tensorflow. This MR will support the path without gvfs scheme. https://github.com/tensorflow/io/blob/master/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc#L618 https://github.com/tensorflow/io/blob/master/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc#L116 ### Why are the changes needed? - support path without Scheme for Hadoop API - #2860 ### Does this PR introduce _any_ user-facing change? - no ### How was this patch tested? - UTs pass --- .../hadoop/GravitinoVirtualFileSystem.java | 85 ++++++------- ...avitinoVirtualFileSystemConfiguration.java | 2 +- .../hadoop/FileSystemTestUtils.java | 25 ++-- .../filesystem/hadoop/TestGvfsBase.java | 119 +++++++++++++++--- 4 files changed, 154 insertions(+), 77 deletions(-) diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index c4ed6379ce6..37d4a90a41b 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -24,12 +24,13 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; @@ -51,6 +52,13 @@ public class GravitinoVirtualFileSystem extends FileSystem { private Cache> filesetCache; private ScheduledThreadPoolExecutor scheduler; + // The pattern is used to match gvfs path. The scheme prefix (gvfs://fileset) is optional. + // The following path can be match: + // gvfs://fileset/fileset_catalog/fileset_schema/fileset1/file.txt + // /fileset_catalog/fileset_schema/fileset1/sub_dir/ + private static final Pattern IDENTIFIER_PATTERN = + Pattern.compile("^(?:gvfs://fileset)?/([^/]+)/([^/]+)/([^/]+)(?:[/[^/]+]*)$"); + @Override public void initialize(URI name, Configuration configuration) throws IOException { if (!name.toString().startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX)) { @@ -214,39 +222,33 @@ private void checkAuthConfig(String authType, String configKey, String configVal authType); } - private String concatVirtualPrefix(NameIdentifier identifier) { - return GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX - + identifier.namespace().level(1) - + "/" - + identifier.namespace().level(2) - + "/" - + identifier.name(); + private String getVirtualLocation(NameIdentifier identifier, boolean withScheme) { + return String.format( + "%s/%s/%s/%s", + withScheme ? GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX : "", + identifier.namespace().level(1), + identifier.namespace().level(2), + identifier.name()); } private Path getActualPathByIdentifier( NameIdentifier identifier, Pair filesetPair, Path path) { String virtualPath = path.toString(); - if (!virtualPath.startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX)) { - throw new InvalidPathException( - String.format( - "Path %s doesn't start with the scheme \"%s\".", - virtualPath, GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX)); - } + boolean withScheme = + virtualPath.startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX); + String virtualLocation = getVirtualLocation(identifier, withScheme); + String storageLocation = filesetPair.getLeft().storageLocation(); try { if (checkMountsSingleFile(filesetPair)) { - String virtualPrefix = concatVirtualPrefix(identifier); Preconditions.checkArgument( - virtualPath.equals(virtualPrefix), + virtualPath.equals(virtualLocation), "Path: %s should be same with the virtual prefix: %s, because the fileset only mounts a single file.", virtualPath, - virtualPrefix); + virtualLocation); - return new Path(filesetPair.getLeft().storageLocation()); + return new Path(storageLocation); } else { - return new Path( - virtualPath.replaceFirst( - concatVirtualPrefix(identifier), - new Path(filesetPair.getLeft().storageLocation()).toString())); + return new Path(virtualPath.replaceFirst(virtualLocation, storageLocation)); } } catch (Exception e) { throw new RuntimeException( @@ -275,37 +277,32 @@ private boolean checkMountsSingleFile(Pair filesetPair) { private FileStatus convertFileStatusPathPrefix( FileStatus fileStatus, String actualPrefix, String virtualPrefix) { String filePath = fileStatus.getPath().toString(); - if (!filePath.startsWith(actualPrefix)) { - throw new InvalidPathException( - String.format("Path %s doesn't start with prefix \"%s\".", filePath, actualPrefix)); - } + Preconditions.checkArgument( + filePath.startsWith(actualPrefix), + "Path %s doesn't start with prefix \"%s\".", + filePath, + actualPrefix); Path path = new Path(filePath.replaceFirst(actualPrefix, virtualPrefix)); fileStatus.setPath(path); return fileStatus; } - private NameIdentifier extractIdentifier(URI virtualUri) { + @VisibleForTesting + NameIdentifier extractIdentifier(URI virtualUri) { + String virtualPath = virtualUri.toString(); Preconditions.checkArgument( - virtualUri - .toString() - .startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX), - "Path %s doesn't start with scheme prefix \"%s\".", - virtualUri, - GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX); - - if (StringUtils.isBlank(virtualUri.toString())) { - throw new InvalidPathException("Uri which need be extracted cannot be null or empty."); - } + StringUtils.isNotBlank(virtualPath), + "Uri which need be extracted cannot be null or empty."); - // remove first '/' symbol with empty string - String[] reservedDirs = - Arrays.stream(virtualUri.getPath().replaceFirst("/", "").split("/")).toArray(String[]::new); + Matcher matcher = IDENTIFIER_PATTERN.matcher(virtualPath); Preconditions.checkArgument( - reservedDirs.length >= 3, "URI %s doesn't contains valid identifier", virtualUri); + matcher.matches() && matcher.groupCount() == 3, + "URI %s doesn't contains valid identifier", + virtualPath); return NameIdentifier.ofFileset( - metalakeName, reservedDirs[0], reservedDirs[1], reservedDirs[2]); + metalakeName, matcher.group(1), matcher.group(2), matcher.group(3)); } private FilesetContext getFilesetContext(Path virtualPath) { @@ -449,7 +446,7 @@ public FileStatus getFileStatus(Path path) throws IOException { return convertFileStatusPathPrefix( fileStatus, context.getFileset().storageLocation(), - concatVirtualPrefix(context.getIdentifier())); + getVirtualLocation(context.getIdentifier(), true)); } @Override @@ -462,7 +459,7 @@ public FileStatus[] listStatus(Path path) throws IOException { convertFileStatusPathPrefix( fileStatus, new Path(context.getFileset().storageLocation()).toString(), - concatVirtualPrefix(context.getIdentifier()))) + getVirtualLocation(context.getIdentifier(), true))) .toArray(FileStatus[]::new); } diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 5333508dee4..fc7390be41e 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -6,7 +6,7 @@ /** Configuration class for Gravitino Virtual File System. */ class GravitinoVirtualFileSystemConfiguration { - public static final String GVFS_FILESET_PREFIX = "gvfs://fileset/"; + public static final String GVFS_FILESET_PREFIX = "gvfs://fileset"; public static final String GVFS_SCHEME = "gvfs"; /** The configuration key for the Gravitino server URI. */ diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java index 66204fa6835..3c61adc1ce0 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java @@ -16,7 +16,7 @@ public class FileSystemTestUtils { private static final String LOCAL_FS_PREFIX = - "file:/tmp/gravitino_test_fs_" + UUID.randomUUID().toString().replace("-", "") + "/"; + "file:/tmp/gravitino_test_fs_" + UUID.randomUUID().toString().replace("-", ""); private static final int BUFFER_SIZE = 3; private static final short REPLICATION = 1; @@ -30,23 +30,24 @@ public static String localRootPrefix() { return LOCAL_FS_PREFIX; } - public static Path createFilesetPath(String filesetCatalog, String schema, String fileset) { - return new Path( - GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX - + "/" - + filesetCatalog - + "/" - + schema - + "/" - + fileset); + public static Path createFilesetPath( + String filesetCatalog, String schema, String fileset, boolean withScheme) { + String filesetPath = + String.format( + "%s/%s/%s/%s", + withScheme ? GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX : "", + filesetCatalog, + schema, + fileset); + return new Path(filesetPath); } public static Path createLocalRootDir(String filesetCatalog) { - return new Path(LOCAL_FS_PREFIX + filesetCatalog); + return new Path(String.format("%s/%s", LOCAL_FS_PREFIX, filesetCatalog)); } public static Path createLocalDirPrefix(String filesetCatalog, String schema, String fileset) { - return new Path(LOCAL_FS_PREFIX + filesetCatalog + "/" + schema + "/" + fileset); + return new Path(String.format("%s/%s/%s/%s", LOCAL_FS_PREFIX, filesetCatalog, schema, fileset)); } public static void create(Path path, FileSystem fileSystem) throws IOException { diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java index baa41e9153e..d190db5fbdc 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java @@ -14,6 +14,8 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.file.Fileset; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -32,6 +34,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class TestGvfsBase extends GravitinoMockServerBase { protected static final String GVFS_IMPL_CLASS = GravitinoVirtualFileSystem.class.getName(); @@ -79,7 +83,7 @@ public void init() { Fileset.Type.MANAGED, localDirPath.toString()); managedFilesetPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, managedFilesetName); + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, managedFilesetName, true); localFilePath = new Path( @@ -93,7 +97,7 @@ public void init() { Fileset.Type.EXTERNAL, localFilePath.toString()); externalFilesetPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, externalFilesetName); + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, externalFilesetName, true); } @AfterEach @@ -164,7 +168,8 @@ public void testInternalCache() throws IOException { .FS_GRAVITINO_FILESET_CACHE_EVICTION_MILLS_AFTER_ACCESS_KEY, "1000"); - Path filesetPath1 = FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "fileset1"); + Path filesetPath1 = + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "fileset1", true); try (FileSystem fs = filesetPath1.getFileSystem(configuration)) { Path localPath1 = FileSystemTestUtils.createLocalDirPrefix(catalogName, schemaName, "fileset1"); @@ -179,7 +184,7 @@ public void testInternalCache() throws IOException { // expired by size Path filesetPath2 = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "fileset2"); + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "fileset2", true); Path localPath2 = FileSystemTestUtils.createLocalDirPrefix(catalogName, schemaName, "fileset2"); mockFilesetDTO( @@ -219,8 +224,9 @@ public void testInternalCache() throws IOException { } } - @Test - public void testCreate() throws IOException { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testCreate(boolean withScheme) throws IOException { try (FileSystem gravitinoFileSystem = managedFilesetPath.getFileSystem(conf); FileSystem localFileSystem = localDirPath.getFileSystem(conf)) { FileSystemTestUtils.mkdirs(managedFilesetPath, gravitinoFileSystem); @@ -234,7 +240,8 @@ public void testCreate() throws IOException { // mock the invalid fileset not in the server String invalidFilesetName = "invalid_fileset"; Path invalidFilesetPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, invalidFilesetName); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, invalidFilesetName, withScheme); assertThrows( RuntimeException.class, () -> FileSystemTestUtils.create(invalidFilesetPath, gravitinoFileSystem)); @@ -255,9 +262,10 @@ public void testCreate() throws IOException { } } - @Test + @ParameterizedTest + @ValueSource(booleans = {true, false}) @Disabled("Append operation is not supported in LocalFileSystem. We can't test it now.") - public void testAppend() throws IOException { + public void testAppend(boolean withScheme) throws IOException { try (FileSystem gravitinoFileSystem = managedFilesetPath.getFileSystem(conf); FileSystem localFileSystem = localDirPath.getFileSystem(conf)) { FileSystemTestUtils.mkdirs(managedFilesetPath, gravitinoFileSystem); @@ -277,7 +285,8 @@ public void testAppend() throws IOException { // mock the invalid fileset not in server String invalidAppendFilesetName = "invalid_fileset"; Path invalidAppendFilesetPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, invalidAppendFilesetName); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, invalidAppendFilesetName, withScheme); assertThrows( RuntimeException.class, () -> FileSystemTestUtils.append(invalidAppendFilesetPath, gravitinoFileSystem)); @@ -299,8 +308,9 @@ public void testAppend() throws IOException { } } - @Test - public void testRename() throws IOException { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRename(boolean withScheme) throws IOException { try (FileSystem gravitinoFileSystem = managedFilesetPath.getFileSystem(conf); FileSystem localFileSystem = localDirPath.getFileSystem(conf)) { FileSystemTestUtils.mkdirs(managedFilesetPath, gravitinoFileSystem); @@ -313,7 +323,7 @@ public void testRename() throws IOException { // cannot rename the identifier Path dstRenamePath1 = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "rename_dst1"); + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "rename_dst1", withScheme); assertThrows( RuntimeException.class, () -> gravitinoFileSystem.rename(srcRenamePath, dstRenamePath1)); @@ -325,15 +335,18 @@ public void testRename() throws IOException { // test invalid src path Path invalidSrcPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "invalid_src_name"); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, "invalid_src_name", withScheme); Path validDstPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, managedFilesetName); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, managedFilesetName, withScheme); assertThrows( RuntimeException.class, () -> gravitinoFileSystem.rename(invalidSrcPath, validDstPath)); // test invalid dst path Path invalidDstPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "invalid_dst_name"); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, "invalid_dst_name", withScheme); assertThrows( RuntimeException.class, () -> gravitinoFileSystem.rename(managedFilesetPath, invalidDstPath)); @@ -343,7 +356,8 @@ public void testRename() throws IOException { assertTrue(gravitinoFileSystem.exists(externalFilesetPath)); assertTrue(gravitinoFileSystem.getFileStatus(externalFilesetPath).isFile()); - Path dstPath = FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "rename_dst"); + Path dstPath = + FileSystemTestUtils.createFilesetPath(catalogName, schemaName, "rename_dst", withScheme); assertThrows( RuntimeException.class, () -> gravitinoFileSystem.rename(externalFilesetPath, dstPath)); localFileSystem.delete(localFilePath, true); @@ -351,8 +365,9 @@ public void testRename() throws IOException { } } - @Test - public void testDelete() throws IOException { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testDelete(boolean withScheme) throws IOException { try (FileSystem gravitinoFileSystem = managedFilesetPath.getFileSystem(conf); FileSystem localFileSystem = localDirPath.getFileSystem(conf)) { @@ -366,7 +381,8 @@ public void testDelete() throws IOException { // mock the invalid fileset not in server String invalidFilesetName = "invalid_fileset"; Path invalidFilesetPath = - FileSystemTestUtils.createFilesetPath(catalogName, schemaName, invalidFilesetName); + FileSystemTestUtils.createFilesetPath( + catalogName, schemaName, invalidFilesetName, withScheme); assertThrows( RuntimeException.class, () -> gravitinoFileSystem.delete(invalidFilesetPath, true)); @@ -481,4 +497,67 @@ public void testMkdirs() throws IOException { FileSystemTestUtils.localRootPrefix())); } } + + @Test + public void testExtractIdentifier() throws IOException, URISyntaxException { + try (GravitinoVirtualFileSystem fs = + (GravitinoVirtualFileSystem) managedFilesetPath.getFileSystem(conf)) { + NameIdentifier identifier = + fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier); + + NameIdentifier identifier2 = + fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier2); + + NameIdentifier identifier3 = + fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/files")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier3); + + NameIdentifier identifier4 = + fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/dir/dir")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier4); + + NameIdentifier identifier5 = + fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/dir/dir/")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier5); + + NameIdentifier identifier6 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier6); + + NameIdentifier identifier7 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier7); + + NameIdentifier identifier8 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier8); + + NameIdentifier identifier9 = + fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir/dir/")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier9); + + NameIdentifier identifier10 = + fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir/dir")); + assertEquals( + NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier10); + + assertThrows( + IllegalArgumentException.class, + () -> fs.extractIdentifier(new URI("gvfs://fileset/catalog1/"))); + assertThrows( + IllegalArgumentException.class, + () -> fs.extractIdentifier(new URI("hdfs://fileset/catalog1/schema1/fileset1"))); + assertThrows( + IllegalArgumentException.class, + () -> fs.extractIdentifier(new URI("/catalog1/schema1/"))); + } + } } From 1db59d753dd0ac6bcace1a881bceb7ace1fb8158 Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 15 Apr 2024 19:57:30 +0800 Subject: [PATCH 028/106] [#2943] feat(core): add event time to event listener system (#2944) ### What changes were proposed in this pull request? add event time to event listener system ### Why are the changes needed? Fix: #2943 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests. --- .../gravitino/listener/api/event/Event.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java index e4ed6d474f0..b0abffeb682 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java @@ -18,6 +18,7 @@ public abstract class Event { private final String user; @Nullable private final NameIdentifier identifier; + private final long eventTime; /** * Constructs an Event instance with the specified user and resource identifier details. @@ -30,6 +31,7 @@ public abstract class Event { protected Event(String user, NameIdentifier identifier) { this.user = user; this.identifier = identifier; + this.eventTime = System.currentTimeMillis(); } /** @@ -54,4 +56,13 @@ public String user() { public NameIdentifier identifier() { return identifier; } + + /** + * Returns the timestamp when the event was created. + * + * @return The event creation time in milliseconds since epoch. + */ + public long eventTime() { + return eventTime; + } } From 8d7a2c90b99f5fc65b9c8b898640a99ae03aa55b Mon Sep 17 00:00:00 2001 From: charliecheng630 <74488612+charliecheng630@users.noreply.github.com> Date: Mon, 15 Apr 2024 21:48:21 +0800 Subject: [PATCH 029/106] [#2916] Improvement(trino): Improvement on Collections. EMPTY_LIST (#2951) ### What changes were proposed in this pull request? Use Collections.emptyList() rather than Collections.EMPTY_LIST ### Why are the changes needed? Use of Collections.EMPTY_LIST can cause ClassCastException exactions at runtime, better to let the compiler catch these sort of issues. Fix: #2916 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? UT --- .../trino/connector/catalog/hive/HiveMetadataAdapter.java | 8 ++++---- .../connector/catalog/iceberg/IcebergMetadataAdapter.java | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java index c4c6824575e..5339b482cee 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java @@ -94,11 +94,11 @@ public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { List partitionColumns = propertyMap.containsKey(HIVE_PARTITION_KEY) ? (List) propertyMap.get(HIVE_PARTITION_KEY) - : Collections.EMPTY_LIST; + : Collections.emptyList(); List bucketColumns = propertyMap.containsKey(HIVE_BUCKET_KEY) ? (List) propertyMap.get(HIVE_BUCKET_KEY) - : Collections.EMPTY_LIST; + : Collections.emptyList(); int bucketCount = propertyMap.containsKey(HIVE_BUCKET_COUNT_KEY) ? (int) propertyMap.get(HIVE_BUCKET_COUNT_KEY) @@ -106,7 +106,7 @@ public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { List sortColumns = propertyMap.containsKey(HIVE_SORT_ORDER_KEY) ? (List) propertyMap.get(HIVE_SORT_ORDER_KEY) - : Collections.EMPTY_LIST; + : Collections.emptyList(); if (!sortColumns.isEmpty() && (bucketColumns.isEmpty() || bucketCount == 0)) { throw new TrinoException( @@ -186,7 +186,7 @@ public ConnectorTableMetadata getTableMetadata(GravitinoTable gravitinoTable) { ((Transform.SingleFieldTransform) ts) .fieldName()[0].toLowerCase(Locale.ENGLISH)) .collect(Collectors.toList()) - : Collections.EMPTY_LIST); + : Collections.emptyList()); } if (gravitinoTable.getDistribution() != null diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java index 1eec034ac53..957460a1433 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java @@ -85,12 +85,12 @@ public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { List partitionColumns = propertyMap.containsKey(ICEBERG_PARTITIONING_PROPERTY) ? (List) propertyMap.get(ICEBERG_PARTITIONING_PROPERTY) - : Collections.EMPTY_LIST; + : Collections.emptyList(); List sortColumns = propertyMap.containsKey(ICEBERG_SORTED_BY_PROPERTY) ? (List) propertyMap.get(ICEBERG_SORTED_BY_PROPERTY) - : Collections.EMPTY_LIST; + : Collections.emptyList(); Map properties = toGravitinoTableProperties( @@ -153,7 +153,7 @@ public ConnectorTableMetadata getTableMetadata(GravitinoTable gravitinoTable) { ? Arrays.stream(gravitinoTable.getPartitioning()) .map(ts -> ((Transform.SingleFieldTransform) ts).fieldName()[0]) .collect(Collectors.toList()) - : Collections.EMPTY_LIST); + : Collections.emptyList()); } if (ArrayUtils.isNotEmpty(gravitinoTable.getSortOrders())) { From f2fb33558045c84f3f6f7d02222e09cfca598e9f Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 15 Apr 2024 22:38:59 +0800 Subject: [PATCH 030/106] [#2767] feat(core): supports fileset event for event listener (#2882) ### What changes were proposed in this pull request? * `CreateFilesetEvent` * `AlterFilesetEvent` * `DropFilesetEvent` * `LoadFilesetEvent` * `ListFilesetEvent` * `CreateFilesetFailureEvent` * `AlterFilesetFailureEvent` * `DropFilesetFailureEvent` * `LoadFilesetFailureEvent` * `ListFilesetFailureEvent` ### Why are the changes needed? Fix: #2767 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../datastrato/gravitino/GravitinoEnv.java | 15 ++- .../gravitino/catalog/FilesetDispatcher.java | 16 +++ .../catalog/FilesetEventDispatcher.java | 127 ++++++++++++++++++ .../catalog/FilesetOperationDispatcher.java | 3 +- .../listener/api/event/AlterFilesetEvent.java | 58 ++++++++ .../api/event/AlterFilesetFailureEvent.java | 46 +++++++ .../api/event/CreateFilesetEvent.java | 41 ++++++ .../api/event/CreateFilesetFailureEvent.java | 47 +++++++ .../listener/api/event/DropFilesetEvent.java | 38 ++++++ .../api/event/DropFilesetFailureEvent.java | 27 ++++ .../listener/api/event/FilesetEvent.java | 34 +++++ .../api/event/FilesetFailureEvent.java | 33 +++++ .../listener/api/event/ListFilesetEvent.java | 38 ++++++ .../api/event/ListFilesetFailureEvent.java | 41 ++++++ .../listener/api/event/LoadFilesetEvent.java | 37 +++++ .../api/event/LoadFilesetFailureEvent.java | 25 ++++ .../listener/api/info/FilesetInfo.java | 78 +++++++++++ .../gravitino/server/GravitinoServer.java | 6 +- .../server/web/rest/FilesetOperations.java | 6 +- .../web/rest/TestFilesetOperations.java | 3 +- 20 files changed, 703 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index c357c9bcc0c..61144fe3e4b 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -7,6 +7,8 @@ import com.datastrato.gravitino.authorization.AccessControlManager; import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager; import com.datastrato.gravitino.catalog.CatalogManager; +import com.datastrato.gravitino.catalog.FilesetDispatcher; +import com.datastrato.gravitino.catalog.FilesetEventDispatcher; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; @@ -43,7 +45,7 @@ public class GravitinoEnv { private TableDispatcher tableDispatcher; - private FilesetOperationDispatcher filesetOperationDispatcher; + private FilesetDispatcher filesetDispatcher; private TopicOperationDispatcher topicOperationDispatcher; @@ -132,8 +134,9 @@ public void initialize(Config config) { TableOperationDispatcher tableOperationDispatcher = new TableOperationDispatcher(catalogManager, entityStore, idGenerator); this.tableDispatcher = new TableEventDispatcher(eventBus, tableOperationDispatcher); - this.filesetOperationDispatcher = + FilesetOperationDispatcher filesetOperationDispatcher = new FilesetOperationDispatcher(catalogManager, entityStore, idGenerator); + this.filesetDispatcher = new FilesetEventDispatcher(eventBus, filesetOperationDispatcher); this.topicOperationDispatcher = new TopicOperationDispatcher(catalogManager, entityStore, idGenerator); @@ -201,12 +204,12 @@ public TableDispatcher tableDispatcher() { } /** - * Get the FilesetOperationDispatcher associated with the Gravitino environment. + * Get the FilesetDispatcher associated with the Gravitino environment. * - * @return The FilesetOperationDispatcher instance. + * @return The FilesetDispatcher instance. */ - public FilesetOperationDispatcher filesetOperationDispatcher() { - return filesetOperationDispatcher; + public FilesetDispatcher filesetDispatcher() { + return filesetDispatcher; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java new file mode 100644 index 00000000000..f2d0daf280d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.file.FilesetCatalog; + +/** + * {@code FilesetDispatcher} interface acts as a specialization of the {@link FilesetCatalog} + * interface. This interface is designed to potentially add custom behaviors or operations related + * to dispatching or handling fileset-related events or actions that are not covered by the standard + * {@code FilesetCatalog} operations. + */ +public interface FilesetDispatcher extends FilesetCatalog {} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java new file mode 100644 index 00000000000..edc98bb1660 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java @@ -0,0 +1,127 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchFilesetException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetChange; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterFilesetEvent; +import com.datastrato.gravitino.listener.api.event.AlterFilesetFailureEvent; +import com.datastrato.gravitino.listener.api.event.CreateFilesetEvent; +import com.datastrato.gravitino.listener.api.event.CreateFilesetFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropFilesetEvent; +import com.datastrato.gravitino.listener.api.event.DropFilesetFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListFilesetEvent; +import com.datastrato.gravitino.listener.api.event.ListFilesetFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadFilesetEvent; +import com.datastrato.gravitino.listener.api.event.LoadFilesetFailureEvent; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code FilesetEventDispatcher} is a decorator for {@link FilesetDispatcher} that not only + * delegates fileset operations to the underlying catalog dispatcher but also dispatches + * corresponding events to an {@link EventBus} after each operation is completed. This allows for + * event-driven workflows or monitoring of fileset operations. + */ +public class FilesetEventDispatcher implements FilesetDispatcher { + private final EventBus eventBus; + private final FilesetDispatcher dispatcher; + + public FilesetEventDispatcher(EventBus eventBus, FilesetDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { + try { + NameIdentifier[] nameIdentifiers = dispatcher.listFilesets(namespace); + eventBus.dispatchEvent(new ListFilesetEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return nameIdentifiers; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListFilesetFailureEvent(PrincipalUtils.getCurrentUserName(), namespace, e)); + throw e; + } + } + + @Override + public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { + try { + Fileset fileset = dispatcher.loadFileset(ident); + eventBus.dispatchEvent( + new LoadFilesetEvent( + PrincipalUtils.getCurrentUserName(), ident, new FilesetInfo(fileset))); + return fileset; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadFilesetFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public Fileset createFileset( + NameIdentifier ident, + String comment, + Fileset.Type type, + String storageLocation, + Map properties) + throws NoSuchSchemaException, FilesetAlreadyExistsException { + try { + Fileset fileset = dispatcher.createFileset(ident, comment, type, storageLocation, properties); + eventBus.dispatchEvent( + new CreateFilesetEvent( + PrincipalUtils.getCurrentUserName(), ident, new FilesetInfo(fileset))); + return fileset; + } catch (Exception e) { + eventBus.dispatchEvent( + new CreateFilesetFailureEvent( + PrincipalUtils.getCurrentUserName(), + ident, + e, + new FilesetInfo(ident.name(), comment, type, storageLocation, properties, null))); + throw e; + } + } + + @Override + public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) + throws NoSuchFilesetException, IllegalArgumentException { + try { + Fileset fileset = dispatcher.alterFileset(ident, changes); + eventBus.dispatchEvent( + new AlterFilesetEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new FilesetInfo(fileset))); + return fileset; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterFilesetFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } + } + + @Override + public boolean dropFileset(NameIdentifier ident) { + try { + boolean isExists = dispatcher.dropFileset(ident); + eventBus.dispatchEvent( + new DropFilesetEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropFilesetFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java index 892b444e1fc..d462e6ac26a 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java @@ -17,12 +17,11 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptyEntityException; import com.datastrato.gravitino.file.Fileset; -import com.datastrato.gravitino.file.FilesetCatalog; import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.storage.IdGenerator; import java.util.Map; -public class FilesetOperationDispatcher extends OperationDispatcher implements FilesetCatalog { +public class FilesetOperationDispatcher extends OperationDispatcher implements FilesetDispatcher { /** * Creates a new FilesetOperationDispatcher instance. * diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java new file mode 100644 index 00000000000..57fed2e8db6 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.file.FilesetChange; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; + +/** Represents an event that occurs when a fileset is altered. */ +@DeveloperApi +public final class AlterFilesetEvent extends FilesetEvent { + private final FilesetInfo updatedFilesetInfo; + private final FilesetChange[] filesetChanges; + + /** + * Constructs a new {@code AlterFilesetEvent} instance. + * + * @param user The username of the individual who initiated the fileset alteration. + * @param identifier The unique identifier of the fileset that was altered. + * @param filesetChanges An array of {@link FilesetChange} objects representing the specific + * changes applied to the fileset. + * @param updatedFilesetInfo The {@link FilesetInfo} object representing the state of the fileset + * after the changes were applied. + */ + public AlterFilesetEvent( + String user, + NameIdentifier identifier, + FilesetChange[] filesetChanges, + FilesetInfo updatedFilesetInfo) { + super(user, identifier); + this.filesetChanges = filesetChanges.clone(); + this.updatedFilesetInfo = updatedFilesetInfo; + } + + /** + * Retrieves the array of changes made to the fileset. + * + * @return An array of {@link FilesetChange} objects detailing the modifications applied to the + * fileset. + */ + public FilesetChange[] filesetChanges() { + return filesetChanges; + } + + /** + * Retrieves the information about the fileset after the alterations. + * + * @return A {@link FilesetInfo} object representing the current state of the fileset + * post-alteration. + */ + public FilesetInfo updatedFilesetInfo() { + return updatedFilesetInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java new file mode 100644 index 00000000000..515f8cdce92 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.file.FilesetChange; + +/** + * Represents an event that is generated when an attempt to alter a fileset fails due to an + * exception. + */ +@DeveloperApi +public final class AlterFilesetFailureEvent extends FilesetFailureEvent { + private final FilesetChange[] filesetChanges; + + /** + * Constructs a new {@code AlterFilesetFailureEvent}, capturing detailed information about the + * failed attempt to alter a fileset. + * + * @param user The user who initiated the fileset alteration operation. + * @param identifier The identifier of the fileset that was attempted to be altered. + * @param exception The exception that was encountered during the alteration attempt, providing + * insight into the cause of the failure. + * @param filesetChanges An array of {@link FilesetChange} objects representing the changes that + * were attempted on the fileset. + */ + public AlterFilesetFailureEvent( + String user, NameIdentifier identifier, Exception exception, FilesetChange[] filesetChanges) { + super(user, identifier, exception); + this.filesetChanges = filesetChanges.clone(); + } + + /** + * Retrieves the changes that were attempted on the fileset, leading to the failure. + * + * @return An array of {@link FilesetChange} objects detailing the modifications that were + * attempted on the fileset. + */ + public FilesetChange[] filesetChanges() { + return filesetChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java new file mode 100644 index 00000000000..d5e091dd725 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; + +/** Represents an event that is triggered following the successful creation of a fileset. */ +@DeveloperApi +public final class CreateFilesetEvent extends FilesetEvent { + private final FilesetInfo createdFilesetInfo; + + /** + * Constructs a new {@code CreateFilesetEvent}, capturing the essential details surrounding the + * successful creation of a fileset. + * + * @param user The username of the person who initiated the creation of the fileset. + * @param identifier The unique identifier of the newly created fileset. + * @param createdFilesetInfo The state of the fileset immediately following its creation, + * including details such as its location, structure, and access permissions. + */ + public CreateFilesetEvent( + String user, NameIdentifier identifier, FilesetInfo createdFilesetInfo) { + super(user, identifier); + this.createdFilesetInfo = createdFilesetInfo; + } + + /** + * Provides information about the fileset as it was configured at the moment of creation. + * + * @return A {@link FilesetInfo} object encapsulating the state of the fileset immediately after + * its creation. + */ + public FilesetInfo createdFilesetInfo() { + return createdFilesetInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java new file mode 100644 index 00000000000..6e2a49c0a01 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; + +/** Represents an event triggered upon the unsuccessful attempt to create a fileset. */ +@DeveloperApi +public final class CreateFilesetFailureEvent extends FilesetFailureEvent { + private final FilesetInfo createFilesetRequest; + + /** + * Constructs a new {@code CreateFilesetFailureEvent}, capturing the specifics of the failed + * fileset creation attempt. + * + * @param user The user who initiated the attempt to create the fileset. + * @param identifier The identifier of the fileset intended for creation. + * @param exception The exception encountered during the fileset creation process, shedding light + * on the potential reasons behind the failure. + * @param createFilesetRequest The original request information used to attempt to create the + * fileset. + */ + public CreateFilesetFailureEvent( + String user, + NameIdentifier identifier, + Exception exception, + FilesetInfo createFilesetRequest) { + super(user, identifier, exception); + this.createFilesetRequest = createFilesetRequest; + } + + /** + * Provides insight into the intended configuration of the fileset at the time of the failed + * creation attempt. + * + * @return The {@link FilesetInfo} instance representing the request information for the failed + * fileset creation attempt. + */ + public FilesetInfo createFilesetRequest() { + return createFilesetRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java new file mode 100644 index 00000000000..dd65eac931c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when a fileset is dropped from the system. */ +@DeveloperApi +public final class DropFilesetEvent extends FilesetEvent { + private final boolean isExists; + + /** + * Constructs a new {@code DropFilesetEvent}, recording the attempt to drop a fileset. + * + * @param user The user who initiated the drop fileset operation. + * @param identifier The identifier of the fileset that was attempted to be dropped. + * @param isExists A boolean flag indicating whether the fileset existed at the time of the + * operation. + */ + public DropFilesetEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the existence status of the fileset at the time of the drop operation. + * + * @return {@code true} if the fileset existed at the time of the operation, otherwise {@code + * false}. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java new file mode 100644 index 00000000000..dcfb8b733ca --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to drop a fileset from the system fails. + */ +@DeveloperApi +public final class DropFilesetFailureEvent extends FilesetFailureEvent { + /** + * Constructs a new {@code DropFilesetFailureEvent}. + * + * @param user The user who initiated the drop fileset operation. + * @param identifier The identifier of the fileset that was attempted to be dropped. + * @param exception The exception that was thrown during the drop operation. This exception is key + * to diagnosing the failure, providing insights into what went wrong during the operation. + */ + public DropFilesetFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java new file mode 100644 index 00000000000..2e9e0722c1a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events related to fileset operations. Extending {@link + * com.datastrato.gravitino.listener.api.event.Event}, this class narrows the focus to operations + * performed on filesets, such as creation, deletion, or modification. It captures vital information + * including the user performing the operation and the identifier of the fileset being manipulated. + * + *

Concrete implementations of this class are expected to provide additional specifics relevant + * to the particular type of fileset operation being represented, enriching the contextual + * understanding of each event. + */ +@DeveloperApi +public abstract class FilesetEvent extends Event { + /** + * Constructs a new {@code FilesetEvent} with the specified user and fileset identifier. + * + * @param user The user responsible for initiating the fileset operation. This information is + * critical for auditing and tracking the origin of actions. + * @param identifier The identifier of the fileset involved in the operation. This includes + * details essential for pinpointing the specific fileset affected by the operation. + */ + protected FilesetEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java new file mode 100644 index 00000000000..010d25c36c3 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a fileset operation fails due to an + * exception. + * + *

Implementations of this class can be used to convey detailed information about failures in + * operations such as creating, updating, deleting, or querying filesets, making it easier to + * diagnose and respond to issues. + */ +@DeveloperApi +public abstract class FilesetFailureEvent extends FailureEvent { + /** + * Constructs a new {@code FilesetFailureEvent} instance, capturing information about the failed + * fileset operation. + * + * @param user The user associated with the failed fileset operation. + * @param identifier The identifier of the fileset that was involved in the failed operation. + * @param exception The exception that was thrown during the fileset operation, indicating the + * cause of the failure. + */ + protected FilesetFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java new file mode 100644 index 00000000000..c5e99d6a70f --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered upon the successful listing of filesets within a system. + */ +@DeveloperApi +public final class ListFilesetEvent extends FilesetEvent { + private final Namespace namespace; + /** + * Constructs a new {@code ListFilesetEvent}. + * + * @param user The user who initiated the listing of filesets. + * @param namespace The namespace within which the filesets are listed. The namespace provides + * contextual information, identifying the scope and boundaries of the listing operation. + */ + public ListFilesetEvent(String user, Namespace namespace) { + super(user, NameIdentifier.of(namespace.toString())); + this.namespace = namespace; + } + + /** + * Retrieves the namespace associated with the failed listing event. + * + * @return The {@link Namespace} that was targeted during the failed listing operation. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java new file mode 100644 index 00000000000..bf063fd891e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list filesets within a namespace fails. + */ +@DeveloperApi +public final class ListFilesetFailureEvent extends FilesetFailureEvent { + private final Namespace namespace; + + /** + * Constructs a new {@code ListFilesetFailureEvent}. + * + * @param user The username of the individual associated with the failed fileset listing + * operation. + * @param namespace The namespace in which the fileset listing was attempted. + * @param exception The exception encountered during the fileset listing attempt, which serves as + * an indicator of the issues that caused the failure. + */ + public ListFilesetFailureEvent(String user, Namespace namespace, Exception exception) { + super(user, NameIdentifier.of(namespace.toString()), exception); + this.namespace = namespace; + } + + /** + * Retrieves the namespace associated with the failed listing event. + * + * @return The {@link Namespace} that was targeted during the failed listing operation. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java new file mode 100644 index 00000000000..8d7d238ff9d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; + +/** Represents an event that occurs when a fileset is loaded into the system. */ +@DeveloperApi +public final class LoadFilesetEvent extends FilesetEvent { + private final FilesetInfo loadedFilesetInfo; + /** + * Constructs a new {@code LoadFilesetEvent}. + * + * @param user The user who initiated the loading of the fileset. + * @param identifier The unique identifier of the fileset being loaded. + * @param loadedFilesetInfo The state of the fileset post-loading. + */ + public LoadFilesetEvent(String user, NameIdentifier identifier, FilesetInfo loadedFilesetInfo) { + super(user, identifier); + this.loadedFilesetInfo = loadedFilesetInfo; + } + + /** + * Retrieves the state of the fileset as it was made available to the user after successful + * loading. + * + * @return A {@link FilesetInfo} instance encapsulating the details of the fileset as loaded. + */ + public FilesetInfo loadedFilesetInfo() { + return loadedFilesetInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java new file mode 100644 index 00000000000..724bd384b88 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when an attempt to load a fileset into the system fails. */ +@DeveloperApi +public final class LoadFilesetFailureEvent extends FilesetFailureEvent { + /** + * Constructs a new {@code FilesetFailureEvent} instance. + * + * @param user The user associated with the failed fileset operation. + * @param identifier The identifier of the fileset that was involved in the failed operation. + * @param exception The exception that was thrown during the fileset operation, indicating the + * cause of the failure. + */ + public LoadFilesetFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java new file mode 100644 index 00000000000..eb81f86510f --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.file.Fileset; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +@DeveloperApi +public final class FilesetInfo { + private final String name; + @Nullable private final String comment; + private final Fileset.Type type; + private final String storageLocation; + private final Map properties; + @Nullable private final Audit audit; + + public FilesetInfo(Fileset fileset) { + this( + fileset.name(), + fileset.comment(), + fileset.type(), + fileset.storageLocation(), + fileset.properties(), + fileset.auditInfo()); + } + + public FilesetInfo( + String name, + String comment, + Fileset.Type type, + String storageLocation, + Map properties, + Audit audit) { + this.name = name; + this.comment = comment; + this.type = type; + this.storageLocation = storageLocation; + if (properties == null) { + this.properties = ImmutableMap.of(); + } else { + this.properties = ImmutableMap.builder().putAll(properties).build(); + } + this.audit = audit; + } + + @Nullable + public Audit auditInfo() { + return audit; + } + + public String name() { + return name; + } + + public Fileset.Type getType() { + return type; + } + + public String getStorageLocation() { + return storageLocation; + } + + @Nullable + public String comment() { + return comment; + } + + public Map properties() { + return properties; + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 67756319a65..9c7ba40a3a6 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -7,7 +7,7 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.catalog.CatalogManager; -import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; +import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; @@ -83,9 +83,7 @@ protected void configure() { .to(SchemaOperationDispatcher.class) .ranked(1); bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); - bind(gravitinoEnv.filesetOperationDispatcher()) - .to(FilesetOperationDispatcher.class) - .ranked(1); + bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); bind(gravitinoEnv.topicOperationDispatcher()) .to(com.datastrato.gravitino.catalog.TopicOperationDispatcher.class) .ranked(1); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/FilesetOperations.java index 49f245b0726..1611422ae9a 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/FilesetOperations.java @@ -8,7 +8,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; +import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.dto.requests.FilesetCreateRequest; import com.datastrato.gravitino.dto.requests.FilesetUpdateRequest; import com.datastrato.gravitino.dto.requests.FilesetUpdatesRequest; @@ -42,12 +42,12 @@ public class FilesetOperations { private static final Logger LOG = LoggerFactory.getLogger(FilesetOperations.class); - private final FilesetOperationDispatcher dispatcher; + private final FilesetDispatcher dispatcher; @Context private HttpServletRequest httpRequest; @Inject - public FilesetOperations(FilesetOperationDispatcher dispatcher) { + public FilesetOperations(FilesetDispatcher dispatcher) { this.dispatcher = dispatcher; } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java index 0d1112cda09..f998b4ce47c 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java @@ -16,6 +16,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.dto.file.FilesetDTO; import com.datastrato.gravitino.dto.requests.FilesetCreateRequest; @@ -94,7 +95,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(dispatcher).to(FilesetOperationDispatcher.class).ranked(2); + bind(dispatcher).to(FilesetDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); From 86e7f46768b6e18d82e91a7a1220a73cb4a71673 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Mon, 15 Apr 2024 23:05:26 +0800 Subject: [PATCH 031/106] [#2239] feat(server): Add the role operation (#2956) ### What changes were proposed in this pull request? Add the operations for the role. ### Why are the changes needed? Fix: #2239 ### Does this PR introduce _any_ user-facing change? Yes, I will add the open api and the document in the later pr. ### How was this patch tested? Add a new UT. Co-authored-by: Heng Qin --- .../authorization/SecurableObjects.java | 24 ++ .../gravitino/dto/authorization/RoleDTO.java | 231 ++++++++++++++ .../dto/requests/RoleCreateRequest.java | 82 +++++ .../gravitino/dto/responses/RoleResponse.java | 59 ++++ .../gravitino/dto/util/DTOConverters.java | 22 ++ .../dto/responses/TestResponses.java | 26 ++ .../gravitino/authorization/RoleManager.java | 2 +- .../datastrato/gravitino/meta/RoleEntity.java | 2 +- .../gravitino/proto/RoleEntitySerDe.java | 21 +- .../datastrato/gravitino/meta/TestEntity.java | 4 +- .../gravitino/proto/TestEntityProtoSerDe.java | 4 +- .../gravitino/storage/TestEntityStorage.java | 2 +- .../server/web/rest/ExceptionHandlers.java | 38 +++ .../server/web/rest/RoleOperations.java | 108 +++++++ .../server/web/rest/TestRoleOperations.java | 298 ++++++++++++++++++ 15 files changed, 896 insertions(+), 27 deletions(-) create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java create mode 100644 server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java create mode 100644 server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java index 48b05f6ef6f..80d68836b84 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java @@ -4,11 +4,16 @@ */ package com.datastrato.gravitino.authorization; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import java.util.Objects; +import org.apache.commons.lang3.StringUtils; /** The helper class for {@link SecurableObject}. */ public class SecurableObjects { + private static final Splitter DOT = Splitter.on('.'); + /** * Create the {@link SecurableObject} with the given names. * @@ -186,4 +191,23 @@ public boolean equals(Object other) { && Objects.equals(name, otherSecurableObject.name()); } } + + /** + * Create a {@link SecurableObject} from the given identifier string. + * + * @param securableObjectIdentifier The identifier string + * @return The created {@link SecurableObject} + */ + public static SecurableObject parse(String securableObjectIdentifier) { + if ("*".equals(securableObjectIdentifier)) { + return SecurableObjects.ofAllCatalogs(); + } + + if (StringUtils.isBlank(securableObjectIdentifier)) { + throw new IllegalArgumentException("securable object identifier can't be blank"); + } + + Iterable parts = DOT.split(securableObjectIdentifier); + return SecurableObjects.of(Iterables.toArray(parts, String.class)); + } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java new file mode 100644 index 00000000000..2f50da3a353 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java @@ -0,0 +1,231 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.authorization; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.authorization.Privilege; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObject; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.dto.AuditDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +/** Represents a Role Data Transfer Object (DTO). */ +public class RoleDTO implements Role { + + @JsonProperty("name") + private String name; + + @JsonProperty("audit") + private AuditDTO audit; + + @Nullable + @JsonProperty("properties") + private Map properties; + + @JsonProperty("privileges") + private List privileges; + + @JsonProperty("securableObject") + private String securableObject; + + /** Default constructor for Jackson deserialization. */ + protected RoleDTO() {} + + /** + * Creates a new instance of RoleDTO. + * + * @param name The name of the Role DTO. + * @param properties The properties of the Role DTO. + * @param privileges The privileges of the Role DTO. + * @param securableObject The securable object of the Role DTO. + * @param audit The audit information of the Role DTO. + */ + protected RoleDTO( + String name, + Map properties, + List privileges, + String securableObject, + AuditDTO audit) { + this.name = name; + this.audit = audit; + this.properties = properties; + this.privileges = privileges; + this.securableObject = securableObject; + } + + /** @return The name of the Role DTO. */ + @Override + public String name() { + return name; + } + + /** + * The properties of the role. Note, this method will return null if the properties are not set. + * + * @return The properties of the role. + */ + @Override + public Map properties() { + return properties; + } + + /** + * The privileges of the role. All privileges belong to one resource. For example: If the resource + * is a table, the privileges could be `READ TABLE`, `WRITE TABLE`, etc. If a schema has the + * privilege of `LOAD TABLE`. It means the role can all tables of the schema. + * + * @return The privileges of the role. + */ + @Override + public List privileges() { + return privileges.stream().map(Privileges::fromString).collect(Collectors.toList()); + } + + /** + * The resource represents a special kind of entity with a unique identifier. All resources are + * organized by tree structure. For example: If the resource is a table, the identifier may be + * `catalog1.schema1.table1`. + * + * @return The securable object of the role. + */ + @Override + public SecurableObject securableObject() { + return SecurableObjects.parse(securableObject); + } + + /** @return The audit information of the Role DTO. */ + @Override + public Audit auditInfo() { + return audit; + } + + /** + * Creates a new Builder for constructing a Role DTO. + * + * @return A new Builder instance. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder class for constructing a RoleDTO instance. + * + * @param The type of the builder instance. + */ + public static class Builder { + + /** The name of the role. */ + protected String name; + + /** The privileges of the role. */ + protected List privileges = Collections.emptyList(); + + /** The audit information of the role. */ + protected AuditDTO audit; + + /** The properties of the role. */ + protected Map properties; + + /** The securable object of the role. */ + protected SecurableObject securableObject; + + /** + * Sets the name of the role. + * + * @param name The name of the role. + * @return The builder instance. + */ + public S withName(String name) { + this.name = name; + return (S) this; + } + + /** + * Sets the privileges of the role. + * + * @param privileges The privileges of the role. + * @return The builder instance. + */ + public S withPrivileges(List privileges) { + if (privileges != null) { + this.privileges = privileges; + } + + return (S) this; + } + + /** + * Sets the properties of the role. + * + * @param properties The properties of the role. + * @return The builder instance. + */ + public S withProperties(Map properties) { + if (properties != null) { + this.properties = properties; + } + + return (S) this; + } + + /** + * Sets the securable object of the role. + * + * @param securableObject The securableObject of the role. + * @return The builder instance. + */ + public S withSecurableObject(SecurableObject securableObject) { + this.securableObject = securableObject; + return (S) this; + } + + /** + * Sets the audit information of the role. + * + * @param audit The audit information of the role. + * @return The builder instance. + */ + public S withAudit(AuditDTO audit) { + this.audit = audit; + return (S) this; + } + + /** + * Builds an instance of RoleDTO using the builder's properties. + * + * @return An instance of RoleDTO. + * @throws IllegalArgumentException If the name or audit are not set. + */ + public RoleDTO build() { + Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be null or empty"); + Preconditions.checkArgument(audit != null, "audit cannot be null"); + Preconditions.checkArgument( + !CollectionUtils.isEmpty(privileges), "privileges can't be empty"); + Preconditions.checkArgument(securableObject != null, "securable object can't null"); + + return new RoleDTO( + name, + properties, + privileges.stream() + .map(Privilege::name) + .map(Objects::toString) + .collect(Collectors.toList()), + securableObject.toString(), + audit); + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java new file mode 100644 index 00000000000..f2deaf72f9a --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.jackson.Jacksonized; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +/** Represents a request to create a role. */ +@Getter +@EqualsAndHashCode +@ToString +@Builder +@Jacksonized +public class RoleCreateRequest implements RESTRequest { + + @JsonProperty("name") + private final String name; + + @Nullable + @JsonProperty("properties") + private Map properties; + + @JsonProperty("privileges") + private List privileges; + + @JsonProperty("securableObject") + private String securableObject; + + /** Default constructor for RoleCreateRequest. (Used for Jackson deserialization.) */ + public RoleCreateRequest() { + this(null, null, null, null); + } + + /** + * Creates a new RoleCreateRequest. + * + * @param name The name of the role. + * @param properties The properties of the role. + * @param securableObject The securable object of the role. + * @param privileges The privileges of the role. + */ + public RoleCreateRequest( + String name, + Map properties, + List privileges, + String securableObject) { + super(); + this.name = name; + this.properties = properties; + this.privileges = privileges; + this.securableObject = securableObject; + } + + /** + * Validates the {@link RoleCreateRequest} request. + * + * @throws IllegalArgumentException If the request is invalid, this exception is thrown. + */ + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" field is required and cannot be empty"); + + Preconditions.checkArgument( + !CollectionUtils.isEmpty(privileges), "\"privileges\" can't be empty"); + + Preconditions.checkArgument(securableObject != null, "\"securableObject\" can't null"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java new file mode 100644 index 00000000000..5c1086bb43c --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.responses; + +import com.datastrato.gravitino.dto.authorization.RoleDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +/** Represents a response for a role. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class RoleResponse extends BaseResponse { + + @JsonProperty("role") + private final RoleDTO role; + + /** + * Constructor for RoleResponse. + * + * @param role The role data transfer object. + */ + public RoleResponse(RoleDTO role) { + super(0); + this.role = role; + } + + /** Default constructor for RoleResponse. (Used for Jackson deserialization.) */ + public RoleResponse() { + super(); + this.role = null; + } + + /** + * Validates the response data. + * + * @throws IllegalArgumentException if the name or audit is not set. + */ + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(role != null, "role must not be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(role.name()), "role 'name' must not be null and empty"); + Preconditions.checkArgument(role.auditInfo() != null, "role 'auditInfo' must not be null"); + Preconditions.checkArgument( + !CollectionUtils.isEmpty(role.privileges()), "role 'privileges' can't be empty"); + Preconditions.checkArgument( + role.securableObject() != null, "role 'securableObject' can't null"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index a2050e53548..ad93bfbf566 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -10,11 +10,13 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Metalake; import com.datastrato.gravitino.authorization.Group; +import com.datastrato.gravitino.authorization.Role; import com.datastrato.gravitino.authorization.User; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.MetalakeDTO; import com.datastrato.gravitino.dto.authorization.GroupDTO; +import com.datastrato.gravitino.dto.authorization.RoleDTO; import com.datastrato.gravitino.dto.authorization.UserDTO; import com.datastrato.gravitino.dto.file.FilesetDTO; import com.datastrato.gravitino.dto.messaging.TopicDTO; @@ -369,6 +371,26 @@ public static GroupDTO toDTO(Group group) { .build(); } + /** + * Converts a role implementation to a RoleDTO. + * + * @param role The role implementation. + * @return The role DTO. + */ + public static RoleDTO toDTO(Role role) { + if (role instanceof RoleDTO) { + return (RoleDTO) role; + } + + return RoleDTO.builder() + .withName(role.name()) + .withSecurableObject(role.securableObject()) + .withPrivileges(role.privileges()) + .withProperties(role.properties()) + .withAudit(toDTO(role.auditInfo())) + .build(); + } + /** * Converts a Expression to an FunctionArg DTO. * diff --git a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java index 4465cc34f11..96b27b0e851 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java @@ -11,16 +11,20 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.MetalakeDTO; import com.datastrato.gravitino.dto.authorization.GroupDTO; +import com.datastrato.gravitino.dto.authorization.RoleDTO; import com.datastrato.gravitino.dto.authorization.UserDTO; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.SchemaDTO; import com.datastrato.gravitino.dto.rel.TableDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.Lists; import java.time.Instant; import org.junit.jupiter.api.Test; @@ -241,6 +245,7 @@ void testUserResponseException() throws IllegalArgumentException { assertThrows(IllegalArgumentException.class, () -> user.validate()); } + @Test void testGroupResponse() throws IllegalArgumentException { AuditDTO audit = AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build(); @@ -254,4 +259,25 @@ void testGroupResponseException() throws IllegalArgumentException { GroupResponse group = new GroupResponse(); assertThrows(IllegalArgumentException.class, () -> group.validate()); } + + @Test + void testRoleResponse() throws IllegalArgumentException { + AuditDTO audit = + AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + RoleDTO role = + RoleDTO.builder() + .withName("role1") + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog("catalog")) + .withAudit(audit) + .build(); + RoleResponse response = new RoleResponse(role); + response.validate(); // No exception thrown + } + + @Test + void testRoleResponseException() throws IllegalArgumentException { + RoleResponse role = new RoleResponse(); + assertThrows(IllegalArgumentException.class, () -> role.validate()); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index 5813744af18..35639e47266 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -66,7 +66,7 @@ public Role createRole( .withId(idGenerator.nextId()) .withName(role) .withProperties(properties) - .securableObject(securableObject) + .withSecurableObject(securableObject) .withPrivileges(privileges) .withNamespace( Namespace.of( diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java index 5e4948fa0b8..b4642f3e5db 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -234,7 +234,7 @@ public Builder withAuditInfo(AuditInfo auditInfo) { * @param securableObject The securable object of the role entity. * @return The builder instance. */ - public Builder securableObject(SecurableObject securableObject) { + public Builder withSecurableObject(SecurableObject securableObject) { roleEntity.securableObject = securableObject; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index ae65c3a117c..e4a96ae7e63 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -5,18 +5,12 @@ package com.datastrato.gravitino.proto; import com.datastrato.gravitino.authorization.Privileges; -import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.RoleEntity; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; public class RoleEntitySerDe implements ProtoSerDe { - private static final Splitter DOT = Splitter.on('.'); - /** * Serializes the provided entity into its corresponding Protocol Buffer message representation. * @@ -59,7 +53,7 @@ public RoleEntity deserialize(Role role) { role.getPrivilegesList().stream() .map(Privileges::fromString) .collect(Collectors.toList())) - .securableObject(parseSecurableObject(role.getSecurableObject())) + .withSecurableObject(SecurableObjects.parse(role.getSecurableObject())) .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo())); if (!role.getPropertiesMap().isEmpty()) { @@ -68,17 +62,4 @@ public RoleEntity deserialize(Role role) { return builder.build(); } - - private static SecurableObject parseSecurableObject(String securableObjectIdentifier) { - if ("*".equals(securableObjectIdentifier)) { - return SecurableObjects.ofAllCatalogs(); - } - - if (StringUtils.isBlank(securableObjectIdentifier)) { - throw new IllegalArgumentException("securable object identifier can't be blank"); - } - - Iterable parts = DOT.split(securableObjectIdentifier); - return SecurableObjects.of(Iterables.toArray(parts, String.class)); - } } diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java index 5d1f689a6ae..b852d340101 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java @@ -266,7 +266,7 @@ public void testRole() { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .securableObject(SecurableObjects.of(catalogName)) + .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withProperties(map) .build(); @@ -286,7 +286,7 @@ public void testRole() { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .securableObject(SecurableObjects.of(catalogName)) + .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .build(); Assertions.assertNull(roleWithoutFields.properties()); diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index 9b63d32167e..8223ac239c9 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -332,7 +332,7 @@ public void testEntitiesSerDe() throws IOException { .withId(roleId) .withName(roleName) .withAuditInfo(auditInfo) - .securableObject(SecurableObjects.of(catalogName)) + .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withProperties(props) .build(); @@ -345,7 +345,7 @@ public void testEntitiesSerDe() throws IOException { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .securableObject(SecurableObjects.of(catalogName)) + .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .build(); roleBytes = protoEntitySerDe.serialize(roleWithoutFields); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index b7272c413a0..4df9a63f7df 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -1182,7 +1182,7 @@ private static RoleEntity createRole(String metalake, String name, AuditInfo aud metalake, CatalogEntity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) .withName(name) .withAuditInfo(auditInfo) - .securableObject(SecurableObjects.of("catalog")) + .withSecurableObject(SecurableObjects.of("catalog")) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withProperties(Collections.emptyMap()) .build(); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java index 0921a8690f6..130c6f2fa82 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.NotFoundException; import com.datastrato.gravitino.exceptions.PartitionAlreadyExistsException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; @@ -68,6 +69,11 @@ public static Response handleGroupException( return GroupExceptionHandler.INSTANCE.handle(op, group, metalake, e); } + public static Response handleRoleException( + OperationType op, String role, String metalake, Exception e) { + return RoleExceptionHandler.INSTANCE.handle(op, role, metalake, e); + } + public static Response handleTopicException( OperationType op, String topic, String schema, Exception e) { return TopicExceptionHandler.INSTANCE.handle(op, topic, schema, e); @@ -345,6 +351,38 @@ public Response handle(OperationType op, String group, String metalake, Exceptio } } + private static class RoleExceptionHandler extends BaseExceptionHandler { + + private static final ExceptionHandler INSTANCE = new RoleExceptionHandler(); + + private static String getRoleErrorMsg( + String role, String operation, String metalake, String reason) { + return String.format( + "Failed to operate role %s operation [%s] under metalake [%s], reason [%s]", + role, operation, metalake, reason); + } + + @Override + public Response handle(OperationType op, String role, String metalake, Exception e) { + String formatted = StringUtil.isBlank(role) ? "" : " [" + role + "]"; + String errorMsg = getRoleErrorMsg(formatted, op.name(), metalake, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof RoleAlreadyExistsException) { + return Utils.alreadyExists(errorMsg, e); + + } else { + return super.handle(op, role, metalake, e); + } + } + } + private static class TopicExceptionHandler extends BaseExceptionHandler { private static final ExceptionHandler INSTANCE = new TopicExceptionHandler(); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java new file mode 100644 index 00000000000..c89f27d0a7c --- /dev/null +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java @@ -0,0 +1,108 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.authorization.AccessControlManager; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.dto.requests.RoleCreateRequest; +import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.RoleResponse; +import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.metrics.MetricNames; +import com.datastrato.gravitino.server.web.Utils; +import java.util.stream.Collectors; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Path("/metalakes/{metalake}/roles") +public class RoleOperations { + private static final Logger LOG = LoggerFactory.getLogger(RoleOperations.class); + + private final AccessControlManager accessControlManager; + + @Context private HttpServletRequest httpRequest; + + public RoleOperations() { + this.accessControlManager = GravitinoEnv.getInstance().accessControlManager(); + } + + @GET + @Path("{role}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "load-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "load-role", absolute = true) + public Response loadRole(@PathParam("metalake") String metalake, @PathParam("role") String role) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new RoleResponse( + DTOConverters.toDTO(accessControlManager.loadRole(metalake, role))))); + } catch (Exception e) { + return ExceptionHandlers.handleRoleException(OperationType.LOAD, role, metalake, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "create-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "create-role", absolute = true) + public Response creatRole(@PathParam("metalake") String metalake, RoleCreateRequest request) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new RoleResponse( + DTOConverters.toDTO( + accessControlManager.createRole( + metalake, + request.getName(), + request.getProperties(), + SecurableObjects.parse(request.getSecurableObject()), + request.getPrivileges().stream() + .map(Privileges::fromString) + .collect(Collectors.toList())))))); + } catch (Exception e) { + return ExceptionHandlers.handleRoleException( + OperationType.CREATE, request.getName(), metalake, e); + } + } + + @DELETE + @Path("{role}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "drop-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "drop-role", absolute = true) + public Response dropRole(@PathParam("metalake") String metalake, @PathParam("role") String role) { + try { + return Utils.doAs( + httpRequest, + () -> { + boolean dropped = accessControlManager.dropRole(metalake, role); + if (!dropped) { + LOG.warn("Failed to drop role {} under metalake {}", role, metalake); + } + return Utils.ok(new DropResponse(dropped)); + }); + } catch (Exception e) { + return ExceptionHandlers.handleRoleException(OperationType.DROP, role, metalake, e); + } + } +} diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java new file mode 100644 index 00000000000..17d74cc02bb --- /dev/null +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java @@ -0,0 +1,298 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import static com.datastrato.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL; +import static com.datastrato.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; +import static com.datastrato.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.authorization.AccessControlManager; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.dto.authorization.RoleDTO; +import com.datastrato.gravitino.dto.requests.RoleCreateRequest; +import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.ErrorConstants; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.RoleResponse; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.datastrato.gravitino.lock.LockManager; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.rest.RESTUtils; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestRoleOperations extends JerseyTest { + + private static final AccessControlManager manager = mock(AccessControlManager.class); + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + @BeforeAll + public static void setup() { + Config config = mock(Config.class); + Mockito.doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY); + Mockito.doReturn(1000L).when(config).get(TREE_LOCK_MIN_NODE_IN_MEMORY); + Mockito.doReturn(36000L).when(config).get(TREE_LOCK_CLEAN_INTERVAL); + GravitinoEnv.getInstance().setLockManager(new LockManager(config)); + GravitinoEnv.getInstance().setAccessControlManager(manager); + } + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(RoleOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testCreateRole() { + RoleCreateRequest req = + new RoleCreateRequest( + "role", + Collections.emptyMap(), + Lists.newArrayList(Privileges.LoadCatalog.get().name().toString()), + SecurableObjects.of("catalog").toString()); + Role role = buildRole("role1"); + + when(manager.createRole(any(), any(), any(), any(), any())).thenReturn(role); + + Response resp = + target("/metalakes/metalake1/roles") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + RoleResponse roleResponse = resp.readEntity(RoleResponse.class); + Assertions.assertEquals(0, roleResponse.getCode()); + + RoleDTO roleDTO = roleResponse.getRole(); + Assertions.assertEquals("role1", roleDTO.name()); + Assertions.assertEquals(SecurableObjects.of("catalog"), roleDTO.securableObject()); + Assertions.assertEquals(Lists.newArrayList(Privileges.LoadCatalog.get()), roleDTO.privileges()); + + // Test to throw NoSuchMetalakeException + doThrow(new NoSuchMetalakeException("mock error")) + .when(manager) + .createRole(any(), any(), any(), any(), any()); + Response resp1 = + target("/metalakes/metalake1/roles") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ErrorResponse errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw RoleAlreadyExistsException + doThrow(new RoleAlreadyExistsException("mock error")) + .when(manager) + .createRole(any(), any(), any(), any(), any()); + Response resp2 = + target("/metalakes/metalake1/roles") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResponse1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResponse1.getCode()); + Assertions.assertEquals( + RoleAlreadyExistsException.class.getSimpleName(), errorResponse1.getType()); + + // Test to throw internal RuntimeException + doThrow(new RuntimeException("mock error")) + .when(manager) + .createRole(any(), any(), any(), any(), any()); + Response resp3 = + target("/metalakes/metalake1/roles") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse2.getType()); + } + + @Test + public void testLoadRole() { + Role role = buildRole("role1"); + + when(manager.loadRole(any(), any())).thenReturn(role); + + Response resp = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + + RoleResponse roleResponse = resp.readEntity(RoleResponse.class); + Assertions.assertEquals(0, roleResponse.getCode()); + RoleDTO roleDTO = roleResponse.getRole(); + Assertions.assertEquals("role1", roleDTO.name()); + Assertions.assertTrue(role.properties().isEmpty()); + Assertions.assertEquals(SecurableObjects.of("catalog"), roleDTO.securableObject()); + Assertions.assertEquals(Lists.newArrayList(Privileges.LoadCatalog.get()), roleDTO.privileges()); + + // Test to throw NoSuchMetalakeException + doThrow(new NoSuchMetalakeException("mock error")).when(manager).loadRole(any(), any()); + Response resp1 = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw NoSuchRoleException + doThrow(new NoSuchRoleException("mock error")).when(manager).loadRole(any(), any()); + Response resp2 = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResponse1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse1.getCode()); + Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse1.getType()); + + // Test to throw internal RuntimeException + doThrow(new RuntimeException("mock error")).when(manager).loadRole(any(), any()); + Response resp3 = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse2.getType()); + } + + private Role buildRole(String role) { + return RoleEntity.builder() + .withId(1L) + .withName(role) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withProperties(Collections.emptyMap()) + .withSecurableObject(SecurableObjects.of("catalog")) + .withAuditInfo( + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + @Test + public void testDropRole() { + when(manager.dropRole(any(), any())).thenReturn(true); + + Response resp = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + DropResponse dropResponse = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResponse.getCode()); + Assertions.assertTrue(dropResponse.dropped()); + + // Test when failed to drop role + when(manager.dropRole(any(), any())).thenReturn(false); + Response resp2 = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + DropResponse dropResponse2 = resp2.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResponse2.getCode()); + Assertions.assertFalse(dropResponse2.dropped()); + + doThrow(new RuntimeException("mock error")).when(manager).dropRole(any(), any()); + Response resp3 = + target("/metalakes/metalake1/roles/role1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + } +} From d1d99954c0ab0651fb634be0f0b0b74bcbd25d97 Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 16 Apr 2024 13:03:07 +0800 Subject: [PATCH 032/106] [#2769] feat(core): supports schema event for event listener (#2880) ### What changes were proposed in this pull request? supports schema events for event listener system ### Why are the changes needed? Fix: #2769 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../datastrato/gravitino/GravitinoEnv.java | 15 +- .../gravitino/catalog/SchemaDispatcher.java | 16 +++ .../catalog/SchemaEventDispatcher.java | 132 ++++++++++++++++++ .../catalog/SchemaOperationDispatcher.java | 3 +- .../listener/api/event/AlterSchemaEvent.java | 47 +++++++ .../api/event/AlterSchemaFailureEvent.java | 35 +++++ .../listener/api/event/CreateSchemaEvent.java | 32 +++++ .../api/event/CreateSchemaFailureEvent.java | 35 +++++ .../listener/api/event/DropSchemaEvent.java | 44 ++++++ .../api/event/DropSchemaFailureEvent.java | 34 +++++ .../listener/api/event/ListSchemaEvent.java | 30 ++++ .../api/event/ListSchemaFailureEvent.java | 33 +++++ .../listener/api/event/LoadSchemaEvent.java | 31 ++++ .../api/event/LoadSchemaFailureEvent.java | 17 +++ .../listener/api/event/SchemaEvent.java | 17 +++ .../api/event/SchemaFailureEvent.java | 20 +++ .../listener/api/info/SchemaInfo.java | 84 +++++++++++ .../gravitino/server/GravitinoServer.java | 6 +- .../server/web/rest/SchemaOperations.java | 6 +- .../server/web/rest/TestSchemaOperations.java | 3 +- 20 files changed, 624 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 61144fe3e4b..e224fa48b2a 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -10,6 +10,8 @@ import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.FilesetEventDispatcher; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; +import com.datastrato.gravitino.catalog.SchemaDispatcher; +import com.datastrato.gravitino.catalog.SchemaEventDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.catalog.TableEventDispatcher; @@ -41,7 +43,7 @@ public class GravitinoEnv { private CatalogManager catalogManager; - private SchemaOperationDispatcher schemaOperationDispatcher; + private SchemaDispatcher schemaDispatcher; private TableDispatcher tableDispatcher; @@ -129,8 +131,9 @@ public void initialize(Config config) { // Create and initialize Catalog related modules this.catalogManager = new CatalogManager(config, entityStore, idGenerator); - this.schemaOperationDispatcher = + SchemaOperationDispatcher schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); + this.schemaDispatcher = new SchemaEventDispatcher(eventBus, schemaOperationDispatcher); TableOperationDispatcher tableOperationDispatcher = new TableOperationDispatcher(catalogManager, entityStore, idGenerator); this.tableDispatcher = new TableEventDispatcher(eventBus, tableOperationDispatcher); @@ -186,12 +189,12 @@ public CatalogManager catalogManager() { } /** - * Get the SchemaOperationDispatcher associated with the Gravitino environment. + * Get the SchemaDispatcher associated with the Gravitino environment. * - * @return The SchemaOperationDispatcher instance. + * @return The SchemaDispatcher instance. */ - public SchemaOperationDispatcher schemaOperationDispatcher() { - return schemaOperationDispatcher; + public SchemaDispatcher schemaDispatcher() { + return schemaDispatcher; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java new file mode 100644 index 00000000000..0667da3df91 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.rel.SupportsSchemas; + +/** + * {@code SchemaDispatcher} interface acts as a specialization of the {@link SupportsSchemas} + * interface. This interface is designed to potentially add custom behaviors or operations related + * to dispatching or handling schema-related events or actions that are not covered by the standard + * {@code SupportsSchemas} operations. + */ +public interface SchemaDispatcher extends SupportsSchemas {} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java new file mode 100644 index 00000000000..7ca34093942 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java @@ -0,0 +1,132 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NonEmptySchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterSchemaEvent; +import com.datastrato.gravitino.listener.api.event.AlterSchemaFailureEvent; +import com.datastrato.gravitino.listener.api.event.CreateSchemaEvent; +import com.datastrato.gravitino.listener.api.event.CreateSchemaFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropSchemaEvent; +import com.datastrato.gravitino.listener.api.event.DropSchemaFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListSchemaEvent; +import com.datastrato.gravitino.listener.api.event.ListSchemaFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadSchemaEvent; +import com.datastrato.gravitino.listener.api.event.LoadSchemaFailureEvent; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.SchemaChange; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code SchemaEventDispatcher} is a decorator for {@link SchemaDispatcher} that not only delegates + * schema operations to the underlying schema dispatcher but also dispatches corresponding events to + * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or + * monitoring of schema operations. + */ +public class SchemaEventDispatcher implements SchemaDispatcher { + private final EventBus eventBus; + private final SchemaDispatcher dispatcher; + + /** + * Constructs a SchemaEventDispatcher with a specified EventBus and SchemaDispatcher. + * + * @param eventBus The EventBus to which events will be dispatched. + * @param dispatcher The underlying {@link SchemaOperationDispatcher} that will perform the actual + * schema operations. + */ + public SchemaEventDispatcher(EventBus eventBus, SchemaDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogException { + try { + NameIdentifier[] nameIdentifiers = dispatcher.listSchemas(namespace); + eventBus.dispatchEvent(new ListSchemaEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return nameIdentifiers; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListSchemaFailureEvent(PrincipalUtils.getCurrentUserName(), namespace, e)); + throw e; + } + } + + @Override + public boolean schemaExists(NameIdentifier ident) { + return dispatcher.schemaExists(ident); + } + + @Override + public Schema createSchema(NameIdentifier ident, String comment, Map properties) + throws NoSuchCatalogException, SchemaAlreadyExistsException { + try { + Schema schema = dispatcher.createSchema(ident, comment, properties); + eventBus.dispatchEvent( + new CreateSchemaEvent( + PrincipalUtils.getCurrentUserName(), ident, new SchemaInfo(schema))); + return schema; + } catch (Exception e) { + SchemaInfo createSchemaRequest = new SchemaInfo(ident.name(), comment, properties, null); + eventBus.dispatchEvent( + new CreateSchemaFailureEvent( + PrincipalUtils.getCurrentUserName(), ident, e, createSchemaRequest)); + throw e; + } + } + + @Override + public Schema loadSchema(NameIdentifier ident) throws NoSuchSchemaException { + try { + Schema schema = dispatcher.loadSchema(ident); + eventBus.dispatchEvent( + new LoadSchemaEvent(PrincipalUtils.getCurrentUserName(), ident, new SchemaInfo(schema))); + return schema; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadSchemaFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public Schema alterSchema(NameIdentifier ident, SchemaChange... changes) + throws NoSuchSchemaException { + try { + Schema schema = dispatcher.alterSchema(ident, changes); + eventBus.dispatchEvent( + new AlterSchemaEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new SchemaInfo(schema))); + return schema; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterSchemaFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } + } + + @Override + public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmptySchemaException { + try { + boolean isExists = dispatcher.dropSchema(ident, cascade); + eventBus.dispatchEvent( + new DropSchemaEvent(PrincipalUtils.getCurrentUserName(), ident, isExists, cascade)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropSchemaFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, cascade)); + throw e; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index 20a952f0ae4..b13124c3af0 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -21,7 +21,6 @@ import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.SchemaChange; -import com.datastrato.gravitino.rel.SupportsSchemas; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.PrincipalUtils; import java.time.Instant; @@ -29,7 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SchemaOperationDispatcher extends OperationDispatcher implements SupportsSchemas { +public class SchemaOperationDispatcher extends OperationDispatcher implements SchemaDispatcher { private static final Logger LOG = LoggerFactory.getLogger(SchemaOperationDispatcher.class); diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java new file mode 100644 index 00000000000..de32ec82067 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; +import com.datastrato.gravitino.rel.SchemaChange; + +/** Represents an event fired when a schema is successfully altered. */ +@DeveloperApi +public final class AlterSchemaEvent extends SchemaEvent { + private final SchemaChange[] schemaChanges; + private final SchemaInfo updatedSchemaInfo; + + public AlterSchemaEvent( + String user, + NameIdentifier identifier, + SchemaChange[] schemaChanges, + SchemaInfo updatedSchemaInfo) { + super(user, identifier); + this.schemaChanges = schemaChanges.clone(); + this.updatedSchemaInfo = updatedSchemaInfo; + } + + /** + * Retrieves the updated state of the schema after the successful alteration. + * + * @return A {@link SchemaInfo} instance encapsulating the details of the altered schema. + */ + public SchemaInfo updatedSchemaInfo() { + return updatedSchemaInfo; + } + + /** + * Retrieves the specific changes that were made to the schema during the alteration process. + * + * @return An array of {@link SchemaChange} objects detailing each modification applied to the + * schema. + */ + public SchemaChange[] schemaChanges() { + return schemaChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java new file mode 100644 index 00000000000..a5b0643e9ef --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.rel.SchemaChange; + +/** + * Represents an event that is triggered when an attempt to alter a schema fails due to an + * exception. + */ +@DeveloperApi +public final class AlterSchemaFailureEvent extends SchemaFailureEvent { + private final SchemaChange[] schemaChanges; + + public AlterSchemaFailureEvent( + String user, NameIdentifier identifier, Exception exception, SchemaChange[] schemaChanges) { + super(user, identifier, exception); + this.schemaChanges = schemaChanges.clone(); + } + + /** + * Retrieves the specific changes that were made to the schema during the alteration process. + * + * @return An array of {@link SchemaChange} objects detailing each modification applied to the + * schema. + */ + public SchemaChange[] schemaChanges() { + return schemaChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java new file mode 100644 index 00000000000..4c5f08e475b --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; + +/** Represents an event triggered upon the successful creation of a schema. */ +@DeveloperApi +public final class CreateSchemaEvent extends SchemaEvent { + private final SchemaInfo createdSchemaInfo; + + public CreateSchemaEvent(String user, NameIdentifier identifier, SchemaInfo schemaInfo) { + super(user, identifier); + this.createdSchemaInfo = schemaInfo; + } + + /** + * Retrieves the final state of the schema as it was returned to the user after successful + * creation. + * + * @return A {@link SchemaInfo} instance encapsulating the comprehensive details of the newly + * created schema. + */ + public SchemaInfo createdSchemaInfo() { + return createdSchemaInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java new file mode 100644 index 00000000000..5215d1b623d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; + +/** + * Represents an event that is generated when an attempt to create a schema fails due to an + * exception. + */ +@DeveloperApi +public final class CreateSchemaFailureEvent extends SchemaFailureEvent { + private final SchemaInfo createSchemaRequest; + + public CreateSchemaFailureEvent( + String user, NameIdentifier identifier, Exception exception, SchemaInfo createSchemaRequest) { + super(user, identifier, exception); + this.createSchemaRequest = createSchemaRequest; + } + + /** + * Retrieves the original request information for the attempted schema creation. + * + * @return The {@link SchemaInfo} instance representing the request information for the failed + * schema creation attempt. + */ + public SchemaInfo createSchemaRequest() { + return createSchemaRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java new file mode 100644 index 00000000000..6c1dbbda917 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is generated after a schema is successfully dropped. */ +@DeveloperApi +public final class DropSchemaEvent extends SchemaEvent { + private final boolean isExists; + private final boolean cascade; + + public DropSchemaEvent( + String user, NameIdentifier identifier, boolean isExists, boolean cascade) { + super(user, identifier); + this.isExists = isExists; + this.cascade = cascade; + } + + /** + * Retrieves the existence status of the schema at the time of the drop operation. + * + * @return A boolean value indicating whether the schema existed. {@code true} if the schema + * existed, otherwise {@code false}. + */ + public boolean isExists() { + return isExists; + } + + /** + * Indicates whether the drop operation was performed with a cascade option. + * + * @return A boolean value indicating whether the drop operation was set to cascade. If {@code + * true}, dependent objects such as tables and views within the schema were also dropped. + * Otherwise, the operation would fail if the schema contained any dependent objects. + */ + public boolean cascade() { + return cascade; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java new file mode 100644 index 00000000000..5fa43362e8d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to drop a schema fails due to an exception. + */ +@DeveloperApi +public final class DropSchemaFailureEvent extends SchemaFailureEvent { + private final boolean cascade; + + public DropSchemaFailureEvent( + String user, NameIdentifier identifier, Exception exception, boolean cascade) { + super(user, identifier, exception); + this.cascade = cascade; + } + + /** + * Indicates whether the drop operation was performed with a cascade option. + * + * @return A boolean value indicating whether the drop operation was set to cascade. If {@code + * true}, dependent objects such as tables and views within the schema were also dropped. + * Otherwise, the operation would fail if the schema contained any dependent objects. + */ + public boolean cascade() { + return cascade; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java new file mode 100644 index 00000000000..dd13768aabb --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is triggered upon the successful list of schemas. */ +@DeveloperApi +public final class ListSchemaEvent extends SchemaEvent { + private final Namespace namespace; + + public ListSchemaEvent(String user, Namespace namespace) { + super(user, NameIdentifier.of(namespace.toString())); + this.namespace = namespace; + } + + /** + * Provides the namespace associated with this event. + * + * @return A {@link Namespace} instance from which schema were listed. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java new file mode 100644 index 00000000000..cb233503db1 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list schemas within a namespace fails + * due to an exception. + */ +@DeveloperApi +public final class ListSchemaFailureEvent extends SchemaFailureEvent { + private final Namespace namespace; + + public ListSchemaFailureEvent(String user, Namespace namespace, Exception exception) { + super(user, NameIdentifier.of(namespace.toString()), exception); + this.namespace = namespace; + } + + /** + * Retrieves the namespace associated with this failure event. + * + * @return A {@link Namespace} instance for which the schema listing was attempted + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java new file mode 100644 index 00000000000..c9418923020 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; + +/** Represents an event triggered upon the successful loading of a schema. */ +@DeveloperApi +public final class LoadSchemaEvent extends SchemaEvent { + private final SchemaInfo loadedSchemaInfo; + + public LoadSchemaEvent(String user, NameIdentifier identifier, SchemaInfo loadedSchemaInfo) { + super(user, identifier); + this.loadedSchemaInfo = loadedSchemaInfo; + } + + /** + * Retrieves the state of the schema as it was made available to the user after successful + * loading. + * + * @return A {@link SchemaInfo} instance encapsulating the details of the schema as loaded. + */ + public SchemaInfo loadedSchemaInfo() { + return loadedSchemaInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java new file mode 100644 index 00000000000..c552568233a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java @@ -0,0 +1,17 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when an attempt to load a schema fails due to an exception. */ +@DeveloperApi +public final class LoadSchemaFailureEvent extends SchemaFailureEvent { + public LoadSchemaFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java new file mode 100644 index 00000000000..60b667afb38 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java @@ -0,0 +1,17 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an abstract base class for events related to schema operations. */ +@DeveloperApi +public abstract class SchemaEvent extends Event { + protected SchemaEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java new file mode 100644 index 00000000000..3c0cd1e4af8 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java @@ -0,0 +1,20 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a schema operation fails due to an + * exception. + */ +@DeveloperApi +public abstract class SchemaFailureEvent extends FailureEvent { + protected SchemaFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java new file mode 100644 index 00000000000..b16192f22ce --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.rel.Schema; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** Provides read-only access to schema information for event listeners. */ +@DeveloperApi +public final class SchemaInfo { + private final String name; + @Nullable private final String comment; + private final Map properties; + @Nullable private final Audit audit; + + /** + * Constructs schema information based on a given schema. + * + * @param schema The schema to extract information from. + */ + public SchemaInfo(Schema schema) { + this(schema.name(), schema.comment(), schema.properties(), schema.auditInfo()); + } + + /** + * Constructs schema information with detailed parameters. + * + * @param name The name of the schema. + * @param comment An optional description of the schema. + * @param properties A map of schema properties. + * @param audit Optional audit information. + */ + public SchemaInfo(String name, String comment, Map properties, Audit audit) { + this.name = name; + this.comment = comment; + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + this.audit = audit; + } + + /** + * Gets the schema name. + * + * @return The schema name. + */ + public String name() { + return name; + } + + /** + * Gets the optional schema comment. + * + * @return The schema comment, or null if not provided. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Gets the schema properties. + * + * @return An immutable map of schema properties. + */ + public Map properties() { + return properties; + } + + /** + * Gets the optional audit information. + * + * @return The audit information, or null if not provided. + */ + @Nullable + public Audit audit() { + return audit; + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 9c7ba40a3a6..9dc90458ba0 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -8,7 +8,7 @@ import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetDispatcher; -import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; +import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; @@ -79,9 +79,7 @@ protected void configure() { bind(gravitinoEnv.metalakesManager()).to(MetalakeManager.class).ranked(1); bind(gravitinoEnv.catalogManager()).to(CatalogManager.class).ranked(1); - bind(gravitinoEnv.schemaOperationDispatcher()) - .to(SchemaOperationDispatcher.class) - .ranked(1); + bind(gravitinoEnv.schemaDispatcher()).to(SchemaDispatcher.class).ranked(1); bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); bind(gravitinoEnv.topicOperationDispatcher()) diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java index 00b4ff6b13a..78e2fa6acfe 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java @@ -8,7 +8,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; +import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.dto.requests.SchemaCreateRequest; import com.datastrato.gravitino.dto.requests.SchemaUpdateRequest; import com.datastrato.gravitino.dto.requests.SchemaUpdatesRequest; @@ -47,12 +47,12 @@ public class SchemaOperations { private static final Logger LOG = LoggerFactory.getLogger(SchemaOperations.class); - private final SchemaOperationDispatcher dispatcher; + private final SchemaDispatcher dispatcher; @Context private HttpServletRequest httpRequest; @Inject - public SchemaOperations(SchemaOperationDispatcher dispatcher) { + public SchemaOperations(SchemaDispatcher dispatcher) { this.dispatcher = dispatcher; } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestSchemaOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestSchemaOperations.java index 90dec8989ff..7b36cac1384 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestSchemaOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestSchemaOperations.java @@ -17,6 +17,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.dto.rel.SchemaDTO; import com.datastrato.gravitino.dto.requests.SchemaCreateRequest; @@ -93,7 +94,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(dispatcher).to(SchemaOperationDispatcher.class).ranked(2); + bind(dispatcher).to(SchemaDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); From 447733a710fadd49ca7faa0f65c9ead203ae355a Mon Sep 17 00:00:00 2001 From: charliecheng630 <74488612+charliecheng630@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:03:44 +0800 Subject: [PATCH 033/106] [#2914] Improvement(trino-connector) Add missing override annotation to overriding GravitinoPlugin method. (#2959) ### What changes were proposed in this pull request? Add missing override annotation to overriding GravitinoPlugin method. ### Why are the changes needed? Methods that override Trino plugin should use the override annotation. Fix: #2914 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? ITs and UTs --------- Co-authored-by: Charlie Cheng --- .../datastrato/gravitino/trino/connector/GravitinoPlugin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java index f2da88362c2..e4bbae0467d 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java @@ -11,6 +11,7 @@ /** Trino plugin endpoint, using java spi mechanism */ public class GravitinoPlugin implements Plugin { + @Override public Iterable getConnectorFactories() { return ImmutableList.of(new GravitinoConnectorFactory()); } From 2bb3e66632660ca5af2608520ca2e1e956e8b638 Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 16 Apr 2024 14:18:06 +0800 Subject: [PATCH 034/106] refactor(core): use capability of column default value (#2859) ### What changes were proposed in this pull request? - remove the column default value validation in Hive catalog and Iceberg catalog - available column default capability in the framework ### Why are the changes needed? Fix: #2953 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? tests added --- .../catalog/hive/HiveCatalogCapability.java | 9 +++ .../catalog/hive/HiveCatalogOperations.java | 29 ---------- .../gravitino/catalog/hive/TestHiveTable.java | 56 ------------------- .../hive/integration/test/CatalogHiveIT.java | 38 +++++++++++++ .../lakehouse/iceberg/IcebergCatalog.java | 6 ++ .../iceberg/IcebergCatalogCapability.java | 17 ++++++ .../iceberg/IcebergCatalogOperations.java | 17 +++--- .../iceberg/ops/IcebergTableOpsHelper.java | 15 ----- .../lakehouse/iceberg/TestIcebergTable.java | 42 -------------- .../integration/test/CatalogIcebergIT.java | 3 +- .../gravitino/catalog/CapabilityHelpers.java | 29 ++++++++-- .../gravitino/connector/BaseCatalog.java | 2 +- .../connector/capability/Capability.java | 42 ++++++++++++-- 13 files changed, 140 insertions(+), 165 deletions(-) create mode 100644 catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java index 3a162b22bb7..d98f6e12d7b 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java @@ -16,4 +16,13 @@ public CapabilityResult columnNotNull() { "The NOT NULL constraint for column is only supported since Hive 3.0, " + "but the current Gravitino Hive catalog only supports Hive 2.x."); } + + @Override + public CapabilityResult columnDefaultValue() { + // The DEFAULT constraint for column is supported since Hive3.0, see + // https://issues.apache.org/jira/browse/HIVE-18726 + return CapabilityResult.unsupported( + "The DEFAULT constraint for column is only supported since Hive 3.0, " + + "but the current Gravitino Hive catalog only supports Hive 2.x."); + } } diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java index e820376d00c..5f5d7c836d0 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java @@ -35,7 +35,6 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.rel.TableChange; -import com.datastrato.gravitino.rel.expressions.Expression; import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; @@ -591,11 +590,6 @@ private void validateColumnChangeForAlter( || !partitionFields.contains(fieldToAdd), "Cannot alter partition column: " + fieldToAdd); - if (c instanceof TableChange.UpdateColumnDefaultValue) { - throw new IllegalArgumentException( - "Hive does not support altering column default value"); - } - if (c instanceof TableChange.UpdateColumnPosition && afterPartitionColumn( partitionFields, ((TableChange.UpdateColumnPosition) c).getPosition())) { @@ -682,12 +676,6 @@ public Table createTable( validatePartitionForCreate(columns, partitioning); validateDistributionAndSort(distribution, sortOrders); - Arrays.stream(columns) - .forEach( - c -> { - validateColumnDefaultValue(c.name(), c.defaultValue()); - }); - TableType tableType = (TableType) tablePropertiesMetadata.getOrDefault(properties, TABLE_TYPE); Preconditions.checkArgument( SUPPORT_TABLE_TYPES.contains(tableType.name()), @@ -784,8 +772,6 @@ public Table alterTable(NameIdentifier tableIdent, TableChange... changes) if (change instanceof TableChange.AddColumn) { TableChange.AddColumn addColumn = (TableChange.AddColumn) change; - String fieldName = String.join(".", addColumn.fieldName()); - validateColumnDefaultValue(fieldName, addColumn.getDefaultValue()); doAddColumn(cols, addColumn); } else if (change instanceof TableChange.DeleteColumn) { @@ -803,10 +789,6 @@ public Table alterTable(NameIdentifier tableIdent, TableChange... changes) } else if (change instanceof TableChange.UpdateColumnType) { doUpdateColumnType(cols, (TableChange.UpdateColumnType) change); - } else if (change instanceof TableChange.UpdateColumnDefaultValue) { - throw new IllegalArgumentException( - "Hive does not support altering column default value"); - } else if (change instanceof TableChange.UpdateColumnAutoIncrement) { throw new IllegalArgumentException( "Hive does not support altering column auto increment"); @@ -854,17 +836,6 @@ public Table alterTable(NameIdentifier tableIdent, TableChange... changes) } } - private void validateColumnDefaultValue(String fieldName, Expression defaultValue) { - // The DEFAULT constraint for column is supported since Hive3.0, see - // https://issues.apache.org/jira/browse/HIVE-18726 - if (!defaultValue.equals(Column.DEFAULT_VALUE_NOT_SET)) { - throw new IllegalArgumentException( - "The DEFAULT constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x. Illegal column: " - + fieldName); - } - } - private int columnPosition(List columns, TableChange.ColumnPosition position) { Preconditions.checkArgument(position != null, "Column position cannot be null"); if (position instanceof TableChange.After) { diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java index c5a98cf5a78..d54889de6a6 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java @@ -25,13 +25,11 @@ import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; -import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.rel.expressions.sorts.NullOrdering; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; import com.datastrato.gravitino.rel.expressions.transforms.Transform; -import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; import java.time.Instant; @@ -201,34 +199,6 @@ public void testCreateHiveTable() { distribution, sortOrders)); Assertions.assertTrue(exception.getMessage().contains("Table already exists")); - - HiveColumn withDefault = - HiveColumn.builder() - .withName("col_3") - .withType(Types.ByteType.get()) - .withComment(HIVE_COMMENT) - .withNullable(true) - .withDefaultValue(Literals.NULL) - .build(); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier, - new Column[] {withDefault}, - HIVE_COMMENT, - properties, - Transforms.EMPTY_TRANSFORM, - distribution, - sortOrders)); - Assertions.assertTrue( - exception - .getMessage() - .contains( - "The DEFAULT constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x"), - "The exception message is: " + exception.getMessage()); } @Test @@ -447,32 +417,6 @@ public void testAlterHiveTable() { () -> tableCatalog.alterTable(tableIdentifier, tableChange6)); Assertions.assertTrue(exception.getMessage().contains("Cannot add column with duplicate name")); - TableChange tableChange8 = - TableChange.addColumn( - new String[] {"col_3"}, Types.ByteType.get(), "comment", Literals.NULL); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> tableCatalog.alterTable(tableIdentifier, tableChange8)); - Assertions.assertTrue( - exception - .getMessage() - .contains( - "The DEFAULT constraint for column is only supported since Hive 3.0, " - + "but the current Gravitino Hive catalog only supports Hive 2.x"), - "The exception message is: " + exception.getMessage()); - - TableChange tableChange9 = - TableChange.updateColumnDefaultValue( - new String[] {"col_1"}, Literals.of("0", Types.ByteType.get())); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> tableCatalog.alterTable(tableIdentifier, tableChange9)); - - Assertions.assertEquals( - "Hive does not support altering column default value", exception.getMessage()); - // test alter tableCatalog.alterTable( tableIdentifier, diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 8207b3bebae..c660d8185d3 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -507,6 +507,30 @@ public void testCreateHiveTable() throws TException, InterruptedException { .contains( "The NOT NULL constraint for column is only supported since Hive 3.0, " + "but the current Gravitino Hive catalog only supports Hive 2.x")); + + // test column default value + Column withDefault = + Column.of( + "default_column", Types.StringType.get(), "default column", true, false, Literals.NULL); + exception = + assertThrows( + IllegalArgumentException.class, + () -> + catalog + .asTableCatalog() + .createTable( + nameIdentifier, + new Column[] {withDefault}, + TABLE_COMMENT, + properties, + Transforms.EMPTY_TRANSFORM)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "The DEFAULT constraint for column is only supported since Hive 3.0, " + + "but the current Gravitino Hive catalog only supports Hive 2.x"), + "The exception message is: " + exception.getMessage()); } @Test @@ -1134,6 +1158,20 @@ public void testAlterHiveTable() throws TException, InterruptedException { "The NOT NULL constraint for column is only supported since Hive 3.0," + " but the current Gravitino Hive catalog only supports Hive 2.x. Illegal column: hive_col_name1")); + // test update column default value exception + TableChange updateDefaultValue = + TableChange.updateColumnDefaultValue(new String[] {HIVE_COL_NAME1}, Literals.NULL); + exception = + assertThrows( + IllegalArgumentException.class, () -> tableCatalog.alterTable(id, updateDefaultValue)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "The DEFAULT constraint for column is only supported since Hive 3.0, " + + "but the current Gravitino Hive catalog only supports Hive 2.x"), + "The exception message is: " + exception.getMessage()); + // test updateColumnPosition exception Column col1 = Column.of("name", Types.StringType.get(), "comment"); Column col2 = Column.of("address", Types.StringType.get(), "comment"); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java index 8ffc98491f6..ffa8c2a0cdd 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.rel.SupportsSchemas; import com.datastrato.gravitino.rel.TableCatalog; import java.util.Map; @@ -31,6 +32,11 @@ protected CatalogOperations newOps(Map config) { return ops; } + @Override + public Capability newCapability() { + return new IcebergCatalogCapability(); + } + /** @return The Iceberg catalog operations as {@link IcebergCatalogOperations}. */ @Override public SupportsSchemas asSchemas() { diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java new file mode 100644 index 00000000000..dbaa85b09d6 --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java @@ -0,0 +1,17 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.lakehouse.iceberg; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class IcebergCatalogCapability implements Capability { + @Override + public CapabilityResult columnDefaultValue() { + // Iceberg column default value is WIP, see + // https://github.com/apache/iceberg/pull/4525 + return CapabilityResult.unsupported("Iceberg does not support column default value."); + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index e3f7a0a1ad3..b3fc85e5883 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -481,16 +481,13 @@ public Table createTable( IcebergColumn[] icebergColumns = Arrays.stream(columns) .map( - column -> { - IcebergTableOpsHelper.validateColumnDefaultValue( - column.name(), column.defaultValue()); - return IcebergColumn.builder() - .withName(column.name()) - .withType(column.dataType()) - .withComment(column.comment()) - .withNullable(column.nullable()) - .build(); - }) + column -> + IcebergColumn.builder() + .withName(column.name()) + .withType(column.dataType()) + .withComment(column.comment()) + .withNullable(column.nullable()) + .build()) .toArray(IcebergColumn[]::new); IcebergTable createdTable = diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java index 29f4acc64cb..6c87ee98a49 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java @@ -7,7 +7,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.ConvertUtil; -import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.TableChange.AddColumn; import com.datastrato.gravitino.rel.TableChange.After; @@ -22,7 +21,6 @@ import com.datastrato.gravitino.rel.TableChange.UpdateColumnPosition; import com.datastrato.gravitino.rel.TableChange.UpdateColumnType; import com.datastrato.gravitino.rel.TableChange.UpdateComment; -import com.datastrato.gravitino.rel.expressions.Expression; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -193,9 +191,6 @@ private void doAddColumn( parentStruct = icebergTableSchema.asStruct(); } - validateColumnDefaultValue( - String.join(".", addColumn.fieldName()), addColumn.getDefaultValue()); - if (addColumn.isAutoIncrement()) { throw new IllegalArgumentException("Iceberg doesn't support auto increment column"); } @@ -259,8 +254,6 @@ private void alterTableColumn( icebergUpdateSchema, (TableChange.UpdateColumnNullability) change); } else if (change instanceof TableChange.UpdateColumnAutoIncrement) { throw new IllegalArgumentException("Iceberg doesn't support auto increment column"); - } else if (change instanceof TableChange.UpdateColumnDefaultValue) { - throw new IllegalArgumentException("Iceberg doesn't support update column default value"); } else { throw new NotSupportedException( "Iceberg doesn't support " + change.getClass().getSimpleName() + " for now"); @@ -269,14 +262,6 @@ private void alterTableColumn( icebergUpdateSchema.commit(); } - public static void validateColumnDefaultValue(String fieldName, Expression defaultValue) { - // Iceberg column default value is WIP, see - // https://github.com/apache/iceberg/pull/4525 - Preconditions.checkArgument( - defaultValue.equals(Column.DEFAULT_VALUE_NOT_SET), - "Iceberg does not support column default value. Illegal column: " + fieldName); - } - public IcebergTableChange buildIcebergTableChanges( NameIdentifier gravitinoNameIdentifier, TableChange... tableChanges) { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java index 859093d9b88..c5367f86a34 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java @@ -4,7 +4,6 @@ */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; -import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.bucket; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.day; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.identity; @@ -25,7 +24,6 @@ import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; -import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.rel.expressions.sorts.NullOrdering; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; @@ -216,31 +214,6 @@ public void testCreateIcebergTable() { Distributions.NONE, sortOrders)); Assertions.assertTrue(exception.getMessage().contains("Table already exists")); - - IcebergColumn withDefaultValue = - IcebergColumn.builder() - .withName("col") - .withType(Types.DateType.get()) - .withComment(ICEBERG_COMMENT) - .withNullable(false) - .withDefaultValue(Literals.NULL) - .build(); - - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.createTable( - tableIdentifier, - new Column[] {withDefaultValue}, - ICEBERG_COMMENT, - properties, - EMPTY_TRANSFORM, - Distributions.NONE, - null)); - Assertions.assertTrue( - exception.getMessage().contains("Iceberg does not support column default value"), - "The exception message is: " + exception.getMessage()); } @Test @@ -481,21 +454,6 @@ public void testAlterIcebergTable() { }; Assertions.assertArrayEquals(expected, alteredTable.columns()); - // test add column with default value exception - TableChange withDefaultValue = - TableChange.addColumn( - new String[] {"col_3"}, Types.StringType.get(), "comment", Literals.NULL); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - tableCatalog.alterTable( - NameIdentifier.of(tableIdentifier.namespace(), "test_iceberg_table_new"), - withDefaultValue)); - Assertions.assertTrue( - exception.getMessage().contains("Iceberg does not support column default value"), - "The exception message is: " + exception.getMessage()); - // test delete column change icebergCatalog .asTableCatalog() diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java index e2a330b94c4..95e54dbf2ae 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergIT.java @@ -701,7 +701,8 @@ public void testAlterIcebergTable() { Assertions.assertTrue( illegalArgumentException .getMessage() - .contains("Iceberg doesn't support update column default value")); + .contains("Iceberg does not support column default value. Illegal column: name"), + "The exception is: " + illegalArgumentException.getMessage()); catalog .asTableCatalog() diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java index c53de1003bf..d08587987ba 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.Expression; import com.google.common.base.Preconditions; import java.util.Arrays; @@ -30,6 +31,10 @@ public static TableChange[] applyCapabilities(Capability capabilities, TableChan } else if (change instanceof TableChange.UpdateColumnNullability) { return applyCapabilities( (TableChange.UpdateColumnNullability) change, capabilities); + + } else if (change instanceof TableChange.UpdateColumnDefaultValue) { + return applyCapabilities( + ((TableChange.UpdateColumnDefaultValue) change), capabilities); } return change; }) @@ -72,6 +77,18 @@ private static TableChange applyCapabilities( updateColumnNullability.nullable()); } + private static TableChange applyCapabilities( + TableChange.UpdateColumnDefaultValue updateColumnDefaultValue, Capability capabilities) { + applyColumnDefaultValue( + String.join(".", updateColumnDefaultValue.fieldName()), + updateColumnDefaultValue.getNewDefaultValue(), + capabilities); + + return TableChange.updateColumnDefaultValue( + applyCaseSensitiveOnColumnName(updateColumnDefaultValue.fieldName(), capabilities), + updateColumnDefaultValue.getNewDefaultValue()); + } + private static Column applyCapabilities(Column column, Capability capabilities) { applyColumnNotNull(column, capabilities); applyColumnDefaultValue(column, capabilities); @@ -112,12 +129,14 @@ private static void applyColumnNotNull( } private static void applyColumnDefaultValue(Column column, Capability capabilities) { + applyColumnDefaultValue(column.name(), column.defaultValue(), capabilities); + } + + private static void applyColumnDefaultValue( + String columnName, Expression defaultValue, Capability capabilities) { Preconditions.checkArgument( - capabilities.columnDefaultValue().supported() - || DEFAULT_VALUE_NOT_SET.equals(column.defaultValue()), - capabilities.columnDefaultValue().unsupportedMessage() - + " Illegal column: " - + column.name()); + capabilities.columnDefaultValue().supported() || DEFAULT_VALUE_NOT_SET.equals(defaultValue), + capabilities.columnDefaultValue().unsupportedMessage() + " Illegal column: " + columnName); } private static void applyNameSpecification( diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java index dd0cf713ea1..83ec0b4cd45 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java @@ -77,7 +77,7 @@ public abstract class BaseCatalog */ @Evolving protected Capability newCapability() { - return new Capability() {}; + return Capability.DEFAULT; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java index 326f96867f2..9c6dde58dc3 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java @@ -13,6 +13,8 @@ @Evolving public interface Capability { + Capability DEFAULT = new DefaultCapability(); + /** The scope of the capability. */ enum Scope { CATALOG, @@ -30,7 +32,7 @@ enum Scope { * @return The check result of the not null constraint. */ default CapabilityResult columnNotNull() { - return CapabilityResult.SUPPORTED; + return DEFAULT.columnNotNull(); } /** @@ -39,7 +41,7 @@ default CapabilityResult columnNotNull() { * @return The check result of the default value. */ default CapabilityResult columnDefaultValue() { - return CapabilityResult.SUPPORTED; + return DEFAULT.columnDefaultValue(); } /** @@ -49,7 +51,7 @@ default CapabilityResult columnDefaultValue() { * @return The capability of the case-sensitive on name. */ default CapabilityResult caseSensitiveOnName(Scope scope) { - return CapabilityResult.SUPPORTED; + return DEFAULT.caseSensitiveOnName(scope); } /** @@ -60,7 +62,7 @@ default CapabilityResult caseSensitiveOnName(Scope scope) { * @return The capability of the specification on name. */ default CapabilityResult specificationOnName(Scope scope, String name) { - return CapabilityResult.SUPPORTED; + return DEFAULT.specificationOnName(scope, name); } /** @@ -70,7 +72,35 @@ default CapabilityResult specificationOnName(Scope scope, String name) { * @return The capability of the managed storage. */ default CapabilityResult managedStorage(Scope scope) { - return CapabilityResult.unsupported( - String.format("The %s entity is not fully managed by Gravitino.", scope)); + return DEFAULT.managedStorage(scope); + } + + /** The default implementation of the capability. */ + class DefaultCapability implements Capability { + @Override + public CapabilityResult columnNotNull() { + return CapabilityResult.SUPPORTED; + } + + @Override + public CapabilityResult columnDefaultValue() { + return CapabilityResult.SUPPORTED; + } + + @Override + public CapabilityResult caseSensitiveOnName(Scope scope) { + return CapabilityResult.SUPPORTED; + } + + @Override + public CapabilityResult specificationOnName(Scope scope, String name) { + return CapabilityResult.SUPPORTED; + } + + @Override + public CapabilityResult managedStorage(Scope scope) { + return CapabilityResult.unsupported( + String.format("The %s entity is not fully managed by Gravitino.", scope)); + } } } From 49b07a2a4d71b4d5e7492fe504582d311fb10da5 Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 16 Apr 2024 14:42:36 +0800 Subject: [PATCH 035/106] [#2903]feat(spark-connector): register specific catalog to Spark catalog manager (#2906) ### What changes were proposed in this pull request? register specific catalog like `GravitinoHiveCatalog` to Spark catalog manager, no the general `GravitinoCatalog` ### Why are the changes needed? The specific catalog could implement different interfaces like `FunctionCatalog`, to support Iceberg partition. Fix: #2903 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../connector/GravitinoCatalogAdaptor.java | 54 --------- .../GravitinoCatalogAdaptorFactory.java | 27 ----- ...GravitinoCatalog.java => BaseCatalog.java} | 70 +++++++++--- .../catalog/GravitinoCatalogManager.java | 7 +- ...Adaptor.java => GravitinoHiveCatalog.java} | 42 ++++--- ...ptor.java => GravitinoIcebergCatalog.java} | 104 +++++++++--------- .../plugin/GravitinoDriverPlugin.java | 61 +++++++--- .../catalog/TestTransformTableChange.java | 26 ++--- 8 files changed, 191 insertions(+), 200 deletions(-) delete mode 100644 spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptor.java delete mode 100644 spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptorFactory.java rename spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/{GravitinoCatalog.java => BaseCatalog.java} (87%) rename spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/{HiveAdaptor.java => GravitinoHiveCatalog.java} (73%) rename spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/{IcebergAdaptor.java => GravitinoIcebergCatalog.java} (89%) diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptor.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptor.java deleted file mode 100644 index a1a9ab90e94..00000000000 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptor.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.spark.connector; - -import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; -import java.util.Map; -import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.TableCatalog; -import org.apache.spark.sql.util.CaseInsensitiveStringMap; - -/** - * GravitinoCatalogAdaptor provides a unified interface for different catalogs to adapt to - * GravitinoCatalog. - */ -public interface GravitinoCatalogAdaptor { - - /** - * Get a PropertiesConverter to transform properties between Gravitino and Spark. - * - * @return an PropertiesConverter - */ - PropertiesConverter getPropertiesConverter(); - - /** - * Create a specific Spark table, combined with gravitinoTable to do DML operations and - * sparkCatalog to do IO operations. - * - * @param identifier Spark's table identifier - * @param gravitinoTable Gravitino table to do DDL operations - * @param sparkCatalog specific Spark catalog to do IO operations - * @param propertiesConverter transform properties between Gravitino and Spark - * @return a specific Spark table - */ - SparkBaseTable createSparkTable( - Identifier identifier, - Table gravitinoTable, - TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter); - - /** - * Create a specific Spark catalog, mainly used to create Spark table. - * - * @param name catalog name - * @param options catalog options from configuration - * @param properties catalog properties from Gravitino - * @return a specific Spark catalog - */ - TableCatalog createAndInitSparkCatalog( - String name, CaseInsensitiveStringMap options, Map properties); -} diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptorFactory.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptorFactory.java deleted file mode 100644 index 0599f5cad1b..00000000000 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/GravitinoCatalogAdaptorFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.spark.connector; - -import com.datastrato.gravitino.spark.connector.hive.HiveAdaptor; -import com.datastrato.gravitino.spark.connector.iceberg.IcebergAdaptor; -import java.util.Locale; - -/** - * GravitinoCatalogAdaptorFactory creates a specific GravitinoCatalogAdaptor according to the - * catalog provider. - */ -public class GravitinoCatalogAdaptorFactory { - public static GravitinoCatalogAdaptor createGravitinoAdaptor(String provider) { - switch (provider.toLowerCase(Locale.ROOT)) { - case "hive": - return new HiveAdaptor(); - case "lakehouse-iceberg": - return new IcebergAdaptor(); - default: - throw new RuntimeException(String.format("Provider:%s is not supported yet", provider)); - } - } -} diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalog.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java similarity index 87% rename from spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalog.java rename to spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java index e76f7f39939..bd6c26f9241 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalog.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java @@ -15,12 +15,11 @@ import com.datastrato.gravitino.rel.SchemaChange; import com.datastrato.gravitino.rel.expressions.literals.Literals; import com.datastrato.gravitino.spark.connector.ConnectorConstants; -import com.datastrato.gravitino.spark.connector.GravitinoCatalogAdaptor; -import com.datastrato.gravitino.spark.connector.GravitinoCatalogAdaptorFactory; import com.datastrato.gravitino.spark.connector.PropertiesConverter; import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.SparkTransformConverter.DistributionAndSortOrdersInfo; import com.datastrato.gravitino.spark.connector.SparkTypeConverter; +import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.util.Arrays; @@ -47,11 +46,18 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** - * GravitinoCatalog is the class registered to Spark CatalogManager, it's lazy loaded which means - * Spark connector loads GravitinoCatalog when it's used. It will create different Spark Tables from - * different Gravitino catalogs. + * BaseCatalog acts as the foundational class for Spark CatalogManager registration, enabling + * seamless integration of various data source catalogs within Spark's ecosystem. This class is + * pivotal in bridging Spark with diverse data sources, ensuring a unified approach to data + * management and manipulation across the platform. + * + *

This class implements essential interfaces for the table and namespace management. Subclasses + * can extend BaseCatalog to implement more specific interfaces tailored to the needs of different + * data sources. Its lazy loading design ensures that instances of BaseCatalog are created only when + * needed, optimizing resource utilization and minimizing the overhead associated with + * initialization. */ -public class GravitinoCatalog implements TableCatalog, SupportsNamespaces { +public abstract class BaseCatalog implements TableCatalog, SupportsNamespaces { // The specific Spark catalog to do IO operations, different catalogs have different spark catalog // implementations, like HiveTableCatalog for Hive, JDBCTableCatalog for JDBC, SparkCatalog for // Iceberg. @@ -63,14 +69,46 @@ public class GravitinoCatalog implements TableCatalog, SupportsNamespaces { private final String metalakeName; private String catalogName; private final GravitinoCatalogManager gravitinoCatalogManager; - // Different catalog use GravitinoCatalogAdaptor to adapt to GravitinoCatalog - private GravitinoCatalogAdaptor gravitinoAdaptor; - public GravitinoCatalog() { + protected BaseCatalog() { gravitinoCatalogManager = GravitinoCatalogManager.get(); metalakeName = gravitinoCatalogManager.getMetalakeName(); } + /** + * Create a specific Spark catalog, mainly used to create Spark table. + * + * @param name catalog name + * @param options catalog options from configuration + * @param properties catalog properties from Gravitino + * @return a specific Spark catalog + */ + protected abstract TableCatalog createAndInitSparkCatalog( + String name, CaseInsensitiveStringMap options, Map properties); + + /** + * Create a specific Spark table, combined with gravitinoTable to do DML operations and + * sparkCatalog to do IO operations. + * + * @param identifier Spark's table identifier + * @param gravitinoTable Gravitino table to do DDL operations + * @param sparkCatalog specific Spark catalog to do IO operations + * @param propertiesConverter transform properties between Gravitino and Spark + * @return a specific Spark table + */ + protected abstract SparkBaseTable createSparkTable( + Identifier identifier, + com.datastrato.gravitino.rel.Table gravitinoTable, + TableCatalog sparkCatalog, + PropertiesConverter propertiesConverter); + + /** + * Get a PropertiesConverter to transform properties between Gravitino and Spark. + * + * @return an PropertiesConverter + */ + protected abstract PropertiesConverter getPropertiesConverter(); + @Override public void initialize(String name, CaseInsensitiveStringMap options) { this.catalogName = name; @@ -78,11 +116,9 @@ public void initialize(String name, CaseInsensitiveStringMap options) { String provider = gravitinoCatalogClient.provider(); Preconditions.checkArgument( StringUtils.isNotBlank(provider), name + " catalog provider is empty"); - this.gravitinoAdaptor = GravitinoCatalogAdaptorFactory.createGravitinoAdaptor(provider); this.sparkCatalog = - gravitinoAdaptor.createAndInitSparkCatalog( - name, options, gravitinoCatalogClient.properties()); - this.propertiesConverter = gravitinoAdaptor.getPropertiesConverter(); + createAndInitSparkCatalog(name, options, gravitinoCatalogClient.properties()); + this.propertiesConverter = getPropertiesConverter(); } @Override @@ -147,7 +183,7 @@ public Table createTable( partitionings, distributionAndSortOrdersInfo.getDistribution(), distributionAndSortOrdersInfo.getSortOrders()); - return gravitinoAdaptor.createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable(ident, table, sparkCatalog, propertiesConverter); } catch (NoSuchSchemaException e) { throw new NoSuchNamespaceException(ident.namespace()); } catch (com.datastrato.gravitino.exceptions.TableAlreadyExistsException e) { @@ -164,7 +200,7 @@ public Table loadTable(Identifier ident) throws NoSuchTableException { .asTableCatalog() .loadTable(NameIdentifier.of(metalakeName, catalogName, database, ident.name())); // Will create a catalog specific table - return gravitinoAdaptor.createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable(ident, table, sparkCatalog, propertiesConverter); } catch (com.datastrato.gravitino.exceptions.NoSuchTableException e) { throw new NoSuchTableException(ident); } @@ -182,7 +218,7 @@ public Table createTable( public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchTableException { com.datastrato.gravitino.rel.TableChange[] gravitinoTableChanges = Arrays.stream(changes) - .map(GravitinoCatalog::transformTableChange) + .map(BaseCatalog::transformTableChange) .toArray(com.datastrato.gravitino.rel.TableChange[]::new); try { com.datastrato.gravitino.rel.Table table = @@ -191,7 +227,7 @@ public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchT .alterTable( NameIdentifier.of(metalakeName, catalogName, getDatabase(ident), ident.name()), gravitinoTableChanges); - return gravitinoAdaptor.createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable(ident, table, sparkCatalog, propertiesConverter); } catch (com.datastrato.gravitino.exceptions.NoSuchTableException e) { throw new NoSuchTableException(ident); } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalogManager.java index e00f63b6203..9884be2d098 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalogManager.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/GravitinoCatalogManager.java @@ -13,9 +13,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import java.util.Arrays; -import java.util.Set; +import java.util.Map; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,8 +79,8 @@ public void loadRelationalCatalogs() { .forEach(catalog -> gravitinoCatalogs.put(catalog.name(), catalog)); } - public Set getCatalogNames() { - return gravitinoCatalogs.asMap().keySet().stream().collect(Collectors.toSet()); + public Map getCatalogs() { + return gravitinoCatalogs.asMap(); } private Catalog loadCatalog(String catalogName) { diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/HiveAdaptor.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java similarity index 73% rename from spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/HiveAdaptor.java rename to spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java index 795c6311aef..64b61754a2e 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/HiveAdaptor.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java @@ -6,9 +6,9 @@ package com.datastrato.gravitino.spark.connector.hive; import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.spark.connector.GravitinoCatalogAdaptor; import com.datastrato.gravitino.spark.connector.GravitinoSparkConfig; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.catalog.BaseCatalog; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -19,29 +19,13 @@ import org.apache.spark.sql.connector.catalog.TableCatalog; import org.apache.spark.sql.util.CaseInsensitiveStringMap; -/** HiveAdaptor provides specific operations for Hive Catalog to adapt to GravitinoCatalog. */ -public class HiveAdaptor implements GravitinoCatalogAdaptor { +public class GravitinoHiveCatalog extends BaseCatalog { @Override - public PropertiesConverter getPropertiesConverter() { - return new HivePropertiesConverter(); - } - - @Override - public SparkBaseTable createSparkTable( - Identifier identifier, - Table gravitinoTable, - TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { - return new SparkHiveTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); - } - - @Override - public TableCatalog createAndInitSparkCatalog( - String name, CaseInsensitiveStringMap options, Map catalogProperties) { - Preconditions.checkArgument( - catalogProperties != null, "Hive Catalog properties should not be null"); - String metastoreUri = catalogProperties.get(GravitinoSparkConfig.GRAVITINO_HIVE_METASTORE_URI); + protected TableCatalog createAndInitSparkCatalog( + String name, CaseInsensitiveStringMap options, Map properties) { + Preconditions.checkArgument(properties != null, "Hive Catalog properties should not be null"); + String metastoreUri = properties.get(GravitinoSparkConfig.GRAVITINO_HIVE_METASTORE_URI); Preconditions.checkArgument( StringUtils.isNotBlank(metastoreUri), "Couldn't get " @@ -55,4 +39,18 @@ public TableCatalog createAndInitSparkCatalog( return hiveCatalog; } + + @Override + protected SparkBaseTable createSparkTable( + Identifier identifier, + Table gravitinoTable, + TableCatalog sparkCatalog, + PropertiesConverter propertiesConverter) { + return new SparkHiveTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); + } + + @Override + protected PropertiesConverter getPropertiesConverter() { + return new HivePropertiesConverter(); + } } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/IcebergAdaptor.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java similarity index 89% rename from spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/IcebergAdaptor.java rename to spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java index cf73dfb0427..e3b9783d41e 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/IcebergAdaptor.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java @@ -6,8 +6,8 @@ package com.datastrato.gravitino.spark.connector.iceberg; import com.datastrato.gravitino.rel.Table; -import com.datastrato.gravitino.spark.connector.GravitinoCatalogAdaptor; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.catalog.BaseCatalog; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -19,8 +19,60 @@ import org.apache.spark.sql.connector.catalog.TableCatalog; import org.apache.spark.sql.util.CaseInsensitiveStringMap; -/** IcebergAdaptor provides specific operations for Iceberg Catalog to adapt to GravitinoCatalog. */ -public class IcebergAdaptor implements GravitinoCatalogAdaptor { +/** + * The GravitinoIcebergCatalog class extends the BaseCatalog to integrate with the Iceberg table + * format, providing specialized support for Iceberg-specific functionalities within Spark's + * ecosystem. This implementation can further adapt to specific interfaces such as + * StagingTableCatalog and FunctionCatalog, allowing for advanced operations like table staging and + * function management tailored to the needs of Iceberg tables. + */ +public class GravitinoIcebergCatalog extends BaseCatalog { + + @Override + protected TableCatalog createAndInitSparkCatalog( + String name, CaseInsensitiveStringMap options, Map properties) { + Preconditions.checkArgument( + properties != null, "Iceberg Catalog properties should not be null"); + + String catalogBackend = + properties.get(IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND); + Preconditions.checkArgument( + StringUtils.isNotBlank(catalogBackend), "Iceberg Catalog backend should not be empty."); + + HashMap all = new HashMap<>(options); + + switch (catalogBackend.toLowerCase(Locale.ENGLISH)) { + case IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_HIVE: + initHiveProperties(catalogBackend, properties, all); + break; + case IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_JDBC: + initJdbcProperties(catalogBackend, properties, all); + break; + default: + // SparkCatalog does not support Memory type catalog + throw new IllegalArgumentException( + "Unsupported Iceberg Catalog backend: " + catalogBackend); + } + + TableCatalog icebergCatalog = new SparkCatalog(); + icebergCatalog.initialize(name, new CaseInsensitiveStringMap(all)); + + return icebergCatalog; + } + + @Override + protected SparkBaseTable createSparkTable( + Identifier identifier, + Table gravitinoTable, + TableCatalog sparkCatalog, + PropertiesConverter propertiesConverter) { + return new SparkIcebergTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); + } + + @Override + protected PropertiesConverter getPropertiesConverter() { + return new IcebergPropertiesConverter(); + } private void initHiveProperties( String catalogBackend, @@ -96,50 +148,4 @@ private void initJdbcProperties( icebergProperties.put(IcebergPropertiesConstants.GRAVITINO_ICEBERG_JDBC_PASSWORD, jdbcPassword); icebergProperties.put(IcebergPropertiesConstants.GRAVITINO_ICEBERG_JDBC_DRIVER, jdbcDriver); } - - @Override - public PropertiesConverter getPropertiesConverter() { - return new IcebergPropertiesConverter(); - } - - @Override - public SparkBaseTable createSparkTable( - Identifier identifier, - Table gravitinoTable, - TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { - return new SparkIcebergTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); - } - - @Override - public TableCatalog createAndInitSparkCatalog( - String name, CaseInsensitiveStringMap options, Map properties) { - Preconditions.checkArgument( - properties != null, "Iceberg Catalog properties should not be null"); - - String catalogBackend = - properties.get(IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND); - Preconditions.checkArgument( - StringUtils.isNotBlank(catalogBackend), "Iceberg Catalog backend should not be empty."); - - HashMap all = new HashMap<>(options); - - switch (catalogBackend.toLowerCase(Locale.ENGLISH)) { - case IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_HIVE: - initHiveProperties(catalogBackend, properties, all); - break; - case IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_JDBC: - initJdbcProperties(catalogBackend, properties, all); - break; - default: - // SparkCatalog does not support Memory type catalog - throw new IllegalArgumentException( - "Unsupported Iceberg Catalog backend: " + catalogBackend); - } - - TableCatalog icebergCatalog = new SparkCatalog(); - icebergCatalog.initialize(name, new CaseInsensitiveStringMap(all)); - - return icebergCatalog; - } } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java index 88235c3877e..3f830de2cdc 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java @@ -5,13 +5,15 @@ package com.datastrato.gravitino.spark.connector.plugin; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.spark.connector.GravitinoSparkConfig; -import com.datastrato.gravitino.spark.connector.catalog.GravitinoCatalog; import com.datastrato.gravitino.spark.connector.catalog.GravitinoCatalogManager; +import com.datastrato.gravitino.spark.connector.hive.GravitinoHiveCatalog; +import com.datastrato.gravitino.spark.connector.iceberg.GravitinoIcebergCatalog; import com.google.common.base.Preconditions; import java.util.Collections; +import java.util.Locale; import java.util.Map; -import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; @@ -45,8 +47,7 @@ public Map init(SparkContext sc, PluginContext pluginContext) { catalogManager = GravitinoCatalogManager.create(gravitinoUri, metalake); catalogManager.loadRelationalCatalogs(); - Set catalogNames = catalogManager.getCatalogNames(); - registerGravitinoCatalogs(conf, catalogNames); + registerGravitinoCatalogs(conf, catalogManager.getCatalogs()); registerSqlExtensions(); return Collections.emptyMap(); } @@ -58,16 +59,48 @@ public void shutdown() { } } - private void registerGravitinoCatalogs(SparkConf sparkConf, Set catalogNames) { - catalogNames.forEach( - catalogName -> { - String sparkCatalogConfigName = "spark.sql.catalog." + catalogName; - Preconditions.checkArgument( - !sparkConf.contains(sparkCatalogConfigName), - catalogName + " is already registered to SparkCatalogManager"); - sparkConf.set(sparkCatalogConfigName, GravitinoCatalog.class.getName()); - LOG.info("Register {} catalog to Spark catalog manager", catalogName); - }); + private void registerGravitinoCatalogs( + SparkConf sparkConf, Map gravitinoCatalogs) { + gravitinoCatalogs + .entrySet() + .forEach( + entry -> { + String catalogName = entry.getKey(); + Catalog gravitinoCatalog = entry.getValue(); + String provider = gravitinoCatalog.provider(); + try { + registerCatalog(sparkConf, catalogName, provider); + } catch (Exception e) { + LOG.warn("Register catalog {} failed.", catalogName, e); + } + }); + } + + private void registerCatalog(SparkConf sparkConf, String catalogName, String provider) { + if (StringUtils.isBlank(provider)) { + LOG.warn("Skip registering {} because catalog provider is empty.", catalogName); + return; + } + + String catalogClassName; + switch (provider.toLowerCase(Locale.ROOT)) { + case "hive": + catalogClassName = GravitinoHiveCatalog.class.getName(); + break; + case "lakehouse-iceberg": + catalogClassName = GravitinoIcebergCatalog.class.getName(); + break; + default: + LOG.warn("Skip registering {} because {} is not supported yet.", catalogName, provider); + return; + } + + String sparkCatalogConfigName = "spark.sql.catalog." + catalogName; + Preconditions.checkArgument( + !sparkConf.contains(sparkCatalogConfigName), + catalogName + " is already registered to SparkCatalogManager"); + sparkConf.set(sparkCatalogConfigName, catalogClassName); + LOG.info("Register {} catalog to Spark catalog manager.", catalogName); } // Todo inject Iceberg extensions diff --git a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java index e5e85b6b785..5a14a65aa14 100644 --- a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java +++ b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java @@ -19,7 +19,7 @@ public class TestTransformTableChange { void testTransformSetProperty() { TableChange sparkSetProperty = TableChange.setProperty("key", "value"); com.datastrato.gravitino.rel.TableChange tableChange = - GravitinoCatalog.transformTableChange(sparkSetProperty); + BaseCatalog.transformTableChange(sparkSetProperty); Assertions.assertTrue( tableChange instanceof com.datastrato.gravitino.rel.TableChange.SetProperty); com.datastrato.gravitino.rel.TableChange.SetProperty gravitinoSetProperty = @@ -32,7 +32,7 @@ void testTransformSetProperty() { void testTransformRemoveProperty() { TableChange sparkRemoveProperty = TableChange.removeProperty("key"); com.datastrato.gravitino.rel.TableChange tableChange = - GravitinoCatalog.transformTableChange(sparkRemoveProperty); + BaseCatalog.transformTableChange(sparkRemoveProperty); Assertions.assertTrue( tableChange instanceof com.datastrato.gravitino.rel.TableChange.RemoveProperty); com.datastrato.gravitino.rel.TableChange.RemoveProperty gravitinoRemoveProperty = @@ -48,7 +48,7 @@ void testTransformRenameColumn() { TableChange.RenameColumn sparkRenameColumn = (TableChange.RenameColumn) TableChange.renameColumn(oldFiledsName, newFiledName); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(sparkRenameColumn); + BaseCatalog.transformTableChange(sparkRenameColumn); Assertions.assertTrue( gravitinoChange instanceof com.datastrato.gravitino.rel.TableChange.RenameColumn); @@ -67,7 +67,7 @@ void testTransformUpdateColumnComment() { TableChange.UpdateColumnComment updateColumnComment = (TableChange.UpdateColumnComment) TableChange.updateColumnComment(fieldNames, newComment); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(updateColumnComment); + BaseCatalog.transformTableChange(updateColumnComment); Assertions.assertTrue( gravitinoChange instanceof com.datastrato.gravitino.rel.TableChange.UpdateColumnComment); @@ -92,7 +92,7 @@ void testTransformAddColumn() { TableChange.addColumn( new String[] {"col1"}, DataTypes.StringType, true, "", first, defaultValue); com.datastrato.gravitino.rel.TableChange gravitinoChangeFirst = - GravitinoCatalog.transformTableChange(sparkAddColumnFirst); + BaseCatalog.transformTableChange(sparkAddColumnFirst); Assertions.assertTrue( gravitinoChangeFirst instanceof com.datastrato.gravitino.rel.TableChange.AddColumn); @@ -112,7 +112,7 @@ void testTransformAddColumn() { TableChange.addColumn( new String[] {"col1"}, DataTypes.StringType, true, "", after, defaultValue); com.datastrato.gravitino.rel.TableChange gravitinoChangeAfter = - GravitinoCatalog.transformTableChange(sparkAddColumnAfter); + BaseCatalog.transformTableChange(sparkAddColumnAfter); Assertions.assertTrue( gravitinoChangeAfter instanceof com.datastrato.gravitino.rel.TableChange.AddColumn); @@ -132,7 +132,7 @@ void testTransformAddColumn() { TableChange.addColumn( new String[] {"col1"}, DataTypes.StringType, true, "", null, defaultValue); com.datastrato.gravitino.rel.TableChange gravitinoChangeDefault = - GravitinoCatalog.transformTableChange(sparkAddColumnDefault); + BaseCatalog.transformTableChange(sparkAddColumnDefault); Assertions.assertTrue( gravitinoChangeDefault instanceof com.datastrato.gravitino.rel.TableChange.AddColumn); @@ -153,7 +153,7 @@ void testTransformDeleteColumn() { TableChange.DeleteColumn sparkDeleteColumn = (TableChange.DeleteColumn) TableChange.deleteColumn(new String[] {"col1"}, true); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(sparkDeleteColumn); + BaseCatalog.transformTableChange(sparkDeleteColumn); Assertions.assertTrue( gravitinoChange instanceof com.datastrato.gravitino.rel.TableChange.DeleteColumn); @@ -170,7 +170,7 @@ void testTransformUpdateColumnType() { (TableChange.UpdateColumnType) TableChange.updateColumnType(new String[] {"col1"}, DataTypes.StringType); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(sparkUpdateColumnType); + BaseCatalog.transformTableChange(sparkUpdateColumnType); Assertions.assertTrue( gravitinoChange instanceof com.datastrato.gravitino.rel.TableChange.UpdateColumnType); @@ -192,7 +192,7 @@ void testTransformUpdateColumnPosition() { (TableChange.UpdateColumnPosition) TableChange.updateColumnPosition(new String[] {"col1"}, first); com.datastrato.gravitino.rel.TableChange gravitinoChangeFirst = - GravitinoCatalog.transformTableChange(sparkUpdateColumnFirst); + BaseCatalog.transformTableChange(sparkUpdateColumnFirst); Assertions.assertTrue( gravitinoChangeFirst @@ -210,7 +210,7 @@ void testTransformUpdateColumnPosition() { (TableChange.UpdateColumnPosition) TableChange.updateColumnPosition(new String[] {"col1"}, after); com.datastrato.gravitino.rel.TableChange gravitinoChangeAfter = - GravitinoCatalog.transformTableChange(sparkUpdateColumnAfter); + BaseCatalog.transformTableChange(sparkUpdateColumnAfter); Assertions.assertTrue( gravitinoChangeAfter @@ -231,7 +231,7 @@ void testTransformUpdateColumnNullability() { (TableChange.UpdateColumnNullability) TableChange.updateColumnNullability(new String[] {"col1"}, true); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(sparkUpdateColumnNullability); + BaseCatalog.transformTableChange(sparkUpdateColumnNullability); Assertions.assertTrue( gravitinoChange @@ -255,7 +255,7 @@ void testUpdateColumnDefaultValue() { TableChange.updateColumnDefaultValue(fieldNames, newDedauleValue); com.datastrato.gravitino.rel.TableChange gravitinoChange = - GravitinoCatalog.transformTableChange(sparkUpdateColumnDefaultValue); + BaseCatalog.transformTableChange(sparkUpdateColumnDefaultValue); Assertions.assertTrue( gravitinoChange From 6a8cf806c57d4d274b4d4693ff0e2abbe06159ec Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 16 Apr 2024 15:32:51 +0800 Subject: [PATCH 036/106] [#2822] feat(core): support catalog events for event listener (#2891) ### What changes were proposed in this pull request? support catalog events for event listener ### Why are the changes needed? Fix: #2822 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../datastrato/gravitino/GravitinoEnv.java | 14 +- .../gravitino/catalog/CatalogDispatcher.java | 16 ++ .../catalog/CatalogEventDispatcher.java | 146 ++++++++++++++++++ .../gravitino/catalog/CatalogManager.java | 3 +- .../listener/api/event/AlterCatalogEvent.java | 60 +++++++ .../api/event/AlterCatalogFailureEvent.java | 44 ++++++ .../listener/api/event/CatalogEvent.java | 27 ++++ .../api/event/CatalogFailureEvent.java | 31 ++++ .../api/event/CreateCatalogEvent.java | 41 +++++ .../api/event/CreateCatalogFailureEvent.java | 50 ++++++ .../listener/api/event/DropCatalogEvent.java | 39 +++++ .../api/event/DropCatalogFailureEvent.java | 29 ++++ .../listener/api/event/ListCatalogEvent.java | 36 +++++ .../api/event/ListCatalogFailureEvent.java | 40 +++++ .../listener/api/event/LoadCatalogEvent.java | 38 +++++ .../api/event/LoadCatalogFailureEvent.java | 25 +++ .../listener/api/info/CatalogInfo.java | 119 ++++++++++++++ .../gravitino/server/GravitinoServer.java | 4 +- .../server/web/rest/CatalogOperations.java | 21 +-- .../web/rest/TestCatalogOperations.java | 3 +- 20 files changed, 767 insertions(+), 19 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index e224fa48b2a..1286b8c9c9e 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -6,6 +6,8 @@ import com.datastrato.gravitino.authorization.AccessControlManager; import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager; +import com.datastrato.gravitino.catalog.CatalogDispatcher; +import com.datastrato.gravitino.catalog.CatalogEventDispatcher; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.FilesetEventDispatcher; @@ -41,6 +43,8 @@ public class GravitinoEnv { private EntityStore entityStore; + private CatalogDispatcher catalogDispatcher; + private CatalogManager catalogManager; private SchemaDispatcher schemaDispatcher; @@ -131,6 +135,8 @@ public void initialize(Config config) { // Create and initialize Catalog related modules this.catalogManager = new CatalogManager(config, entityStore, idGenerator); + this.catalogDispatcher = new CatalogEventDispatcher(eventBus, catalogManager); + SchemaOperationDispatcher schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); this.schemaDispatcher = new SchemaEventDispatcher(eventBus, schemaOperationDispatcher); @@ -180,12 +186,12 @@ public EntityStore entityStore() { } /** - * Get the CatalogManager associated with the Gravitino environment. + * Get the CatalogDispatcher associated with the Gravitino environment. * - * @return The CatalogManager instance. + * @return The CatalogDispatcher instance. */ - public CatalogManager catalogManager() { - return catalogManager; + public CatalogDispatcher catalogDispatcher() { + return catalogDispatcher; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java new file mode 100644 index 00000000000..a1d4d4031b6 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.SupportsCatalogs; + +/** + * {@code CatalogDispatcher} interface acts as a specialization of the {@link SupportsCatalogs} + * interface. This interface is designed to potentially add custom behaviors or operations related + * to dispatching or handling catalog-related events or actions that are not covered by the standard + * {@code SupportsCatalogs} operations. + */ +public interface CatalogDispatcher extends SupportsCatalogs {} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java new file mode 100644 index 00000000000..dc8e1dbb575 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java @@ -0,0 +1,146 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterCatalogEvent; +import com.datastrato.gravitino.listener.api.event.AlterCatalogFailureEvent; +import com.datastrato.gravitino.listener.api.event.CreateCatalogEvent; +import com.datastrato.gravitino.listener.api.event.CreateCatalogFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropCatalogEvent; +import com.datastrato.gravitino.listener.api.event.DropCatalogFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListCatalogEvent; +import com.datastrato.gravitino.listener.api.event.ListCatalogFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadCatalogEvent; +import com.datastrato.gravitino.listener.api.event.LoadCatalogFailureEvent; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code CatalogEventDispatcher} is a decorator for {@link CatalogDispatcher} that not only + * delegates catalog operations to the underlying catalog dispatcher but also dispatches + * corresponding events to an {@link EventBus} after each operation is completed. This allows for + * event-driven workflows or monitoring of catalog operations. + */ +public class CatalogEventDispatcher implements CatalogDispatcher { + private final EventBus eventBus; + private final CatalogDispatcher dispatcher; + + /** + * Constructs a CatalogEventDispatcher with a specified EventBus and CatalogDispatcher. + * + * @param eventBus The EventBus to which events will be dispatched. + * @param dispatcher The underlying {@link CatalogDispatcher} that will perform the actual catalog + * operations. + */ + public CatalogEventDispatcher(EventBus eventBus, CatalogDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeException { + try { + NameIdentifier[] nameIdentifiers = dispatcher.listCatalogs(namespace); + eventBus.dispatchEvent(new ListCatalogEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return nameIdentifiers; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListCatalogFailureEvent(PrincipalUtils.getCurrentUserName(), e, namespace)); + throw e; + } + } + + @Override + public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeException { + try { + Catalog[] catalogs = dispatcher.listCatalogsInfo(namespace); + eventBus.dispatchEvent(new ListCatalogEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return catalogs; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListCatalogFailureEvent(PrincipalUtils.getCurrentUserName(), e, namespace)); + throw e; + } + } + + @Override + public Catalog loadCatalog(NameIdentifier ident) throws NoSuchCatalogException { + try { + Catalog catalog = dispatcher.loadCatalog(ident); + eventBus.dispatchEvent( + new LoadCatalogEvent( + PrincipalUtils.getCurrentUserName(), ident, new CatalogInfo(catalog))); + return catalog; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadCatalogFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public Catalog createCatalog( + NameIdentifier ident, + Catalog.Type type, + String provider, + String comment, + Map properties) + throws NoSuchMetalakeException, CatalogAlreadyExistsException { + try { + Catalog catalog = dispatcher.createCatalog(ident, type, provider, comment, properties); + eventBus.dispatchEvent( + new CreateCatalogEvent( + PrincipalUtils.getCurrentUserName(), ident, new CatalogInfo(catalog))); + return catalog; + } catch (Exception e) { + CatalogInfo createCatalogRequest = + new CatalogInfo(ident.name(), type, provider, comment, properties, null); + eventBus.dispatchEvent( + new CreateCatalogFailureEvent( + PrincipalUtils.getCurrentUserName(), ident, e, createCatalogRequest)); + throw e; + } + } + + @Override + public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) + throws NoSuchCatalogException, IllegalArgumentException { + try { + Catalog catalog = dispatcher.alterCatalog(ident, changes); + eventBus.dispatchEvent( + new AlterCatalogEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new CatalogInfo(catalog))); + return catalog; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterCatalogFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } + } + + @Override + public boolean dropCatalog(NameIdentifier ident) { + try { + boolean isExists = dispatcher.dropCatalog(ident); + eventBus.dispatchEvent( + new DropCatalogEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropCatalogFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 93857febf1c..884a5adfff0 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -22,7 +22,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.SupportsCatalogs; import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.HasPropertyMetadata; import com.datastrato.gravitino.connector.capability.Capability; @@ -73,7 +72,7 @@ import org.slf4j.LoggerFactory; /** Manages the catalog instances and operations. */ -public class CatalogManager implements SupportsCatalogs, Closeable { +public class CatalogManager implements CatalogDispatcher, Closeable { private static final String CATALOG_DOES_NOT_EXIST_MSG = "Catalog %s does not exist"; private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java new file mode 100644 index 00000000000..8d712f75cf3 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; + +/** Represents an event triggered upon the successful creation of a catalog. */ +@DeveloperApi +public final class AlterCatalogEvent extends CatalogEvent { + private final CatalogInfo updatedCatalogInfo; + private final CatalogChange[] catalogChanges; + + /** + * Constructs an instance of {@code AlterCatalogEvent}, encapsulating the key details about the + * successful alteration of a catalog. + * + * @param user The username of the individual responsible for initiating the catalog alteration. + * @param identifier The unique identifier of the altered catalog, serving as a clear reference + * point for the catalog in question. + * @param catalogChanges An array of {@link CatalogChange} objects representing the specific + * changes applied to the catalog during the alteration process. + * @param updatedCatalogInfo The post-alteration state of the catalog. + */ + public AlterCatalogEvent( + String user, + NameIdentifier identifier, + CatalogChange[] catalogChanges, + CatalogInfo updatedCatalogInfo) { + super(user, identifier); + this.catalogChanges = catalogChanges.clone(); + this.updatedCatalogInfo = updatedCatalogInfo; + } + + /** + * Retrieves the final state of the catalog as it was returned to the user after successful + * creation. + * + * @return A {@link CatalogInfo} instance encapsulating the comprehensive details of the newly + * created catalog. + */ + public CatalogInfo updatedCatalogInfo() { + return updatedCatalogInfo; + } + + /** + * Retrieves the specific changes that were made to the catalog during the alteration process. + * + * @return An array of {@link CatalogChange} objects detailing each modification applied to the + * catalog. + */ + public CatalogChange[] catalogChanges() { + return catalogChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java new file mode 100644 index 00000000000..a47781cdf10 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to create a catalog fails due to an + * exception. + */ +@DeveloperApi +public final class AlterCatalogFailureEvent extends CatalogFailureEvent { + private final CatalogChange[] catalogChanges; + + /** + * Constructs an {@code AlterCatalogFailureEvent} instance, capturing detailed information about + * the failed catalog alteration attempt. + * + * @param user The user who initiated the catalog alteration operation. + * @param identifier The identifier of the catalog that was attempted to be altered. + * @param exception The exception that was thrown during the catalog alteration operation. + * @param catalogChanges The changes that were attempted on the catalog. + */ + public AlterCatalogFailureEvent( + String user, NameIdentifier identifier, Exception exception, CatalogChange[] catalogChanges) { + super(user, identifier, exception); + this.catalogChanges = catalogChanges.clone(); + } + + /** + * Retrieves the specific changes that were made to the catalog during the alteration process. + * + * @return An array of {@link CatalogChange} objects detailing each modification applied to the + * catalog. + */ + public CatalogChange[] catalogChanges() { + return catalogChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java new file mode 100644 index 00000000000..03223c90ff2 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events related to catalog operations. This class extends + * {@link Event} to provide a more specific context involving operations on catalogs, such as + * creation, deletion, or modification. + */ +@DeveloperApi +public abstract class CatalogEvent extends Event { + /** + * Constructs a new {@code CatalogEvent} with the specified user and catalog identifier. + * + * @param user The user responsible for triggering the catalog operation. + * @param identifier The identifier of the catalog involved in the operation. + */ + protected CatalogEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java new file mode 100644 index 00000000000..80fbbffb167 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a catalog operation fails due to an + * exception. This class extends {@link FailureEvent} to provide a more specific context related to + * catalog operations, encapsulating details about the user who initiated the operation, the + * identifier of the catalog involved, and the exception that led to the failure. + */ +@DeveloperApi +public abstract class CatalogFailureEvent extends FailureEvent { + /** + * Constructs a new {@code CatalogFailureEvent} instance, capturing information about the failed + * catalog operation. + * + * @param user The user associated with the failed catalog operation. + * @param identifier The identifier of the catalog that was involved in the failed operation. + * @param exception The exception that was thrown during the catalog operation, indicating the + * cause of the failure. + */ + protected CatalogFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java new file mode 100644 index 00000000000..8bc04f1895e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; + +/** Represents an event that is activated upon the successful creation of a catalog. */ +@DeveloperApi +public class CreateCatalogEvent extends CatalogEvent { + private final CatalogInfo createdCatalogInfo; + + /** + * Constructs an instance of {@code CreateCatalogEvent}, capturing essential details about the + * successful creation of a catalog. + * + * @param user The username of the individual who initiated the catalog creation. + * @param identifier The unique identifier of the catalog that was created. + * @param createdCatalogInfo The final state of the catalog post-creation. + */ + public CreateCatalogEvent( + String user, NameIdentifier identifier, CatalogInfo createdCatalogInfo) { + super(user, identifier); + this.createdCatalogInfo = createdCatalogInfo; + } + + /** + * Provides the final state of the catalog as it is presented to the user following the successful + * creation. + * + * @return A {@link CatalogInfo} object that encapsulates the detailed characteristics of the + * newly created catalog. + */ + public CatalogInfo createdCatalogInfo() { + return createdCatalogInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java new file mode 100644 index 00000000000..2194ce0b9bc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; + +/** + * Represents an event that is generated when an attempt to create a catalog fails due to an + * exception. + */ +@DeveloperApi +public final class CreateCatalogFailureEvent extends CatalogFailureEvent { + private final CatalogInfo createCatalogRequest; + + /** + * Constructs a {@code CreateCatalogFailureEvent} instance, capturing detailed information about + * the failed catalog creation attempt. + * + * @param user The user who initiated the catalog creation operation. + * @param identifier The identifier of the catalog that was attempted to be created. + * @param exception The exception that was thrown during the catalog creation operation, providing + * insight into what went wrong. + * @param createCatalogRequest The original request information used to attempt to create the + * catalog. This includes details such as the intended catalog schema, properties, and other + * configuration options that were specified. + */ + public CreateCatalogFailureEvent( + String user, + NameIdentifier identifier, + Exception exception, + CatalogInfo createCatalogRequest) { + super(user, identifier, exception); + this.createCatalogRequest = createCatalogRequest; + } + + /** + * Retrieves the original request information for the attempted catalog creation. + * + * @return The {@link CatalogInfo} instance representing the request information for the failed + * catalog creation attempt. + */ + public CatalogInfo createCatalogRequest() { + return createCatalogRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java new file mode 100644 index 00000000000..bf2f01effbf --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is generated after a catalog is successfully dropped. */ +@DeveloperApi +public final class DropCatalogEvent extends CatalogEvent { + private final boolean isExists; + + /** + * Constructs a new {@code DropCatalogEvent} instance, encapsulating information about the outcome + * of a catalog drop operation. + * + * @param user The user who initiated the drop catalog operation. + * @param identifier The identifier of the catalog that was attempted to be dropped. + * @param isExists A boolean flag indicating whether the catalog existed at the time of the drop + * operation. + */ + public DropCatalogEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the existence status of the catalog at the time of the drop operation. + * + * @return A boolean value indicating whether the catalog existed. {@code true} if the catalog + * existed, otherwise {@code false}. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java new file mode 100644 index 00000000000..48319ed8e45 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to drop a catalog fails due to an + * exception. + */ +@DeveloperApi +public final class DropCatalogFailureEvent extends CatalogFailureEvent { + /** + * Constructs a new {@code DropCatalogFailureEvent} instance, capturing detailed information about + * the failed attempt to drop a catalog. + * + * @param user The user who initiated the drop catalog operation. + * @param identifier The identifier of the catalog that the operation attempted to drop. + * @param exception The exception that was thrown during the drop catalog operation, offering + * insights into what went wrong and why the operation failed. + */ + public DropCatalogFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java new file mode 100644 index 00000000000..37b4d666f97 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is triggered upon the successful list of catalogs. */ +@DeveloperApi +public final class ListCatalogEvent extends CatalogEvent { + private final Namespace namespace; + + /** + * Constructs an instance of {@code ListCatalogEvent}. + * + * @param user The username of the individual who initiated the catalog listing. + * @param namespace The namespace from which catalogs were listed. + */ + public ListCatalogEvent(String user, Namespace namespace) { + super(user, NameIdentifier.of(namespace.toString())); + this.namespace = namespace; + } + + /** + * Provides the namespace associated with this event. + * + * @return A {@link Namespace} instance from which catalogs were listed. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java new file mode 100644 index 00000000000..30e4b24e3e6 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list catalogs within a namespace fails + * due to an exception. + */ +@DeveloperApi +public final class ListCatalogFailureEvent extends CatalogFailureEvent { + private final Namespace namespace; + + /** + * Constructs a {@code ListCatalogFailureEvent} instance. + * + * @param user The username of the individual who initiated the operation to list catalogs. + * @param namespace The namespace for which the catalog listing was attempted. + * @param exception The exception encountered during the attempt to list catalogs. + */ + public ListCatalogFailureEvent(String user, Exception exception, Namespace namespace) { + super(user, NameIdentifier.of(namespace.toString()), exception); + this.namespace = namespace; + } + + /** + * Provides the namespace associated with this event. + * + * @return A {@link Namespace} instance from which catalogs were listed. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java new file mode 100644 index 00000000000..f07d2cda6d3 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; + +/** Represents an event triggered upon the successful loading of a catalog. */ +@DeveloperApi +public final class LoadCatalogEvent extends CatalogEvent { + private final CatalogInfo loadedCatalogInfo; + + /** + * Constructs an instance of {@code LoadCatalogEvent}. + * + * @param user The username of the individual who initiated the catalog loading. + * @param identifier The unique identifier of the catalog that was loaded. + * @param loadedCatalogInfo The state of the catalog post-loading. + */ + public LoadCatalogEvent(String user, NameIdentifier identifier, CatalogInfo loadedCatalogInfo) { + super(user, identifier); + this.loadedCatalogInfo = loadedCatalogInfo; + } + + /** + * Retrieves the state of the catalog as it was made available to the user after successful + * loading. + * + * @return A {@link CatalogInfo} instance encapsulating the details of the catalog as loaded. + */ + public CatalogInfo loadedCatalogInfo() { + return loadedCatalogInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java new file mode 100644 index 00000000000..3972fb0526c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when an attempt to load a catalog fails due to an exception. */ +@DeveloperApi +public final class LoadCatalogFailureEvent extends CatalogFailureEvent { + /** + * Constructs a {@code LoadCatalogFailureEvent} instance. + * + * @param user The user who initiated the catalog loading operation. + * @param identifier The identifier of the catalog that the loading attempt was made for. + * @param exception The exception that was thrown during the catalog loading operation, offering + * insight into the issues encountered. + */ + public LoadCatalogFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java new file mode 100644 index 00000000000..0e1f64f9272 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** Encapsulates read-only information about a catalog, intended for use in event listeners. */ +@DeveloperApi +public final class CatalogInfo { + private final String name; + private final Catalog.Type type; + private final String provider; + @Nullable private final String comment; + private final Map properties; + @Nullable private final Audit auditInfo; + + /** + * Constructs catalog information from a given catalog instance. + * + * @param catalog The source catalog. + */ + public CatalogInfo(Catalog catalog) { + this( + catalog.name(), + catalog.type(), + catalog.provider(), + catalog.comment(), + catalog.properties(), + catalog.auditInfo()); + } + + /** + * Constructs catalog information with specified details. + * + * @param name The catalog name. + * @param type The catalog type. + * @param provider The catalog provider. + * @param comment An optional comment about the catalog. + * @param properties Catalog properties. + * @param auditInfo Optional audit information. + */ + public CatalogInfo( + String name, + Catalog.Type type, + String provider, + String comment, + Map properties, + Audit auditInfo) { + this.name = name; + this.type = type; + this.provider = provider; + this.comment = comment; + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + this.auditInfo = auditInfo; + } + + /** + * Returns the catalog name. + * + * @return Catalog name. + */ + public String name() { + return name; + } + + /** + * Returns the catalog type. + * + * @return Catalog type. + */ + public Catalog.Type type() { + return type; + } + + /** + * Returns the catalog provider. + * + * @return Catalog provider. + */ + public String provider() { + return provider; + } + + /** + * Returns an optional comment about the catalog. + * + * @return Catalog comment, or null if not provided. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Returns the catalog properties. + * + * @return An immutable map of catalog properties. + */ + public Map properties() { + return properties; + } + + /** + * Returns optional audit information for the catalog. + * + * @return Audit information, or null if not provided. + */ + @Nullable + public Audit auditInfo() { + return auditInfo; + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 9dc90458ba0..bc1182865b2 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -6,7 +6,7 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.GravitinoEnv; -import com.datastrato.gravitino.catalog.CatalogManager; +import com.datastrato.gravitino.catalog.CatalogDispatcher; import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; @@ -77,7 +77,7 @@ private void initializeRestApi() { @Override protected void configure() { bind(gravitinoEnv.metalakesManager()).to(MetalakeManager.class).ranked(1); - bind(gravitinoEnv.catalogManager()).to(CatalogManager.class).ranked(1); + bind(gravitinoEnv.catalogDispatcher()).to(CatalogDispatcher.class).ranked(1); bind(gravitinoEnv.schemaDispatcher()).to(SchemaDispatcher.class).ranked(1); bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java index cefbca7b60c..3abbeb20c5e 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java @@ -8,7 +8,7 @@ import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.catalog.CatalogManager; +import com.datastrato.gravitino.catalog.CatalogDispatcher; import com.datastrato.gravitino.dto.requests.CatalogCreateRequest; import com.datastrato.gravitino.dto.requests.CatalogUpdateRequest; import com.datastrato.gravitino.dto.requests.CatalogUpdatesRequest; @@ -45,13 +45,13 @@ public class CatalogOperations { private static final Logger LOG = LoggerFactory.getLogger(CatalogOperations.class); - private final CatalogManager manager; + private final CatalogDispatcher catalogDispatcher; @Context private HttpServletRequest httpRequest; @Inject - public CatalogOperations(CatalogManager manager) { - this.manager = manager; + public CatalogOperations(CatalogDispatcher catalogDispatcher) { + this.catalogDispatcher = catalogDispatcher; } @GET @@ -70,10 +70,10 @@ public Response listCatalogs( LockType.READ, () -> { if (verbose) { - Catalog[] catalogs = manager.listCatalogsInfo(catalogNS); + Catalog[] catalogs = catalogDispatcher.listCatalogsInfo(catalogNS); return Utils.ok(new CatalogListResponse(DTOConverters.toDTOs(catalogs))); } else { - NameIdentifier[] idents = manager.listCatalogs(catalogNS); + NameIdentifier[] idents = catalogDispatcher.listCatalogs(catalogNS); return Utils.ok(new EntityListResponse(idents)); } }); @@ -98,7 +98,7 @@ public Response createCatalog( NameIdentifier.ofMetalake(metalake), LockType.WRITE, () -> - manager.createCatalog( + catalogDispatcher.createCatalog( ident, request.getType(), request.getProvider(), @@ -121,7 +121,8 @@ public Response loadCatalog( try { NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); Catalog catalog = - TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> manager.loadCatalog(ident)); + TreeLockUtils.doWithTreeLock( + ident, LockType.READ, () -> catalogDispatcher.loadCatalog(ident)); return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); } catch (Exception e) { @@ -151,7 +152,7 @@ public Response alterCatalog( TreeLockUtils.doWithTreeLock( NameIdentifier.ofMetalake(metalakeName), LockType.WRITE, - () -> manager.alterCatalog(ident, changes)); + () -> catalogDispatcher.alterCatalog(ident, changes)); return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); }); @@ -175,7 +176,7 @@ public Response dropCatalog( TreeLockUtils.doWithTreeLock( NameIdentifier.ofMetalake(metalakeName), LockType.WRITE, - () -> manager.dropCatalog(ident)); + () -> catalogDispatcher.dropCatalog(ident)); if (!dropped) { LOG.warn("Failed to drop catalog {} under metalake {}", catalogName, metalakeName); } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestCatalogOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestCatalogOperations.java index 009b824e8a9..2ffb138a452 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestCatalogOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestCatalogOperations.java @@ -17,6 +17,7 @@ import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.CatalogDispatcher; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.requests.CatalogCreateRequest; @@ -91,7 +92,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(manager).to(CatalogManager.class).ranked(2); + bind(manager).to(CatalogDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); From 6b9a47baefd3871b114c077e222e94abbcf8c15a Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 16 Apr 2024 16:10:46 +0800 Subject: [PATCH 037/106] [#1276] improvment(core): Optimize logic about dropping old version of data in KvGcCollector (#2918) ### What changes were proposed in this pull request? Introduce a variable to mark the last transaction ID and perform the GC from the last transaction ID next time to fulfill `incremental GC`. ### Why are the changes needed? Full GC for the old version of the data takes a lot of time, we'd better not use this method. Fix: #1276 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Existing tests and test locally. --- .../storage/kv/KvGarbageCollector.java | 115 ++++++++++++++++-- .../storage/kv/TestKvGarbageCollector.java | 94 ++++++++++++++ rfc/rfc-3/Transaction-implementation-on-kv.md | 1 + 3 files changed, 197 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java index d505a541864..309323f490c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java @@ -21,8 +21,8 @@ import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -44,6 +44,15 @@ public final class KvGarbageCollector implements Closeable { private final KvBackend kvBackend; private final Config config; private final EntityKeyEncoder entityKeyEncoder; + private static final byte[] LAST_COLLECT_COMMIT_ID_KEY = + Bytes.concat( + new byte[] {0x1D, 0x00, 0x03}, "last_collect_commit_id".getBytes(StandardCharsets.UTF_8)); + + // Keep the last collect commit id to avoid collecting the same data multiple times, the first + // time the commit is 1 (minimum), and assuming we have collected the data with transaction id + // (1, 100], then the second time we collect the data and current tx_id is 200, + // then the current transaction id range is (100, 200] and so on. + byte[] commitIdHasBeenCollected; private long frequencyInMinutes; private static final String TIME_STAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; @@ -139,7 +148,18 @@ private void collectAndRemoveOldVersionData() throws IOException { long transactionIdToDelete = deleteTimeLine << 18; LOG.info("Start to remove data which is older than {}", transactionIdToDelete); byte[] startKey = TransactionalKvBackendImpl.generateCommitKey(transactionIdToDelete); - byte[] endKey = endOfTransactionId(); + commitIdHasBeenCollected = kvBackend.get(LAST_COLLECT_COMMIT_ID_KEY); + if (commitIdHasBeenCollected == null) { + commitIdHasBeenCollected = endOfTransactionId(); + } + + long lastGCId = getTransactionId(getBinaryTransactionId(commitIdHasBeenCollected)); + LOG.info( + "Start to collect data which is modified between '{}({})' (exclusive) and '{}({})' (inclusive)", + lastGCId, + lastGCId == 1 ? lastGCId : DateFormatUtils.format(lastGCId >> 18, TIME_STAMP_FORMAT), + transactionIdToDelete, + DateFormatUtils.format(deleteTimeLine, TIME_STAMP_FORMAT)); // Get all commit marks // TODO(yuqi), Use multi-thread to scan the data in case of the data is too large. @@ -147,16 +167,11 @@ private void collectAndRemoveOldVersionData() throws IOException { kvBackend.scan( new KvRange.KvRangeBuilder() .start(startKey) - .end(endKey) + .end(commitIdHasBeenCollected) .startInclusive(true) .endInclusive(false) .build()); - // Why should we reverse the order? Because we need to delete the data from the oldest data to - // the latest ones. kvs is sorted by transaction id in ascending order (Keys with bigger - // transaction id - // is smaller than keys with smaller transaction id). So we need to reverse the order. - Collections.sort(kvs, (o1, o2) -> Bytes.wrap(o2.getKey()).compareTo(o1.getKey())); for (Pair kv : kvs) { List keysInTheTransaction = SerializationUtils.deserialize(kv.getValue()); byte[] transactionId = getBinaryTransactionId(kv.getKey()); @@ -174,15 +189,19 @@ private void collectAndRemoveOldVersionData() throws IOException { // Value has deleted mark, we can remove it. if (null == TransactionalKvBackendImpl.getRealValue(rawValue)) { + // Delete the key of all versions. + removeAllVersionsOfKey(rawKey, key, false); + LogHelper logHelper = decodeKey(key, transactionId); + kvBackend.delete(rawKey); LOG.info( - "Physically delete key that has marked deleted: name identifier: '{}', entity type: '{}', createTime: '{}({})', key: '{}'", + "Physically delete key that has marked deleted: name identifier: '{}', entity type: '{}'," + + " createTime: '{}({})', key: '{}'", logHelper.identifier, logHelper.type, logHelper.createTimeAsString, logHelper.createTimeInMs, Bytes.wrap(key)); - kvBackend.delete(rawKey); keysDeletedCount++; continue; } @@ -200,12 +219,17 @@ private void collectAndRemoveOldVersionData() throws IOException { .limit(1) .build()); if (!newVersionOfKey.isEmpty()) { + // Have a new version, we can safely remove all old versions. + removeAllVersionsOfKey(rawKey, key, false); + // Has a newer version, we can remove it. LogHelper logHelper = decodeKey(key, transactionId); byte[] newVersionKey = newVersionOfKey.get(0).getKey(); LogHelper newVersionLogHelper = decodeKey(newVersionKey); + kvBackend.delete(rawKey); LOG.info( - "Physically delete key that has newer version: name identifier: '{}', entity type: '{}', createTime: '{}({})', newVersion createTime: '{}({})'," + "Physically delete key that has newer version: name identifier: '{}', entity type: '{}'," + + " createTime: '{}({})', newVersion createTime: '{}({})'," + " key: '{}', newVersion key: '{}'", logHelper.identifier, logHelper.type, @@ -215,13 +239,13 @@ private void collectAndRemoveOldVersionData() throws IOException { newVersionLogHelper.createTimeInMs, Bytes.wrap(rawKey), Bytes.wrap(newVersionKey)); - kvBackend.delete(rawKey); keysDeletedCount++; } } // All keys in this transaction have been deleted, we can remove the commit mark. if (keysDeletedCount == keysInTheTransaction.size()) { + kvBackend.delete(kv.getKey()); long timestamp = getTransactionId(transactionId) >> 18; LOG.info( "Physically delete commit mark: {}, createTime: '{}({})', key: '{}'", @@ -229,7 +253,72 @@ private void collectAndRemoveOldVersionData() throws IOException { DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT), timestamp, Bytes.wrap(kv.getKey())); - kvBackend.delete(kv.getKey()); + } + } + + commitIdHasBeenCollected = kvs.isEmpty() ? startKey : kvs.get(0).getKey(); + kvBackend.put(LAST_COLLECT_COMMIT_ID_KEY, commitIdHasBeenCollected, true); + } + + /** + * Remove all versions of the key. + * + * @param rawKey raw key, it contains the transaction id. + * @param key key, it's the real key and does not contain the transaction id + * @param includeStart whether include the start key. + * @throws IOException if an I/O exception occurs during deletion. + */ + private void removeAllVersionsOfKey(byte[] rawKey, byte[] key, boolean includeStart) + throws IOException { + List> kvs = + kvBackend.scan( + new KvRange.KvRangeBuilder() + .start(rawKey) + .end(generateKey(key, 1)) + .startInclusive(includeStart) + .endInclusive(false) + .build()); + + for (Pair kv : kvs) { + // Delete real data. + kvBackend.delete(kv.getKey()); + + LogHelper logHelper = decodeKey(kv.getKey()); + LOG.info( + "Physically delete key that has marked deleted: name identifier: '{}', entity type: '{}'," + + " createTime: '{}({})', key: '{}'", + logHelper.identifier, + logHelper.type, + logHelper.createTimeAsString, + logHelper.createTimeInMs, + Bytes.wrap(key)); + + // Try to delete commit id if the all keys in the transaction id have been dropped. + byte[] transactionId = getBinaryTransactionId(kv.getKey()); + byte[] transactionKey = generateCommitKey(transactionId); + byte[] transactionValue = kvBackend.get(transactionKey); + + List keysInTheTransaction = SerializationUtils.deserialize(transactionValue); + + boolean allDropped = true; + for (byte[] keyInTheTransaction : keysInTheTransaction) { + if (kvBackend.get(generateKey(keyInTheTransaction, transactionId)) != null) { + // There is still a key in the transaction, we cannot delete the commit mark. + allDropped = false; + break; + } + } + + // Try to delete the commit mark. + if (allDropped) { + kvBackend.delete(transactionKey); + long timestamp = TransactionalKvBackendImpl.getTransactionId(transactionId) >> 18; + LOG.info( + "Physically delete commit mark: {}, createTime: '{}({})', key: '{}'", + Bytes.wrap(kv.getKey()), + DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT), + timestamp, + Bytes.wrap(kv.getKey())); } } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java index fea6ac7bb41..2ac7468906a 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java @@ -16,6 +16,8 @@ import static com.datastrato.gravitino.storage.kv.TestKvEntityStorage.createFilesetEntity; import static com.datastrato.gravitino.storage.kv.TestKvEntityStorage.createSchemaEntity; import static com.datastrato.gravitino.storage.kv.TestKvEntityStorage.createTableEntity; +import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.getBinaryTransactionId; +import static com.datastrato.gravitino.storage.kv.TransactionalKvBackendImpl.getTransactionId; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; @@ -409,4 +411,96 @@ void testRemoveWithGCCollector2() throws IOException, InterruptedException { TableEntity.class)); } } + + @Test + void testIncrementalGC() throws Exception { + Config config = getConfig(); + Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); + + try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { + store.initialize(config); + + if (!(store instanceof KvEntityStore)) { + return; + } + KvEntityStore kvEntityStore = (KvEntityStore) store; + + store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + BaseMetalake metalake1 = createBaseMakeLake(1L, "metalake1", auditInfo); + BaseMetalake metalake2 = createBaseMakeLake(2L, "metalake2", auditInfo); + BaseMetalake metalake3 = createBaseMakeLake(3L, "metalake3", auditInfo); + + for (int i = 0; i < 10; i++) { + store.put(metalake1); + store.put(metalake2); + store.put(metalake3); + + store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); + + Thread.sleep(10); + } + + store.put(metalake1); + store.put(metalake2); + store.put(metalake3); + + Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); + Thread.sleep(1500); + + // Scan raw key-value data from storage to confirm the data is deleted + kvEntityStore.kvGarbageCollector.collectAndClean(); + List> allData = + kvEntityStore.backend.scan( + new KvRange.KvRangeBuilder() + .start("_".getBytes()) + .end("z".getBytes()) + .startInclusive(false) + .endInclusive(false) + .build()); + + Assertions.assertEquals(3, allData.size()); + + long transactionId = + getTransactionId( + getBinaryTransactionId(kvEntityStore.kvGarbageCollector.commitIdHasBeenCollected)); + Assertions.assertNotEquals(1, transactionId); + + for (int i = 0; i < 10; i++) { + store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); + store.put(metalake1); + store.put(metalake2); + store.put(metalake3); + Thread.sleep(10); + } + store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); + store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); + + Thread.sleep(1500); + kvEntityStore.kvGarbageCollector.collectAndClean(); + + allData = + kvEntityStore.backend.scan( + new KvRange.KvRangeBuilder() + .start("_".getBytes()) + .end("z".getBytes()) + .startInclusive(false) + .endInclusive(false) + .build()); + + Assertions.assertTrue(allData.isEmpty()); + + long transactionIdV2 = + getTransactionId( + getBinaryTransactionId(kvEntityStore.kvGarbageCollector.commitIdHasBeenCollected)); + Assertions.assertTrue(transactionIdV2 > transactionId); + } + } } diff --git a/rfc/rfc-3/Transaction-implementation-on-kv.md b/rfc/rfc-3/Transaction-implementation-on-kv.md index 33f905b619c..fb2fdd93db3 100644 --- a/rfc/rfc-3/Transaction-implementation-on-kv.md +++ b/rfc/rfc-3/Transaction-implementation-on-kv.md @@ -123,5 +123,6 @@ Scan and range query are almost the same as that of read process, for more detai - Keys that start with 0x'1D0000' store the contents of id-name mapping. for more please refer to class `KvNameMappingService`. - Keys that start with 0x'1D0001' store the data of current timestamp which is used for generating transaction id, for more please refer to class `TransactionIdGeneratorImpl`. - Keys that start with 0x'1D0002' store the information of storage layout version. For more please refer to `KvEntityStore#initStorageVersionInfo` +- Keys that start with 0x'1D0003' store tha transaction id that was used by `KvGarbageCollector` last time. - Keys that start with 0x'1E' store transaction marks which mark the transaction is committed or not. - Other key spaces are used to store gravitino entities like `metalakes`,`catalogs`, `scheams`, `tables` and so on. it usually starts with from 0x'20'(space) to 0x'7F'(delete). For more please refer to class `KvEntityStoreImpl`. \ No newline at end of file From c122edb98363856194691c27c1ade826bc946ee7 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 16 Apr 2024 16:25:30 +0800 Subject: [PATCH 038/106] [#2634] fix(jdbc-mysql): Fix the backquote issues in drop schema (#2961) ### What changes were proposed in this pull request? Backquote the schema(database) name when dropping it. ### Why are the changes needed? We must backquote the name if the name contains special character like `%`, '-', '/' and so on. Fix: #2634 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Add new IT `testDropTableWithSpecificName`. --- .../operation/MysqlDatabaseOperations.java | 2 +- .../test/TestMysqlDatabaseOperations.java | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java index 9971b9ee490..2c574edbf99 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java @@ -67,7 +67,7 @@ public String generateDropDatabaseSql(String databaseName, boolean cascade) { } try (final Connection connection = this.dataSource.getConnection()) { - String query = "SHOW TABLES IN " + databaseName; + String query = "SHOW TABLES IN `" + databaseName + "`"; try (Statement statement = connection.createStatement()) { // Execute the query and check if there exists any tables in the database try (ResultSet resultSet = statement.executeQuery(query)) { diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlDatabaseOperations.java index 85a2e4105cb..5f24ea7bcfc 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlDatabaseOperations.java @@ -6,10 +6,17 @@ import static com.datastrato.gravitino.catalog.mysql.operation.MysqlDatabaseOperations.SYS_MYSQL_DATABASE_NAMES; +import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.utils.RandomNameUtils; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -29,4 +36,37 @@ public void testBaseOperationDatabase() { testBaseOperation(databaseName, properties, comment); testDropDatabase(databaseName); } + + @Test + void testDropTableWithSpecificName() { + String databaseName = RandomNameUtils.genRandomName("ct_db") + "-abc-" + "end"; + Map properties = new HashMap<>(); + DATABASE_OPERATIONS.create(databaseName, null, properties); + DATABASE_OPERATIONS.delete(databaseName, false); + + databaseName = RandomNameUtils.genRandomName("ct_db") + "--------end"; + DATABASE_OPERATIONS.create(databaseName, null, properties); + + String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; + String tableComment = "test_comment"; + List columns = new ArrayList<>(); + columns.add( + JdbcColumn.builder() + .withName("col_1") + .withType(Types.VarCharType.of(100)) + .withComment("test_comment") + .withNullable(true) + .build()); + + TABLE_OPERATIONS.create( + databaseName, + tableName, + columns.toArray(new JdbcColumn[0]), + tableComment, + properties, + new Transform[0], + Distributions.NONE, + new Index[0]); + DATABASE_OPERATIONS.delete(databaseName, true); + } } From 6eccc74c91f96171e45956b30f4c0f2ad915a2f1 Mon Sep 17 00:00:00 2001 From: Kang Date: Tue, 16 Apr 2024 16:42:13 +0800 Subject: [PATCH 039/106] [#2572] feat(catalog-jdbc-doris): support table operation for Doris catalog (#2875) ### What changes were proposed in this pull request? support doris table operation ### Why are the changes needed? Fix: #2572 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? UT --- .../jdbc/operation/JdbcTableOperations.java | 12 +- .../DorisColumnDefaultValueConverter.java | 78 ++- .../converter/DorisExceptionConverter.java | 10 + .../doris/converter/DorisTypeConverter.java | 93 ++- .../doris/operation/DorisTableOperations.java | 548 +++++++++++++++++- .../TestDorisExceptionConverter.java | 6 + .../test/DorisTableOperationsIT.java | 406 +++++++++++++ .../mysql/operation/MysqlTableOperations.java | 3 +- .../test/container/DorisContainer.java | 6 +- 9 files changed, 1142 insertions(+), 20 deletions(-) create mode 100644 catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java index 71c68f947dd..e42a5f0d368 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java @@ -167,7 +167,7 @@ public JdbcTable load(String databaseName, String tableName) throws NoSuchTableE jdbcTableBuilder.withColumns(jdbcColumns.toArray(new JdbcColumn[0])); // 3.Get index information - List indexes = getIndexes(databaseName, tableName, connection.getMetaData()); + List indexes = getIndexes(connection, databaseName, tableName); jdbcTableBuilder.withIndexes(indexes.toArray(new Index[0])); // 4.Get table properties @@ -175,7 +175,7 @@ public JdbcTable load(String databaseName, String tableName) throws NoSuchTableE jdbcTableBuilder.withProperties(tableProperties); // 5.Leave the information to the bottom layer to append the table - correctJdbcTableFields(connection, tableName, jdbcTableBuilder); + correctJdbcTableFields(connection, databaseName, tableName, jdbcTableBuilder); return jdbcTableBuilder.build(); } catch (SQLException e) { throw exceptionMapper.toGravitinoException(e); @@ -275,13 +275,17 @@ protected ResultSet getColumns(Connection connection, String databaseName, Strin * @throws SQLException */ protected void correctJdbcTableFields( - Connection connection, String tableName, JdbcTable.Builder jdbcTableBuilder) + Connection connection, + String databaseName, + String tableName, + JdbcTable.Builder jdbcTableBuilder) throws SQLException { // nothing to do } - protected List getIndexes(String databaseName, String tableName, DatabaseMetaData metaData) + protected List getIndexes(Connection connection, String databaseName, String tableName) throws SQLException { + DatabaseMetaData metaData = connection.getMetaData(); List indexes = new ArrayList<>(); // Get primary key information diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java index 429270537ce..b9b1b0f70b4 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java @@ -4,20 +4,94 @@ */ package com.datastrato.gravitino.catalog.doris.converter; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.BIGINT; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.CHAR; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.DATETIME; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.DECIMAL; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.DOUBLE; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.FLOAT; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.INT; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.SMALLINT; +import static com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter.TINYINT; import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; +import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_OF_CURRENT_TIMESTAMP; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; import com.datastrato.gravitino.rel.expressions.Expression; +import com.datastrato.gravitino.rel.expressions.UnparsedExpression; +import com.datastrato.gravitino.rel.expressions.literals.Literals; +import com.datastrato.gravitino.rel.types.Decimal; +import com.datastrato.gravitino.rel.types.Types; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; public class DorisColumnDefaultValueConverter extends JdbcColumnDefaultValueConverter { + @Override public Expression toGravitino( JdbcTypeConverter.JdbcTypeBean columnType, String columnDefaultValue, boolean isExpression, boolean nullable) { - // TODO: add implementation for doris catalog - return DEFAULT_VALUE_NOT_SET; + if (columnDefaultValue == null) { + return nullable ? Literals.NULL : DEFAULT_VALUE_NOT_SET; + } + + if (columnDefaultValue.equalsIgnoreCase(NULL)) { + return Literals.NULL; + } + + if (isExpression) { + if (columnDefaultValue.equals(CURRENT_TIMESTAMP)) { + return DEFAULT_VALUE_OF_CURRENT_TIMESTAMP; + } + // The parsing of Doris expressions is complex, so we are not currently undertaking the + // parsing. + return UnparsedExpression.of(columnDefaultValue); + } + + switch (columnType.getTypeName().toLowerCase()) { + case TINYINT: + return Literals.byteLiteral(Byte.valueOf(columnDefaultValue)); + case SMALLINT: + return Literals.shortLiteral(Short.valueOf(columnDefaultValue)); + case INT: + return Literals.integerLiteral(Integer.valueOf(columnDefaultValue)); + case BIGINT: + return Literals.longLiteral(Long.valueOf(columnDefaultValue)); + case FLOAT: + return Literals.floatLiteral(Float.valueOf(columnDefaultValue)); + case DOUBLE: + return Literals.doubleLiteral(Double.valueOf(columnDefaultValue)); + case DECIMAL: + return Literals.decimalLiteral( + Decimal.of( + columnDefaultValue, + Integer.parseInt(columnType.getColumnSize()), + Integer.parseInt(columnType.getScale()))); + case JdbcTypeConverter.DATE: + return Literals.dateLiteral(LocalDate.parse(columnDefaultValue, DATE_TIME_FORMATTER)); + case JdbcTypeConverter.TIME: + return Literals.timeLiteral(LocalTime.parse(columnDefaultValue, DATE_TIME_FORMATTER)); + case JdbcTypeConverter.TIMESTAMP: + case DATETIME: + return CURRENT_TIMESTAMP.equals(columnDefaultValue) + ? DEFAULT_VALUE_OF_CURRENT_TIMESTAMP + : Literals.timestampLiteral( + LocalDateTime.parse(columnDefaultValue, DATE_TIME_FORMATTER)); + case JdbcTypeConverter.VARCHAR: + return Literals.of( + columnDefaultValue, Types.VarCharType.of(Integer.parseInt(columnType.getColumnSize()))); + case CHAR: + return Literals.of( + columnDefaultValue, + Types.FixedCharType.of(Integer.parseInt(columnType.getColumnSize()))); + case JdbcTypeConverter.TEXT: + return Literals.stringLiteral(columnDefaultValue); + default: + throw new IllegalArgumentException("Unknown data columnType for literal: " + columnType); + } } } diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java index 59bbd93bcd7..60bcf61b29a 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java @@ -35,6 +35,12 @@ public class DorisExceptionConverter extends JdbcExceptionConverter { private static final Pattern DATABASE_ALREADY_EXISTS_PATTERN = Pattern.compile(DATABASE_ALREADY_EXISTS_PATTERN_STRING); + private static final String TABLE_NOT_EXIST_PATTERN_STRING = + ".*detailMessage = Unknown table '.*' in .*:.*"; + + private static final Pattern TABLE_NOT_EXIST_PATTERN = + Pattern.compile(TABLE_NOT_EXIST_PATTERN_STRING); + @SuppressWarnings("FormatStringAnnotation") @Override public GravitinoRuntimeException toGravitinoException(SQLException se) { @@ -70,6 +76,10 @@ static int getErrorCodeFromMessage(String message) { return CODE_DATABASE_EXISTS; } + if (TABLE_NOT_EXIST_PATTERN.matcher(message).matches()) { + return CODE_NO_SUCH_TABLE; + } + return CODE_OTHER; } } diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java index 08b75e20b92..aecbf3dc5c4 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java @@ -10,15 +10,102 @@ /** Type converter for Doris. */ public class DorisTypeConverter extends JdbcTypeConverter { - // TODO: add implementation for doris catalog + static final String BOOLEAN = "boolean"; + static final String TINYINT = "tinyint"; + static final String SMALLINT = "smallint"; + static final String INT = "int"; + static final String BIGINT = "bigint"; + static final String FLOAT = "float"; + static final String DOUBLE = "double"; + static final String DECIMAL = "decimal"; + static final String DATETIME = "datetime"; + static final String CHAR = "char"; + static final String STRING = "string"; @Override public Type toGravitinoType(JdbcTypeBean typeBean) { - return Types.UnparsedType.of(typeBean.getTypeName()); + switch (typeBean.getTypeName().toLowerCase()) { + case BOOLEAN: + return Types.BooleanType.get(); + case TINYINT: + return Types.ByteType.get(); + case SMALLINT: + return Types.ShortType.get(); + case INT: + return Types.IntegerType.get(); + case BIGINT: + return Types.LongType.get(); + case FLOAT: + return Types.FloatType.get(); + case DOUBLE: + return Types.DoubleType.get(); + case DECIMAL: + return Types.DecimalType.of( + Integer.parseInt(typeBean.getColumnSize()), Integer.parseInt(typeBean.getScale())); + case DATE: + return Types.DateType.get(); + case DATETIME: + return Types.TimestampType.withTimeZone(); + case CHAR: + return Types.FixedCharType.of(Integer.parseInt(typeBean.getColumnSize())); + case VARCHAR: + return Types.VarCharType.of(Integer.parseInt(typeBean.getColumnSize())); + case STRING: + case TEXT: + return Types.StringType.get(); + default: + throw new IllegalArgumentException("Not a supported type: " + typeBean); + } } @Override public String fromGravitinoType(Type type) { - throw new IllegalArgumentException("unsupported type: " + type.simpleString()); + if (type instanceof Types.BooleanType) { + return BOOLEAN; + } else if (type instanceof Types.ByteType) { + return TINYINT; + } else if (type instanceof Types.ShortType) { + return SMALLINT; + } else if (type instanceof Types.IntegerType) { + return INT; + } else if (type instanceof Types.LongType) { + return BIGINT; + } else if (type instanceof Types.FloatType) { + return FLOAT; + } else if (type instanceof Types.DoubleType) { + return DOUBLE; + } else if (type instanceof Types.DecimalType) { + return DECIMAL + + "(" + + ((Types.DecimalType) type).precision() + + "," + + ((Types.DecimalType) type).scale() + + ")"; + } else if (type instanceof Types.DateType) { + return DATE; + } else if (type instanceof Types.TimestampType) { + return DATETIME; + } else if (type instanceof Types.VarCharType) { + int length = ((Types.VarCharType) type).length(); + if (length < 1 || length > 65533) { + throw new IllegalArgumentException( + String.format( + "Type %s is invalid, length should be between 1 and 65533", type.simpleString())); + } + return VARCHAR + "(" + ((Types.VarCharType) type).length() + ")"; + } else if (type instanceof Types.FixedCharType) { + int length = ((Types.FixedCharType) type).length(); + if (length < 1 || length > 255) { + throw new IllegalArgumentException( + String.format( + "Type %s is invalid, length should be between 1 and 255", type.simpleString())); + } + + return CHAR + "(" + ((Types.FixedCharType) type).length() + ")"; + } else if (type instanceof Types.StringType) { + return STRING; + } + throw new IllegalArgumentException( + String.format("Couldn't convert Gravitino type %s to Doris type", type.simpleString())); } } diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 396dbf7bf1b..f302a77dfef 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -4,18 +4,66 @@ */ package com.datastrato.gravitino.catalog.doris.operation; +import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; + +import com.datastrato.gravitino.StringIdentifier; +import com.datastrato.gravitino.catalog.doris.utils.DorisUtils; import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import com.datastrato.gravitino.exceptions.NoSuchColumnException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Strategy; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; /** Table operations for Doris. */ public class DorisTableOperations extends JdbcTableOperations { - // TODO: add implementation for doris catalog + private static final String BACK_QUOTE = "`"; + private static final String DORIS_AUTO_INCREMENT = "AUTO_INCREMENT"; + + private static final String NEW_LINE = "\n"; + + @Override + public List listTables(String databaseName) throws NoSuchSchemaException { + final List names = Lists.newArrayList(); + + try (Connection connection = getConnection(databaseName); + ResultSet tables = getTables(connection)) { + while (tables.next()) { + if (Objects.equals(tables.getString("TABLE_CAT"), databaseName)) { + names.add(tables.getString("TABLE_NAME")); + } + } + LOG.info("Finished listing tables size {} for database name {} ", names.size(), databaseName); + return names; + } catch (final SQLException se) { + throw this.exceptionMapper.toGravitinoException(se); + } + } @Override protected String generateCreateTableSql( @@ -25,34 +73,516 @@ protected String generateCreateTableSql( Map properties, Transform[] partitioning, Distribution distribution, - Index[] indexes) { - throw new UnsupportedOperationException(); + com.datastrato.gravitino.rel.indexes.Index[] indexes) { + + validateIncrementCol(columns); + validateDistribution(distribution, columns); + + StringBuilder sqlBuilder = new StringBuilder(); + + sqlBuilder.append(String.format("CREATE TABLE `%s` (", tableName)).append(NEW_LINE); + + // Add columns + sqlBuilder.append( + Arrays.stream(columns) + .map( + column -> { + StringBuilder columnsSql = new StringBuilder(); + columnsSql + .append(SPACE) + .append(BACK_QUOTE) + .append(column.name()) + .append(BACK_QUOTE); + appendColumnDefinition(column, columnsSql); + return columnsSql.toString(); + }) + .collect(Collectors.joining(",\n"))); + + appendIndexesSql(indexes, sqlBuilder); + + sqlBuilder.append(NEW_LINE).append(")"); + + // Add table comment if specified + if (StringUtils.isNotEmpty(comment)) { + sqlBuilder.append(" COMMENT \"").append(comment).append("\""); + } + + // Add distribution info + if (distribution.strategy() == Strategy.HASH) { + sqlBuilder.append(NEW_LINE).append(" DISTRIBUTED BY HASH("); + sqlBuilder.append( + Arrays.stream(distribution.expressions()) + .map(column -> BACK_QUOTE + column.toString() + BACK_QUOTE) + .collect(Collectors.joining(", "))); + sqlBuilder.append(")"); + } else if (distribution.strategy() == Strategy.EVEN) { + sqlBuilder.append(NEW_LINE).append(" DISTRIBUTED BY ").append("RANDOM"); + } + + if (distribution.number() != 0) { + sqlBuilder.append(" BUCKETS ").append(distribution.number()); + } + + // Add table properties + sqlBuilder.append(NEW_LINE).append(DorisUtils.generatePropertiesSql(properties)); + + // Add Partition Info + if (partitioning != null && partitioning.length > 0) { + // TODO: Add partitioning support + throw new UnsupportedOperationException("Currently we do not support Partitioning in Doris"); + } + + // Return the generated SQL statement + String result = sqlBuilder.toString(); + + LOG.info("Generated create table:{} sql: {}", tableName, result); + return result; + } + + private static void validateIncrementCol(JdbcColumn[] columns) { + // Get all auto increment column + List autoIncrementCols = + Arrays.stream(columns).filter(Column::autoIncrement).collect(Collectors.toList()); + + // Doris does not support auto increment column before version 2.1.0 + Preconditions.checkArgument( + autoIncrementCols.isEmpty(), "Doris does not support auto-increment column"); + } + + private static void validateDistribution(Distribution distribution, JdbcColumn[] columns) { + Preconditions.checkArgument(null != distribution, "Doris must set distribution"); + + Preconditions.checkArgument( + Strategy.HASH == distribution.strategy() || Strategy.EVEN == distribution.strategy(), + "Doris only supports HASH or EVEN distribution strategy"); + + if (distribution.strategy() == Strategy.HASH) { + // Check if the distribution column exists + Arrays.stream(distribution.expressions()) + .forEach( + expression -> { + Preconditions.checkArgument( + Arrays.stream(columns) + .anyMatch(column -> column.name().equalsIgnoreCase(expression.toString())), + "Distribution column " + expression + " does not exist in the table columns"); + }); + } + } + + @VisibleForTesting + static void appendIndexesSql( + com.datastrato.gravitino.rel.indexes.Index[] indexes, StringBuilder sqlBuilder) { + + if (indexes.length == 0) { + return; + } + + // validate indexes + Arrays.stream(indexes) + .forEach( + index -> { + if (index.fieldNames().length > 1) { + throw new IllegalArgumentException("Index does not support multi fields in Doris"); + } + }); + + String indexSql = + Arrays.stream(indexes) + .map(index -> String.format("INDEX %s (%s)", index.name(), index.fieldNames()[0][0])) + .collect(Collectors.joining(",\n")); + + sqlBuilder.append(",").append(NEW_LINE).append(indexSql); + } + + @Override + protected boolean getAutoIncrementInfo(ResultSet resultSet) throws SQLException { + return "YES".equalsIgnoreCase(resultSet.getString("IS_AUTOINCREMENT")); + } + + @Override + protected Map getTableProperties(Connection connection, String tableName) + throws SQLException { + + String showCreateTableSQL = String.format("SHOW CREATE TABLE `%s`", tableName); + + StringBuilder createTableSqlSb = new StringBuilder(); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(showCreateTableSQL)) { + while (resultSet.next()) { + createTableSqlSb.append(resultSet.getString("Create Table")); + } + } + + String createTableSql = createTableSqlSb.toString(); + + if (StringUtils.isEmpty(createTableSql)) { + throw new NoSuchTableException( + "Table %s does not exist in %s.", tableName, connection.getCatalog()); + } + + return Collections.unmodifiableMap(DorisUtils.extractPropertiesFromSql(createTableSql)); + } + + @Override + protected List getIndexes(Connection connection, String databaseName, String tableName) + throws SQLException { + String sql = String.format("SHOW INDEX FROM `%s` FROM `%s`", tableName, databaseName); + + // get Indexes from SQL + try (PreparedStatement preparedStatement = connection.prepareStatement(sql); + ResultSet resultSet = preparedStatement.executeQuery()) { + + List indexes = new ArrayList<>(); + while (resultSet.next()) { + String indexName = resultSet.getString("Key_name"); + String columnName = resultSet.getString("Column_name"); + indexes.add( + Indexes.of(Index.IndexType.PRIMARY_KEY, indexName, new String[][] {{columnName}})); + } + return indexes; + } catch (SQLException e) { + throw exceptionMapper.toGravitinoException(e); + } + } + + @Override + protected void correctJdbcTableFields( + Connection connection, String databaseName, String tableName, JdbcTable.Builder tableBuilder) + throws SQLException { + + if (StringUtils.isNotEmpty(tableBuilder.comment())) { + return; + } + + // Doris Cannot get comment from JDBC 8.x, so we need to get comment from sql + String comment = ""; + String sql = + "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"; + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + preparedStatement.setString(1, databaseName); + preparedStatement.setString(2, tableName); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + comment += resultSet.getString("TABLE_COMMENT"); + } + } + tableBuilder.withComment(comment); + } catch (SQLException e) { + throw exceptionMapper.toGravitinoException(e); + } } @Override protected String generateRenameTableSql(String oldTableName, String newTableName) { - throw new UnsupportedOperationException(); + return String.format("ALTER TABLE `%s` RENAME `%s`", oldTableName, newTableName); } @Override protected String generateDropTableSql(String tableName) { - throw new UnsupportedOperationException(); + return String.format("DROP TABLE `%s`", tableName); } @Override protected String generatePurgeTableSql(String tableName) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "Doris does not support purge table in Gravitino, please use drop table"); } @Override protected String generateAlterTableSql( String databaseName, String tableName, TableChange... changes) { - throw new UnsupportedOperationException(); + // Not all operations require the original table information, so lazy loading is used here + JdbcTable lazyLoadTable = null; + TableChange.UpdateComment updateComment = null; + List setProperties = new ArrayList<>(); + List alterSql = new ArrayList<>(); + for (int i = 0; i < changes.length; i++) { + TableChange change = changes[i]; + if (change instanceof TableChange.UpdateComment) { + updateComment = (TableChange.UpdateComment) change; + } else if (change instanceof TableChange.SetProperty) { + // The set attribute needs to be added at the end. + setProperties.add(((TableChange.SetProperty) change)); + } else if (change instanceof TableChange.RemoveProperty) { + // Doris only support set properties, remove property is not supported yet + throw new IllegalArgumentException("Remove property is not supported yet"); + } else if (change instanceof TableChange.AddColumn) { + TableChange.AddColumn addColumn = (TableChange.AddColumn) change; + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + alterSql.add(addColumnFieldDefinition(addColumn)); + } else if (change instanceof TableChange.RenameColumn) { + throw new IllegalArgumentException("Rename column is not supported yet"); + } else if (change instanceof TableChange.UpdateColumnType) { + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + TableChange.UpdateColumnType updateColumnType = (TableChange.UpdateColumnType) change; + alterSql.add(updateColumnTypeFieldDefinition(updateColumnType, lazyLoadTable)); + } else if (change instanceof TableChange.UpdateColumnComment) { + TableChange.UpdateColumnComment updateColumnComment = + (TableChange.UpdateColumnComment) change; + alterSql.add(updateColumnCommentFieldDefinition(updateColumnComment)); + } else if (change instanceof TableChange.UpdateColumnPosition) { + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + TableChange.UpdateColumnPosition updateColumnPosition = + (TableChange.UpdateColumnPosition) change; + alterSql.add(updateColumnPositionFieldDefinition(updateColumnPosition, lazyLoadTable)); + } else if (change instanceof TableChange.DeleteColumn) { + TableChange.DeleteColumn deleteColumn = (TableChange.DeleteColumn) change; + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + String deleteColSql = deleteColumnFieldDefinition(deleteColumn, lazyLoadTable); + if (StringUtils.isNotEmpty(deleteColSql)) { + alterSql.add(deleteColSql); + } + } else if (change instanceof TableChange.UpdateColumnNullability) { + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + alterSql.add( + updateColumnNullabilityDefinition( + (TableChange.UpdateColumnNullability) change, lazyLoadTable)); + } else if (change instanceof TableChange.AddIndex) { + alterSql.add(addIndexDefinition((TableChange.AddIndex) change)); + } else if (change instanceof TableChange.DeleteIndex) { + lazyLoadTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + alterSql.add(deleteIndexDefinition(lazyLoadTable, (TableChange.DeleteIndex) change)); + } else { + throw new IllegalArgumentException( + "Unsupported table change type: " + change.getClass().getName()); + } + } + if (!setProperties.isEmpty()) { + alterSql.add(generateTableProperties(setProperties)); + } + + // Last modified comment + if (null != updateComment) { + String newComment = updateComment.getNewComment(); + if (null == StringIdentifier.fromComment(newComment)) { + // Detect and add gravitino id. + JdbcTable jdbcTable = getOrCreateTable(databaseName, tableName, lazyLoadTable); + StringIdentifier identifier = StringIdentifier.fromComment(jdbcTable.comment()); + if (null != identifier) { + newComment = StringIdentifier.addToComment(identifier, newComment); + } + } + alterSql.add("MODIFY COMMENT \"" + newComment + "\""); + } + + if (!setProperties.isEmpty()) { + alterSql.add(generateTableProperties(setProperties)); + } + + if (CollectionUtils.isEmpty(alterSql)) { + return ""; + } + // Return the generated SQL statement + String result = "ALTER TABLE `" + tableName + "`\n" + String.join(",\n", alterSql) + ";"; + LOG.info("Generated alter table:{} sql: {}", databaseName + "." + tableName, result); + return result; + } + + private String updateColumnNullabilityDefinition( + TableChange.UpdateColumnNullability change, JdbcTable table) { + validateUpdateColumnNullable(change, table); + String col = change.fieldName()[0]; + JdbcColumn column = getJdbcColumnFromTable(table, col); + JdbcColumn updateColumn = + JdbcColumn.builder() + .withName(col) + .withDefaultValue(column.defaultValue()) + .withNullable(change.nullable()) + .withType(column.dataType()) + .withComment(column.comment()) + .withAutoIncrement(column.autoIncrement()) + .build(); + return "MODIFY COLUMN " + + BACK_QUOTE + + col + + BACK_QUOTE + + appendColumnDefinition(updateColumn, new StringBuilder()); + } + + private String generateTableProperties(List setProperties) { + return setProperties.stream() + .map( + setProperty -> + String.format("\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue())) + .collect(Collectors.joining(",\n")); } - @Override protected JdbcTable getOrCreateTable( String databaseName, String tableName, JdbcTable lazyLoadCreateTable) { - throw new UnsupportedOperationException(); + return null != lazyLoadCreateTable ? lazyLoadCreateTable : load(databaseName, tableName); + } + + private String updateColumnCommentFieldDefinition( + TableChange.UpdateColumnComment updateColumnComment) { + String newComment = updateColumnComment.getNewComment(); + if (updateColumnComment.fieldName().length > 1) { + throw new UnsupportedOperationException("Doris does not support nested column names."); + } + String col = updateColumnComment.fieldName()[0]; + + return String.format("MODIFY COLUMN `%s` COMMENT '%s'", col, newComment); + } + + private String addColumnFieldDefinition(TableChange.AddColumn addColumn) { + String dataType = (String) typeConverter.fromGravitinoType(addColumn.getDataType()); + if (addColumn.fieldName().length > 1) { + throw new UnsupportedOperationException("Doris does not support nested column names."); + } + String col = addColumn.fieldName()[0]; + + StringBuilder columnDefinition = new StringBuilder(); + columnDefinition + .append("ADD COLUMN ") + .append(BACK_QUOTE) + .append(col) + .append(BACK_QUOTE) + .append(SPACE) + .append(dataType) + .append(SPACE); + + if (!addColumn.isNullable()) { + columnDefinition.append("NOT NULL "); + } + // Append comment if available + if (StringUtils.isNotEmpty(addColumn.getComment())) { + columnDefinition.append("COMMENT '").append(addColumn.getComment()).append("' "); + } + + // Append position if available + if (addColumn.getPosition() instanceof TableChange.First) { + columnDefinition.append("FIRST"); + } else if (addColumn.getPosition() instanceof TableChange.After) { + TableChange.After afterPosition = (TableChange.After) addColumn.getPosition(); + columnDefinition + .append("AFTER ") + .append(BACK_QUOTE) + .append(afterPosition.getColumn()) + .append(BACK_QUOTE); + } else if (addColumn.getPosition() instanceof TableChange.Default) { + // do nothing, follow the default behavior of doris + } else { + throw new IllegalArgumentException("Invalid column position."); + } + return columnDefinition.toString(); + } + + private String updateColumnPositionFieldDefinition( + TableChange.UpdateColumnPosition updateColumnPosition, JdbcTable jdbcTable) { + if (updateColumnPosition.fieldName().length > 1) { + throw new UnsupportedOperationException("Doris does not support nested column names."); + } + String col = updateColumnPosition.fieldName()[0]; + JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col); + StringBuilder columnDefinition = new StringBuilder(); + columnDefinition.append("MODIFY COLUMN ").append(BACK_QUOTE).append(col).append(BACK_QUOTE); + appendColumnDefinition(column, columnDefinition); + if (updateColumnPosition.getPosition() instanceof TableChange.First) { + columnDefinition.append("FIRST"); + } else if (updateColumnPosition.getPosition() instanceof TableChange.After) { + TableChange.After afterPosition = (TableChange.After) updateColumnPosition.getPosition(); + columnDefinition + .append("AFTER ") + .append(BACK_QUOTE) + .append(afterPosition.getColumn()) + .append(BACK_QUOTE); + } else { + Arrays.stream(jdbcTable.columns()) + .reduce((column1, column2) -> column2) + .map(Column::name) + .ifPresent(s -> columnDefinition.append("AFTER ").append(s)); + } + return columnDefinition.toString(); + } + + private String deleteColumnFieldDefinition( + TableChange.DeleteColumn deleteColumn, JdbcTable jdbcTable) { + if (deleteColumn.fieldName().length > 1) { + throw new UnsupportedOperationException("Doris does not support nested column names."); + } + String col = deleteColumn.fieldName()[0]; + boolean colExists = true; + try { + getJdbcColumnFromTable(jdbcTable, col); + } catch (NoSuchColumnException noSuchColumnException) { + colExists = false; + } + if (!colExists) { + if (BooleanUtils.isTrue(deleteColumn.getIfExists())) { + return ""; + } else { + throw new IllegalArgumentException("Delete column does not exist: " + col); + } + } + return "DROP COLUMN " + BACK_QUOTE + col + BACK_QUOTE; + } + + private String updateColumnTypeFieldDefinition( + TableChange.UpdateColumnType updateColumnType, JdbcTable jdbcTable) { + if (updateColumnType.fieldName().length > 1) { + throw new UnsupportedOperationException("Doris does not support nested column names."); + } + String col = updateColumnType.fieldName()[0]; + JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col); + StringBuilder sqlBuilder = new StringBuilder("MODIFY COLUMN " + BACK_QUOTE + col + BACK_QUOTE); + JdbcColumn newColumn = + JdbcColumn.builder() + .withName(col) + .withType(updateColumnType.getNewDataType()) + .withComment(column.comment()) + .withDefaultValue(DEFAULT_VALUE_NOT_SET) + .withNullable(column.nullable()) + .withAutoIncrement(column.autoIncrement()) + .build(); + return appendColumnDefinition(newColumn, sqlBuilder).toString(); + } + + private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sqlBuilder) { + // Add data type + sqlBuilder + .append(SPACE) + .append(typeConverter.fromGravitinoType(column.dataType())) + .append(SPACE); + + // Add NOT NULL if the column is marked as such + if (column.nullable()) { + sqlBuilder.append("NULL "); + } else { + sqlBuilder.append("NOT NULL "); + } + + // Add DEFAULT value if specified + if (!DEFAULT_VALUE_NOT_SET.equals(column.defaultValue())) { + sqlBuilder + .append("DEFAULT ") + .append(columnDefaultValueConverter.fromGravitino(column.defaultValue())) + .append(SPACE); + } + + // Add column auto_increment if specified + if (column.autoIncrement()) { + sqlBuilder.append(DORIS_AUTO_INCREMENT).append(" "); + } + + // Add column comment if specified + if (StringUtils.isNotEmpty(column.comment())) { + sqlBuilder.append("COMMENT '").append(column.comment()).append("' "); + } + return sqlBuilder; + } + + static String addIndexDefinition(TableChange.AddIndex addIndex) { + return String.format("ADD INDEX %s (%s)", addIndex.getName(), addIndex.getFieldNames()[0][0]); + } + + static String deleteIndexDefinition( + JdbcTable lazyLoadTable, TableChange.DeleteIndex deleteIndex) { + if (deleteIndex.isIfExists()) { + Preconditions.checkArgument( + Arrays.stream(lazyLoadTable.index()) + .anyMatch(index -> index.name().equals(deleteIndex.getName())), + "Index does not exist"); + } + return "DROP INDEX " + deleteIndex.getName(); } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java index 5f88a6f2f26..88d484d35b3 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java @@ -15,5 +15,11 @@ public void testGetErrorCodeFromMessage() { Assertions.assertEquals( DorisExceptionConverter.CODE_DATABASE_EXISTS, DorisExceptionConverter.getErrorCodeFromMessage(msg)); + + msg = + "errCode = 2, detailMessage = Unknown table 'table_name' in default_cluster:database_name"; + Assertions.assertEquals( + DorisExceptionConverter.CODE_NO_SUCH_TABLE, + DorisExceptionConverter.getErrorCodeFromMessage(msg)); } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java new file mode 100644 index 00000000000..f1114dc9552 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java @@ -0,0 +1,406 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.integration.test; + +import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; +import com.datastrato.gravitino.catalog.jdbc.JdbcTable; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.utils.RandomNameUtils; +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +@Tag("gravitino-docker-it") +public class DorisTableOperationsIT extends TestDorisAbstractIT { + private static final Type VARCHAR_255 = Types.VarCharType.of(255); + private static final Type VARCHAR_1024 = Types.VarCharType.of(1024); + + private static final Type INT = Types.IntegerType.get(); + + private static final String databaseName = GravitinoITUtils.genRandomName("doris_test_db"); + + private static final long MAX_WAIT = 5; + + private static final long WAIT_INTERVAL = 1; + + @BeforeAll + public static void startup() { + TestDorisAbstractIT.startup(); + createDatabase(); + } + + private static void createDatabase() { + DATABASE_OPERATIONS.create(databaseName, "test_comment", new HashMap<>()); + } + + private static Map createProperties() { + Map properties = Maps.newHashMap(); + properties.put("replication_allocation", "tag.location.default: 1"); + return properties; + } + + @Test + public void testBasicTableOperation() { + String tableName = GravitinoITUtils.genRandomName("doris_basic_test_table"); + String tableComment = "test_comment"; + List columns = new ArrayList<>(); + JdbcColumn col_1 = + JdbcColumn.builder().withName("col_1").withType(INT).withComment("id").build(); + columns.add(col_1); + JdbcColumn col_2 = + JdbcColumn.builder().withName("col_2").withType(VARCHAR_255).withComment("col_2").build(); + columns.add(col_2); + JdbcColumn col_3 = + JdbcColumn.builder().withName("col_3").withType(VARCHAR_255).withComment("col_3").build(); + columns.add(col_3); + Map properties = new HashMap<>(); + + Distribution distribution = Distributions.hash(32, NamedReference.field("col_1")); + Index[] indexes = new Index[] {}; + + // create table + TABLE_OPERATIONS.create( + databaseName, + tableName, + columns.toArray(new JdbcColumn[0]), + tableComment, + createProperties(), + null, + distribution, + indexes); + List listTables = TABLE_OPERATIONS.listTables(databaseName); + Assertions.assertTrue(listTables.contains(tableName)); + JdbcTable load = TABLE_OPERATIONS.load(databaseName, tableName); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); + + // rename table + String newName = GravitinoITUtils.genRandomName("new_table"); + Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.rename(databaseName, tableName, newName)); + Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.load(databaseName, newName)); + + Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.drop(databaseName, newName)); + + listTables = TABLE_OPERATIONS.listTables(databaseName); + Assertions.assertFalse(listTables.contains(newName)); + + Assertions.assertThrows( + NoSuchTableException.class, () -> TABLE_OPERATIONS.drop(databaseName, newName)); + } + + @Test + public void testAlterTable() { + String tableName = GravitinoITUtils.genRandomName("doris_alter_test_table"); + + String tableComment = "test_comment"; + List columns = new ArrayList<>(); + JdbcColumn col_1 = + JdbcColumn.builder().withName("col_1").withType(INT).withComment("id").build(); + columns.add(col_1); + JdbcColumn col_2 = + JdbcColumn.builder().withName("col_2").withType(VARCHAR_255).withComment("col_2").build(); + columns.add(col_2); + JdbcColumn col_3 = + JdbcColumn.builder().withName("col_3").withType(VARCHAR_255).withComment("col_3").build(); + columns.add(col_3); + Map properties = new HashMap<>(); + + Distribution distribution = Distributions.hash(32, NamedReference.field("col_1")); + Index[] indexes = new Index[] {}; + + // create table + TABLE_OPERATIONS.create( + databaseName, + tableName, + columns.toArray(new JdbcColumn[0]), + tableComment, + createProperties(), + null, + distribution, + indexes); + JdbcTable load = TABLE_OPERATIONS.load(databaseName, tableName); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); + + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.updateColumnType(new String[] {col_3.name()}, VARCHAR_1024)); + + // After modifying the type, check it + columns.clear(); + col_3 = + JdbcColumn.builder() + .withName(col_3.name()) + .withType(VARCHAR_1024) + .withComment(col_3.comment()) + .build(); + columns.add(col_1); + columns.add(col_2); + columns.add(col_3); + + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + indexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + + String colNewComment = "new_comment"; + // update column comment + + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.updateColumnComment(new String[] {col_2.name()}, colNewComment)); + load = TABLE_OPERATIONS.load(databaseName, tableName); + + columns.clear(); + col_2 = + JdbcColumn.builder() + .withName(col_2.name()) + .withType(col_2.dataType()) + .withComment(colNewComment) + .build(); + columns.add(col_1); + columns.add(col_2); + columns.add(col_3); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); + + // add new column + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.addColumn(new String[] {"col_4"}, VARCHAR_255, "txt4", true)); + + columns.clear(); + JdbcColumn col_4 = + JdbcColumn.builder().withName("col_4").withType(VARCHAR_255).withComment("txt4").build(); + columns.add(col_1); + columns.add(col_2); + columns.add(col_3); + columns.add(col_4); + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + indexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + + // change column position + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.updateColumnPosition( + new String[] {"col_3"}, TableChange.ColumnPosition.after("col_4"))); + + columns.clear(); + columns.add(col_1); + columns.add(col_2); + columns.add(col_4); + columns.add(col_3); + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + indexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + + // drop column if exist + TABLE_OPERATIONS.alterTable( + databaseName, tableName, TableChange.deleteColumn(new String[] {"col_4"}, true)); + columns.clear(); + columns.add(col_1); + columns.add(col_2); + columns.add(col_3); + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + indexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + + // delete column that does not exist + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.deleteColumn(new String[] {"col_4"}, false))); + + Assertions.assertEquals( + "Delete column does not exist: col_4", illegalArgumentException.getMessage()); + Assertions.assertDoesNotThrow( + () -> + TABLE_OPERATIONS.alterTable( + databaseName, tableName, TableChange.deleteColumn(new String[] {"col_4"}, true))); + + // test add index + TABLE_OPERATIONS.alterTable( + databaseName, + tableName, + TableChange.addIndex( + Index.IndexType.PRIMARY_KEY, "k2_index", new String[][] {{"col_2"}, {"col_3"}})); + + Index[] newIndexes = + new Index[] {Indexes.primary("k2_index", new String[][] {{"col_2"}, {"col_3"}})}; + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + newIndexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + + // test delete index + TABLE_OPERATIONS.alterTable(databaseName, tableName, TableChange.deleteIndex("k2_index", true)); + + Awaitility.await() + .atMost(MAX_WAIT, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertionsTableInfo( + tableName, + tableComment, + columns, + properties, + indexes, + TABLE_OPERATIONS.load(databaseName, tableName))); + } + + @Test + public void testCreateAllTypeTable() { + String tableName = GravitinoITUtils.genRandomName("all_type_table"); + String tableComment = "test_comment"; + List columns = new ArrayList<>(); + columns.add(JdbcColumn.builder().withName("col_1").withType(Types.IntegerType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_2").withType(Types.BooleanType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_3").withType(Types.ByteType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_4").withType(Types.ShortType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_5").withType(Types.IntegerType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_6").withType(Types.LongType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_7").withType(Types.FloatType.get()).build()); + columns.add(JdbcColumn.builder().withName("col_8").withType(Types.DoubleType.get()).build()); + columns.add( + JdbcColumn.builder().withName("col_9").withType(Types.DecimalType.of(10, 2)).build()); + columns.add(JdbcColumn.builder().withName("col_10").withType(Types.DateType.get()).build()); + columns.add( + JdbcColumn.builder().withName("col_11").withType(Types.FixedCharType.of(10)).build()); + columns.add(JdbcColumn.builder().withName("col_12").withType(Types.VarCharType.of(10)).build()); + columns.add(JdbcColumn.builder().withName("col_13").withType(Types.StringType.get()).build()); + + Distribution distribution = Distributions.hash(32, NamedReference.field("col_1")); + Index[] indexes = new Index[] {}; + // create table + TABLE_OPERATIONS.create( + databaseName, + tableName, + columns.toArray(new JdbcColumn[0]), + tableComment, + createProperties(), + null, + distribution, + indexes); + + JdbcTable load = TABLE_OPERATIONS.load(databaseName, tableName); + assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), null, load); + } + + @Test + public void testCreateNotSupportTypeTable() { + String tableName = RandomNameUtils.genRandomName("unspport_type_table"); + String tableComment = "test_comment"; + List columns = new ArrayList<>(); + List notSupportType = + Arrays.asList( + Types.FixedType.of(10), + Types.IntervalDayType.get(), + Types.IntervalYearType.get(), + Types.UUIDType.get(), + Types.ListType.of(Types.DateType.get(), true), + Types.MapType.of(Types.StringType.get(), Types.IntegerType.get(), true), + Types.UnionType.of(Types.IntegerType.get()), + Types.StructType.of( + Types.StructType.Field.notNullField("col_1", Types.IntegerType.get()))); + + for (Type type : notSupportType) { + columns.clear(); + columns.add(JdbcColumn.builder().withName("col_1").withType(Types.IntegerType.get()).build()); + columns.add( + JdbcColumn.builder().withName("col_2").withType(type).withNullable(false).build()); + + JdbcColumn[] jdbcCols = columns.toArray(new JdbcColumn[0]); + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + TABLE_OPERATIONS.create( + databaseName, + tableName, + jdbcCols, + tableComment, + createProperties(), + null, + Distributions.hash(32, NamedReference.field("col_1")), + Indexes.EMPTY_INDEXES); + }); + Assertions.assertTrue( + illegalArgumentException + .getMessage() + .contains( + String.format( + "Couldn't convert Gravitino type %s to Doris type", type.simpleString()))); + } + } +} diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 09ebab3506b..1538b31c551 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -252,7 +252,8 @@ protected Map getTableProperties(Connection connection, String t @Override protected void correctJdbcTableFields( - Connection connection, String tableName, JdbcTable.Builder tableBuilder) throws SQLException { + Connection connection, String databaseName, String tableName, JdbcTable.Builder tableBuilder) + throws SQLException { if (StringUtils.isEmpty(tableBuilder.comment())) { // In Mysql version 5.7, the comment field value cannot be obtained in the driver API. LOG.warn("Not found comment in mysql driver api. Will try to get comment from sql"); diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java index 6794689dfbd..8291fd6e5f2 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java @@ -103,7 +103,11 @@ protected boolean checkContainerStatus(int retryLimit) { try (ResultSet resultSet = statement.executeQuery(query)) { while (resultSet.next()) { String alive = resultSet.getString("Alive"); - if (alive.equalsIgnoreCase("true")) { + String totalCapacity = resultSet.getString("TotalCapacity"); + float totalCapacityFloat = Float.parseFloat(totalCapacity.split(" ")[0]); + + // alive should be true and totalCapacity should not be 0.000 + if (alive.equalsIgnoreCase("true") && totalCapacityFloat > 0.0f) { LOG.info("Doris container startup success!"); return true; } From 84f4b0139a2b93949ea87b49851d662187e0bd8e Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 16 Apr 2024 19:16:46 +0800 Subject: [PATCH 040/106] [#2821] feat(core): supports metalake event for event listener (#2897) ### What changes were proposed in this pull request? supports metalake event for event listener ### Why are the changes needed? Fix: #2821 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../datastrato/gravitino/GravitinoEnv.java | 15 +- .../api/event/AlterMetalakeEvent.java | 58 ++++++++ .../api/event/AlterMetalakeFailureEvent.java | 47 +++++++ .../api/event/CreateMetalakeEvent.java | 40 ++++++ .../api/event/CreateMetalakeFailureEvent.java | 38 +++++ .../listener/api/event/DropMetalakeEvent.java | 41 ++++++ .../api/event/DropMetalakeFailureEvent.java | 29 ++++ .../listener/api/event/ListMetalakeEvent.java | 21 +++ .../api/event/ListMetalakeFailureEvent.java | 26 ++++ .../listener/api/event/LoadMetalakeEvent.java | 38 +++++ .../api/event/LoadMetalakeFailureEvent.java | 28 ++++ .../listener/api/event/MetalakeEvent.java | 28 ++++ .../api/event/MetalakeFailureEvent.java | 31 +++++ .../listener/api/info/MetalakeInfo.java | 88 ++++++++++++ .../metalake/MetalakeDispatcher.java | 16 +++ .../metalake/MetalakeEventDispatcher.java | 130 ++++++++++++++++++ .../gravitino/metalake/MetalakeManager.java | 3 +- .../gravitino/server/GravitinoServer.java | 4 +- .../server/web/rest/MetalakeOperations.java | 28 ++-- .../web/rest/TestMetalakeOperations.java | 3 +- 20 files changed, 687 insertions(+), 25 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java create mode 100644 core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 1286b8c9c9e..02498d2d077 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -22,6 +22,8 @@ import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.EventListenerManager; import com.datastrato.gravitino.lock.LockManager; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; +import com.datastrato.gravitino.metalake.MetalakeEventDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.JVMMetricsSource; @@ -55,7 +57,7 @@ public class GravitinoEnv { private TopicOperationDispatcher topicOperationDispatcher; - private MetalakeManager metalakeManager; + private MetalakeDispatcher metalakeDispatcher; private AccessControlManager accessControlManager; @@ -131,7 +133,8 @@ public void initialize(Config config) { EventBus eventBus = eventListenerManager.createEventBus(); // Create and initialize metalake related modules - this.metalakeManager = new MetalakeManager(entityStore, idGenerator); + MetalakeManager metalakeManager = new MetalakeManager(entityStore, idGenerator); + this.metalakeDispatcher = new MetalakeEventDispatcher(eventBus, metalakeManager); // Create and initialize Catalog related modules this.catalogManager = new CatalogManager(config, entityStore, idGenerator); @@ -231,12 +234,12 @@ public TopicOperationDispatcher topicOperationDispatcher() { } /** - * Get the MetalakeManager associated with the Gravitino environment. + * Get the MetalakeDispatcher associated with the Gravitino environment. * - * @return The MetalakeManager instance. + * @return The MetalakeDispatcher instance. */ - public MetalakeManager metalakesManager() { - return metalakeManager; + public MetalakeDispatcher metalakeDispatcher() { + return metalakeDispatcher; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java new file mode 100644 index 00000000000..ba1baaafe86 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; + +/** Represents an event fired when a metalake is successfully altered. */ +@DeveloperApi +public final class AlterMetalakeEvent extends MetalakeEvent { + private final MetalakeInfo updatedMetalakeInfo; + private final MetalakeChange[] metalakeChanges; + + /** + * Constructs an instance of {@code AlterMetalakeEvent}, encapsulating the key details about the + * successful alteration of a metalake. + * + * @param user The username of the individual responsible for initiating the metalake alteration. + * @param identifier The unique identifier of the altered metalake, serving as a clear reference + * point for the metalake in question. + * @param metalakeChanges An array of {@link MetalakeChange} objects representing the specific + * changes applied to the metalake during the alteration process. + * @param updatedMetalakeInfo The post-alteration state of the metalake. + */ + public AlterMetalakeEvent( + String user, + NameIdentifier identifier, + MetalakeChange[] metalakeChanges, + MetalakeInfo updatedMetalakeInfo) { + super(user, identifier); + this.metalakeChanges = metalakeChanges.clone(); + this.updatedMetalakeInfo = updatedMetalakeInfo; + } + + /** + * Retrieves the updated state of the metalake after the successful alteration. + * + * @return A {@link MetalakeInfo} instance encapsulating the details of the altered metalake. + */ + public MetalakeInfo updatedMetalakeInfo() { + return updatedMetalakeInfo; + } + + /** + * Retrieves the specific changes that were made to the metalake during the alteration process. + * + * @return An array of {@link MetalakeChange} objects detailing each modification applied to the + * metalake. + */ + public MetalakeChange[] metalakeChanges() { + return metalakeChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java new file mode 100644 index 00000000000..dd58aad26ab --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to alter a metalake fails due to an + * exception. + */ +@DeveloperApi +public final class AlterMetalakeFailureEvent extends MetalakeFailureEvent { + private final MetalakeChange[] metalakeChanges; + + /** + * Constructs an {@code AlterMetalakeFailureEvent} instance, capturing detailed information about + * the failed metalake alteration attempt. + * + * @param user The user who initiated the metalake alteration operation. + * @param identifier The identifier of the metalake that was attempted to be altered. + * @param exception The exception that was thrown during the metalake alteration operation. + * @param metalakeChanges The changes that were attempted on the metalake. + */ + public AlterMetalakeFailureEvent( + String user, + NameIdentifier identifier, + Exception exception, + MetalakeChange[] metalakeChanges) { + super(user, identifier, exception); + this.metalakeChanges = metalakeChanges.clone(); + } + + /** + * Retrieves the changes that were attempted on the metalake. + * + * @return An array of {@link MetalakeChange} objects representing the attempted modifications to + * the metalake. + */ + public MetalakeChange[] metalakeChanges() { + return metalakeChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java new file mode 100644 index 00000000000..b3c9c4a1f50 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; + +/** Represents an event triggered upon the successful creation of a Metalake. */ +@DeveloperApi +public final class CreateMetalakeEvent extends MetalakeEvent { + private final MetalakeInfo createdMetalakeInfo; + /** + * Constructs an instance of {@code CreateMetalakeEvent}, capturing essential details about the + * successful creation of a metalake. + * + * @param user The username of the individual who initiated the metalake creation. + * @param identifier The unique identifier of the metalake that was created. + * @param createdMetalakeInfo The final state of the metalake post-creation. + */ + public CreateMetalakeEvent( + String user, NameIdentifier identifier, MetalakeInfo createdMetalakeInfo) { + super(user, identifier); + this.createdMetalakeInfo = createdMetalakeInfo; + } + + /** + * Retrieves the final state of the Metalake as it was returned to the user after successful + * creation. + * + * @return A {@link MetalakeInfo} instance encapsulating the comprehensive details of the newly + * created Metalake. + */ + public MetalakeInfo createdMetalakeInfo() { + return createdMetalakeInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java new file mode 100644 index 00000000000..64fa352cd13 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; + +/** + * Represents an event that is generated when an attempt to create a Metalake fails due to an + * exception. + */ +@DeveloperApi +public final class CreateMetalakeFailureEvent extends MetalakeFailureEvent { + private final MetalakeInfo createMetalakeRequest; + + public CreateMetalakeFailureEvent( + String user, + NameIdentifier identifier, + Exception exception, + MetalakeInfo createMetalakeRequest) { + super(user, identifier, exception); + this.createMetalakeRequest = createMetalakeRequest; + } + + /** + * Retrieves the original request information for the attempted Metalake creation. + * + * @return The {@link MetalakeInfo} instance representing the request information for the failed + * Metalake creation attempt. + */ + public MetalakeInfo createMetalakeRequest() { + return createMetalakeRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java new file mode 100644 index 00000000000..944bed7a9de --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated after a Metalake is successfully removed from the system. + */ +@DeveloperApi +public final class DropMetalakeEvent extends MetalakeEvent { + private final boolean isExists; + + /** + * Constructs a new {@code DropMetalakeEvent} instance, encapsulating information about the + * outcome of a metalake drop operation. + * + * @param user The user who initiated the drop metalake operation. + * @param identifier The identifier of the metalake that was attempted to be dropped. + * @param isExists A boolean flag indicating whether the metalake existed at the time of the drop + * operation. + */ + public DropMetalakeEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the existence status of the Metalake at the time of the removal operation. + * + * @return A boolean value indicating whether the Metalake existed. {@code true} if the Metalake + * existed, otherwise {@code false}. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java new file mode 100644 index 00000000000..4a70b33a14c --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to remove a Metalake from the system fails + * due to an exception. + */ +@DeveloperApi +public final class DropMetalakeFailureEvent extends MetalakeFailureEvent { + /** + * Constructs a new {@code DropMetalakeFailureEvent} instance, capturing detailed information + * about the failed attempt to drop a metalake. + * + * @param user The user who initiated the drop metalake operation. + * @param identifier The identifier of the metalake that the operation attempted to drop. + * @param exception The exception that was thrown during the drop metalake operation, offering + * insights into what went wrong and why the operation failed. + */ + public DropMetalakeFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java new file mode 100644 index 00000000000..1d72b6477f4 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java @@ -0,0 +1,21 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is triggered upon the successful list of metalakes. */ +@DeveloperApi +public final class ListMetalakeEvent extends MetalakeEvent { + /** + * Constructs an instance of {@code ListMetalakeEvent}. + * + * @param user The username of the individual who initiated the metalake listing. + */ + public ListMetalakeEvent(String user) { + super(user, null); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java new file mode 100644 index 00000000000..bd065e909c7 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java @@ -0,0 +1,26 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list metalakes fails due to an + * exception. + */ +@DeveloperApi +public final class ListMetalakeFailureEvent extends MetalakeFailureEvent { + + /** + * Constructs a {@code ListMetalakeFailureEvent} instance. + * + * @param user The username of the individual who initiated the operation to list metalakes. + * @param exception The exception encountered during the attempt to list metalakes. + */ + public ListMetalakeFailureEvent(String user, Exception exception) { + super(user, null, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java new file mode 100644 index 00000000000..10827eb081e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; + +/** Represents an event that is generated when a Metalake is successfully loaded. */ +@DeveloperApi +public final class LoadMetalakeEvent extends MetalakeEvent { + private final MetalakeInfo loadedMetalakeInfo; + + /** + * Constructs an instance of {@code LoadMetalakeEvent}. + * + * @param user The username of the individual who initiated the metalake loading. + * @param identifier The unique identifier of the metalake that was loaded. + * @param metalakeInfo The state of the metalake post-loading. + */ + public LoadMetalakeEvent(String user, NameIdentifier identifier, MetalakeInfo metalakeInfo) { + super(user, identifier); + this.loadedMetalakeInfo = metalakeInfo; + } + + /** + * Retrieves detailed information about the Metalake that was successfully loaded. + * + * @return A {@link MetalakeInfo} instance containing comprehensive details of the Metalake, + * including its configuration, properties, and state at the time of loading. + */ + public MetalakeInfo loadedMetalakeInfo() { + return loadedMetalakeInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java new file mode 100644 index 00000000000..51cda87dd08 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to load a Metalake into the system fails + * due to an exception. + */ +@DeveloperApi +public final class LoadMetalakeFailureEvent extends MetalakeFailureEvent { + /** + * Constructs a {@code LoadMetalakeFailureEvent} instance. + * + * @param user The user who initiated the metalake loading operation. + * @param identifier The identifier of the metalake that the loading attempt was made for. + * @param exception The exception that was thrown during the metalake loading operation, offering + * insight into the issues encountered. + */ + public LoadMetalakeFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java new file mode 100644 index 00000000000..7820003c023 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events related to Metalake operations. This class extends + * {@link Event} to provide a more specific context involving operations on Metalakes, such as + * creation, deletion, or modification. It captures essential information including the user + * performing the operation and the identifier of the Metalake being operated on. + */ +@DeveloperApi +public abstract class MetalakeEvent extends Event { + /** + * Constructs a new {@code MetalakeEvent} with the specified user and Metalake identifier. + * + * @param user The user responsible for triggering the Metalake operation. + * @param identifier The identifier of the Metalake involved in the operation. + */ + protected MetalakeEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java new file mode 100644 index 00000000000..01637c6aeea --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a Metalake operation fails due to + * an exception. This class extends {@link FailureEvent} to provide a more specific context related + * to Metalake operations, encapsulating details about the user who initiated the operation, the + * identifier of the Metalake involved, and the exception that led to the failure. + */ +@DeveloperApi +public abstract class MetalakeFailureEvent extends FailureEvent { + /** + * Constructs a new {@code MetalakeFailureEvent} instance, capturing information about the failed + * Metalake operation. + * + * @param user The user associated with the failed Metalake operation. + * @param identifier The identifier of the Metalake that was involved in the failed operation. + * @param exception The exception that was thrown during the Metalake operation, indicating the + * cause of the failure. + */ + protected MetalakeFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java new file mode 100644 index 00000000000..d17566b1929 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java @@ -0,0 +1,88 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.Metalake; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Provides access to metadata about a Metalake instance, designed for use by event listeners. This + * class encapsulates the essential attributes of a Metalake, including its name, optional + * description, properties, and audit information. + */ +@DeveloperApi +public final class MetalakeInfo { + private final String name; + @Nullable private final String comment; + private final Map properties; + @Nullable private final Audit audit; + + /** + * Constructs MetalakeInfo from an existing Metalake object. + * + * @param metalake The Metalake instance to extract information from. + */ + public MetalakeInfo(Metalake metalake) { + this(metalake.name(), metalake.comment(), metalake.properties(), metalake.auditInfo()); + } + + /** + * Directly constructs MetalakeInfo with specified details. + * + * @param name The name of the Metalake. + * @param comment An optional description for the Metalake. + * @param properties A map of properties associated with the Metalake. + * @param audit Optional audit details for the Metalake. + */ + public MetalakeInfo(String name, String comment, Map properties, Audit audit) { + this.name = name; + this.comment = comment; + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + this.audit = audit; + } + + /** + * Returns the audit information of the Metalake. + * + * @return Audit details, or null if not available. + */ + @Nullable + public Audit auditInfo() { + return audit; + } + + /** + * Returns the name of the Metalake. + * + * @return The Metalake's name. + */ + public String name() { + return name; + } + + /** + * Returns the optional comment describing the Metalake. + * + * @return The comment, or null if not provided. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Returns the properties of the Metalake. + * + * @return A map of Metalake properties. + */ + public Map properties() { + return properties; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java new file mode 100644 index 00000000000..c7848326a14 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.metalake; + +import com.datastrato.gravitino.SupportsMetalakes; + +/** + * {@code MetalakeDispatcher} interface acts as a specialization of the {@link SupportsMetalakes} + * interface. This interface is designed to potentially add custom behaviors or operations related + * to dispatching or handling metalake-related events or actions that are not covered by the + * standard {@code SupportsMetalakes} operations. + */ +public interface MetalakeDispatcher extends SupportsMetalakes {} diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java new file mode 100644 index 00000000000..16744247eae --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java @@ -0,0 +1,130 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.metalake; + +import com.datastrato.gravitino.Metalake; +import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterMetalakeEvent; +import com.datastrato.gravitino.listener.api.event.AlterMetalakeFailureEvent; +import com.datastrato.gravitino.listener.api.event.CreateMetalakeEvent; +import com.datastrato.gravitino.listener.api.event.CreateMetalakeFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropMetalakeEvent; +import com.datastrato.gravitino.listener.api.event.DropMetalakeFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListMetalakeEvent; +import com.datastrato.gravitino.listener.api.event.ListMetalakeFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadMetalakeEvent; +import com.datastrato.gravitino.listener.api.event.LoadMetalakeFailureEvent; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code MetalakeEventDispatcher} is a decorator for {@link MetalakeDispatcher} that not only + * delegates metalake operations to the underlying metalake dispatcher but also dispatches + * corresponding events to an {@link EventBus} after each operation is completed. This allows for + * event-driven workflows or monitoring of metalake operations. + */ +public class MetalakeEventDispatcher implements MetalakeDispatcher { + private final EventBus eventBus; + private final MetalakeDispatcher dispatcher; + + /** + * Constructs a MetalakeEventDispatcher with a specified EventBus and MetalakeDispatcher. + * + * @param eventBus The EventBus to which events will be dispatched. + * @param dispatcher The underlying {@link MetalakeDispatcher} that will perform the actual + * metalake operations. + */ + public MetalakeEventDispatcher(EventBus eventBus, MetalakeDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public Metalake[] listMetalakes() { + try { + Metalake[] metalakes = dispatcher.listMetalakes(); + eventBus.dispatchEvent(new ListMetalakeEvent(PrincipalUtils.getCurrentUserName())); + return metalakes; + } catch (Exception e) { + eventBus.dispatchEvent(new ListMetalakeFailureEvent(PrincipalUtils.getCurrentUserName(), e)); + throw e; + } + } + + @Override + public Metalake loadMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + Metalake metalake = dispatcher.loadMetalake(ident); + eventBus.dispatchEvent( + new LoadMetalakeEvent( + PrincipalUtils.getCurrentUserName(), ident, new MetalakeInfo(metalake))); + return metalake; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadMetalakeFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public boolean metalakeExists(NameIdentifier ident) { + return dispatcher.metalakeExists(ident); + } + + @Override + public Metalake createMetalake( + NameIdentifier ident, String comment, Map properties) + throws MetalakeAlreadyExistsException { + try { + Metalake metalake = dispatcher.createMetalake(ident, comment, properties); + eventBus.dispatchEvent( + new CreateMetalakeEvent( + PrincipalUtils.getCurrentUserName(), ident, new MetalakeInfo(metalake))); + return metalake; + } catch (Exception e) { + MetalakeInfo metalakeInfo = new MetalakeInfo(ident.name(), comment, properties, null); + eventBus.dispatchEvent( + new CreateMetalakeFailureEvent( + PrincipalUtils.getCurrentUserName(), ident, e, metalakeInfo)); + throw e; + } + } + + @Override + public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) + throws NoSuchMetalakeException, IllegalArgumentException { + try { + Metalake metalake = dispatcher.alterMetalake(ident, changes); + eventBus.dispatchEvent( + new AlterMetalakeEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new MetalakeInfo(metalake))); + return metalake; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterMetalakeFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } + } + + @Override + public boolean dropMetalake(NameIdentifier ident) { + try { + boolean isExists = dispatcher.dropMetalake(ident); + eventBus.dispatchEvent( + new DropMetalakeEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropMetalakeFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java index d67bc8aa27b..3f445cbbf22 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.SupportsMetalakes; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; @@ -30,7 +29,7 @@ import org.slf4j.LoggerFactory; /** Manages Metalakes within the Gravitino system. */ -public class MetalakeManager implements SupportsMetalakes { +public class MetalakeManager implements MetalakeDispatcher { private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index bc1182865b2..88ca4a815ab 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -10,7 +10,7 @@ import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; -import com.datastrato.gravitino.metalake.MetalakeManager; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.MetricsSource; import com.datastrato.gravitino.server.authentication.ServerAuthenticator; @@ -76,7 +76,7 @@ private void initializeRestApi() { new AbstractBinder() { @Override protected void configure() { - bind(gravitinoEnv.metalakesManager()).to(MetalakeManager.class).ranked(1); + bind(gravitinoEnv.metalakeDispatcher()).to(MetalakeDispatcher.class).ranked(1); bind(gravitinoEnv.catalogDispatcher()).to(CatalogDispatcher.class).ranked(1); bind(gravitinoEnv.schemaDispatcher()).to(SchemaDispatcher.class).ranked(1); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java index c0a91613faa..b2037ee2b7b 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java @@ -6,6 +6,7 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import com.datastrato.gravitino.Metalake; import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -19,8 +20,7 @@ import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.lock.LockType; import com.datastrato.gravitino.lock.TreeLockUtils; -import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.metalake.MetalakeManager; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; import com.datastrato.gravitino.metrics.MetricNames; import com.datastrato.gravitino.server.web.Utils; import java.util.Arrays; @@ -47,13 +47,13 @@ public class MetalakeOperations { private static final Logger LOG = LoggerFactory.getLogger(MetalakeOperations.class); - private final MetalakeManager manager; + private final MetalakeDispatcher metalakeDispatcher; @Context private HttpServletRequest httpRequest; @Inject - public MetalakeOperations(MetalakeManager manager) { - this.manager = manager; + public MetalakeOperations(MetalakeDispatcher dispatcher) { + this.metalakeDispatcher = dispatcher; } @GET @@ -65,8 +65,8 @@ public Response listMetalakes() { return Utils.doAs( httpRequest, () -> { - BaseMetalake[] metalakes = - TreeLockUtils.doWithRootTreeLock(LockType.READ, manager::listMetalakes); + Metalake[] metalakes = + TreeLockUtils.doWithRootTreeLock(LockType.READ, metalakeDispatcher::listMetalakes); MetalakeDTO[] metalakeDTOS = Arrays.stream(metalakes).map(DTOConverters::toDTO).toArray(MetalakeDTO[]::new); return Utils.ok(new MetalakeListResponse(metalakeDTOS)); @@ -89,11 +89,11 @@ public Response createMetalake(MetalakeCreateRequest request) { () -> { request.validate(); NameIdentifier ident = NameIdentifier.ofMetalake(request.getName()); - BaseMetalake metalake = + Metalake metalake = TreeLockUtils.doWithRootTreeLock( LockType.WRITE, () -> - manager.createMetalake( + metalakeDispatcher.createMetalake( ident, request.getComment(), request.getProperties())); return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); }); @@ -114,9 +114,9 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { httpRequest, () -> { NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); - BaseMetalake metalake = + Metalake metalake = TreeLockUtils.doWithTreeLock( - identifier, LockType.READ, () -> manager.loadMetalake(identifier)); + identifier, LockType.READ, () -> metalakeDispatcher.loadMetalake(identifier)); return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); }); @@ -142,9 +142,9 @@ public Response alterMetalake( updatesRequest.getUpdates().stream() .map(MetalakeUpdateRequest::metalakeChange) .toArray(MetalakeChange[]::new); - BaseMetalake updatedMetalake = + Metalake updatedMetalake = TreeLockUtils.doWithRootTreeLock( - LockType.WRITE, () -> manager.alterMetalake(identifier, changes)); + LockType.WRITE, () -> metalakeDispatcher.alterMetalake(identifier, changes)); return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(updatedMetalake))); }); @@ -166,7 +166,7 @@ public Response dropMetalake(@PathParam("name") String metalakeName) { NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); boolean dropped = TreeLockUtils.doWithRootTreeLock( - LockType.WRITE, () -> manager.dropMetalake(identifier)); + LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier)); if (!dropped) { LOG.warn("Failed to drop metalake by name {}", metalakeName); } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeOperations.java index ca0a2e1e959..fd2e9a6bafe 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeOperations.java @@ -29,6 +29,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.rest.RESTUtils; import com.google.common.collect.ImmutableMap; @@ -87,7 +88,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(metalakeManager).to(MetalakeManager.class).ranked(2); + bind(metalakeManager).to(MetalakeDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); From dfa42d1524cd1c8daaf97e9bf8a650c1f6c928eb Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 16 Apr 2024 20:54:28 +0800 Subject: [PATCH 041/106] [#2786] improvement(jdbc-common): Make value `testOnBorrow` configurable in JDBC catalog (#2876) ### What changes were proposed in this pull request? Make the value `testOnBorrow` in JDBC catalogs can be tuned by users. ### Why are the changes needed? When we do performance test for the JDBC catalog, we'd better set `testOnBorrow` to false as testing connection is time-consuming Fix: #2786 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Add unit test. --- .../catalog/jdbc/config/JdbcConfig.java | 11 ++++++++ .../catalog/jdbc/utils/DataSourceUtils.java | 2 +- .../catalog/jdbc/utils/TestJdbcConfig.java | 25 +++++++++++++++++++ docs/jdbc-mysql-catalog.md | 16 ++++++------ docs/jdbc-postgresql-catalog.md | 18 ++++++------- 5 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java index 39480ab3a09..de397d3eb18 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java @@ -70,6 +70,13 @@ public class JdbcConfig extends Config { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(10); + public static final ConfigEntry TEST_ON_BORROW = + new ConfigBuilder("jdbc.pool.test-on-borrow") + .doc("Whether to test the connection on borrow") + .version(ConfigConstants.VERSION_0_5_0) + .booleanConf() + .createWithDefault(true); + public String getJdbcUrl() { return get(JDBC_URL); } @@ -98,6 +105,10 @@ public String getJdbcDatabase() { return get(JDBC_DATABASE); } + public boolean getTestOnBorrow() { + return get(TEST_ON_BORROW); + } + public JdbcConfig(Map properties) { super(false); loadFromMap(properties, k -> true); diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java index 37865323988..95eaf682348 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java @@ -51,7 +51,7 @@ private static DataSource createDBCPDataSource(JdbcConfig jdbcConfig) throws Exc basicDataSource.setMinIdle(jdbcConfig.getPoolMinSize()); // Set each time a connection is taken out from the connection pool, a test statement will be // executed to confirm whether the connection is valid. - basicDataSource.setTestOnBorrow(true); + basicDataSource.setTestOnBorrow(jdbcConfig.getTestOnBorrow()); basicDataSource.setValidationQuery(POOL_TEST_QUERY); return basicDataSource; } diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java new file mode 100644 index 00000000000..3d7cbaa6413 --- /dev/null +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog.jdbc.utils; + +import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestJdbcConfig { + + @Test + void testOnBorrow() { + JdbcConfig jdbcConfig = new JdbcConfig(Maps.newHashMap()); + Assertions.assertTrue(jdbcConfig.getTestOnBorrow()); + + ImmutableMap immutableMap = ImmutableMap.of("jdbc.pool.test-on-borrow", "false"); + jdbcConfig = new JdbcConfig(immutableMap); + Assertions.assertFalse(jdbcConfig.getTestOnBorrow()); + } +} diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index dc0a51833b2..9dd3ee5a449 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -39,14 +39,14 @@ Check the relevant data source configuration in [data source properties](https:/ If you use a JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-user` and `jdbc-password` to catalog properties. -| Configuration item | Description | Default value | Required | Since Version | -|-------------------------|---------------------------------------------------------------------------------------------------------|---------------|----------|---------------| -| `jdbc-url` | JDBC URL for connecting to the database. For example, `jdbc:mysql://localhost:3306` | (none) | Yes | 0.3.0 | -| `jdbc-driver` | The driver of the JDBC connection. For example, `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver`. | (none) | Yes | 0.3.0 | -| `jdbc-user` | The JDBC user name. | (none) | Yes | 0.3.0 | -| `jdbc-password` | The JDBC password. | (none) | Yes | 0.3.0 | -| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by default. | `2` | No | 0.3.0 | -| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10` by default. | `10` | No | 0.3.0 | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------|--------------------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `jdbc-url` | JDBC URL for connecting to the database. For example, `jdbc:mysql://localhost:3306` | (none) | Yes | 0.3.0 | +| `jdbc-driver` | The driver of the JDBC connection. For example, `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver`. | (none) | Yes | 0.3.0 | +| `jdbc-user` | The JDBC user name. | (none) | Yes | 0.3.0 | +| `jdbc-password` | The JDBC password. | (none) | Yes | 0.3.0 | +| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by default. | `2` | No | 0.3.0 | +| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10` by default. | `10` | No | 0.3.0 | :::caution You must download the corresponding JDBC driver to the `catalogs/jdbc-mysql/libs` directory. diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 4a6d747f253..6b85373db51 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -37,15 +37,15 @@ You can check the relevant data source configuration in [data source properties] If you use JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-database`, `jdbc-user` and `jdbc-password` to catalog properties. -| Configuration item | Description | Default value | Required | Since Version | -|----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| -| `jdbc-url` | JDBC URL for connecting to the database. You need to specify the database in the URL. For example `jdbc:postgresql://localhost:3306/pg_database?sslmode=require`. | (none) | Yes | 0.3.0 | -| `jdbc-driver` | The driver of the JDBC connection. For example `org.postgresql.Driver`. | (none) | Yes | 0.3.0 | -| `jdbc-database` | The database of the JDBC connection. Configure it with the same value as the database in the `jdbc-url`. For example `pg_database`. | (none) | Yes | 0.3.0 | -| `jdbc-user` | The JDBC user name. | (none) | Yes | 0.3.0 | -| `jdbc-password` | The JDBC password. | (none) | Yes | 0.3.0 | -| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by default. | `2` | No | 0.3.0 | -| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10` by default. | `10` | No | 0.3.0 | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `jdbc-url` | JDBC URL for connecting to the database. You need to specify the database in the URL. For example `jdbc:postgresql://localhost:3306/pg_database?sslmode=require`. | (none) | Yes | 0.3.0 | +| `jdbc-driver` | The driver of the JDBC connection. For example `org.postgresql.Driver`. | (none) | Yes | 0.3.0 | +| `jdbc-database` | The database of the JDBC connection. Configure it with the same value as the database in the `jdbc-url`. For example `pg_database`. | (none) | Yes | 0.3.0 | +| `jdbc-user` | The JDBC user name. | (none) | Yes | 0.3.0 | +| `jdbc-password` | The JDBC password. | (none) | Yes | 0.3.0 | +| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by default. | `2` | No | 0.3.0 | +| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10` by default. | `10` | No | 0.3.0 | :::caution You must download the corresponding JDBC driver to the `catalogs/jdbc-postgresql/libs` directory. From 71c687d27d675ca86d9a57f16560f73f3b4430f7 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Tue, 16 Apr 2024 22:01:14 +0800 Subject: [PATCH 042/106] [MINOR] fix(ci): fix build pnpm errors (#2982) ### What changes were proposed in this pull request? fix build pnpm errors. Since the official pnpm update came out, there was a conflict with lockfile version and one of the frontend dependencies(`react-hooks-form`) after downloading it. ### Why are the changes needed? Upgrading lockfile and that dependency fixed the issue. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- web/package.json | 2 +- web/pnpm-lock.yaml | 6054 ++++++++++++++++++++++++-------------------- 2 files changed, 3336 insertions(+), 2720 deletions(-) diff --git a/web/package.json b/web/package.json index 10e25307948..62a3b9a05fd 100644 --- a/web/package.json +++ b/web/package.json @@ -37,7 +37,7 @@ "qs": "^6.11.2", "react": "^18", "react-dom": "^18", - "react-hook-form": "^7.48.2", + "react-hook-form": "^7.51.3", "react-hot-toast": "^2.4.1", "react-redux": "^8.1.3", "react-use": "^17.4.1", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index c6c0280e304..4b8a3023228 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -1,310 +1,227 @@ -lockfileVersion: '6.0' +lockfileVersion: '9.0' settings: autoInstallPeers: true excludeLinksFromLockfile: false -dependencies: - '@emotion/cache': - specifier: ^11.11.0 - version: 11.11.0 - '@emotion/react': - specifier: ^11.11.1 - version: 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': - specifier: ^11.11.0 - version: 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - '@hookform/resolvers': - specifier: ^3.3.2 - version: 3.3.4(react-hook-form@7.49.3) - '@mui/icons-material': - specifier: ^5.15.11 - version: 5.15.11(@mui/material@5.15.3)(@types/react@18.2.47)(react@18.2.0) - '@mui/lab': - specifier: ^5.0.0-alpha.153 - version: 5.0.0-alpha.159(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@mui/material@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/material': - specifier: ^5.14.18 - version: 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/x-data-grid': - specifier: ^6.18.2 - version: 6.18.7(@mui/material@5.15.3)(@mui/system@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/x-tree-view': - specifier: ^6.17.0 - version: 6.17.0(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@mui/material@5.15.3)(@mui/system@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@reduxjs/toolkit': - specifier: ^1.9.7 - version: 1.9.7(react-redux@8.1.3)(react@18.2.0) - antd: - specifier: ^5.13.3 - version: 5.13.3(react-dom@18.2.0)(react@18.2.0) - axios: - specifier: ^1.6.8 - version: 1.6.8 - chroma-js: - specifier: ^2.4.2 - version: 2.4.2 - clsx: - specifier: ^2.0.0 - version: 2.1.0 - dayjs: - specifier: ^1.11.10 - version: 1.11.10 - lodash-es: - specifier: ^4.17.21 - version: 4.17.21 - next: - specifier: 14.0.3 - version: 14.0.3(react-dom@18.2.0)(react@18.2.0) - nprogress: - specifier: ^0.2.0 - version: 0.2.0 - qs: - specifier: ^6.11.2 - version: 6.11.2 - react: - specifier: ^18 - version: 18.2.0 - react-dom: - specifier: ^18 - version: 18.2.0(react@18.2.0) - react-hook-form: - specifier: ^7.48.2 - version: 7.49.3(react@18.2.0) - react-hot-toast: - specifier: ^2.4.1 - version: 2.4.1(csstype@3.1.3)(react-dom@18.2.0)(react@18.2.0) - react-redux: - specifier: ^8.1.3 - version: 8.1.3(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0)(redux@4.2.1) - react-use: - specifier: ^17.4.1 - version: 17.4.2(react-dom@18.2.0)(react@18.2.0) - yup: - specifier: ^1.3.2 - version: 1.3.3 - -devDependencies: - '@iconify/react': - specifier: ^4.1.1 - version: 4.1.1(react@18.2.0) - '@next/bundle-analyzer': - specifier: ^14.0.4 - version: 14.0.4 - '@types/lodash-es': - specifier: ^4.17.12 - version: 4.17.12 - '@types/node': - specifier: ^20.10.5 - version: 20.10.7 - '@types/qs': - specifier: ^6.9.11 - version: 6.9.11 - '@types/react': - specifier: ^18.2.46 - version: 18.2.47 - autoprefixer: - specifier: ^10.4.16 - version: 10.4.16(postcss@8.4.33) - env-cmd: - specifier: ^10.1.0 - version: 10.1.0 - eslint: - specifier: ^8 - version: 8.56.0 - eslint-config-next: - specifier: 14.0.3 - version: 14.0.3(eslint@8.56.0)(typescript@5.3.3) - eslint-config-prettier: - specifier: ^9.0.0 - version: 9.1.0(eslint@8.56.0) - postcss: - specifier: ^8 - version: 8.4.33 - prettier: - specifier: ^3.1.0 - version: 3.1.1 - tailwindcss: - specifier: ^3.3.5 - version: 3.4.1 - typescript: - specifier: ^5.3.3 - version: 5.3.3 +importers: + + .: + dependencies: + '@emotion/cache': + specifier: ^11.11.0 + version: 11.11.0 + '@emotion/react': + specifier: ^11.11.1 + version: 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': + specifier: ^11.11.0 + version: 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@hookform/resolvers': + specifier: ^3.3.2 + version: 3.3.4(react-hook-form@7.51.3(react@18.2.0)) + '@mui/icons-material': + specifier: ^5.15.11 + version: 5.15.11(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/lab': + specifier: ^5.0.0-alpha.153 + version: 5.0.0-alpha.159(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/material': + specifier: ^5.14.18 + version: 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/x-data-grid': + specifier: ^6.18.2 + version: 6.18.7(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mui/system@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/x-tree-view': + specifier: ^6.17.0 + version: 6.17.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mui/system@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@reduxjs/toolkit': + specifier: ^1.9.7 + version: 1.9.7(react-redux@8.1.3(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(redux@4.2.1))(react@18.2.0) + antd: + specifier: ^5.13.3 + version: 5.13.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + axios: + specifier: ^1.6.8 + version: 1.6.8 + chroma-js: + specifier: ^2.4.2 + version: 2.4.2 + clsx: + specifier: ^2.0.0 + version: 2.1.0 + dayjs: + specifier: ^1.11.10 + version: 1.11.10 + lodash-es: + specifier: ^4.17.21 + version: 4.17.21 + next: + specifier: 14.0.3 + version: 14.0.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + nprogress: + specifier: ^0.2.0 + version: 0.2.0 + qs: + specifier: ^6.11.2 + version: 6.11.2 + react: + specifier: ^18 + version: 18.2.0 + react-dom: + specifier: ^18 + version: 18.2.0(react@18.2.0) + react-hook-form: + specifier: ^7.51.3 + version: 7.51.3(react@18.2.0) + react-hot-toast: + specifier: ^2.4.1 + version: 2.4.1(csstype@3.1.3)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react-redux: + specifier: ^8.1.3 + version: 8.1.3(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(redux@4.2.1) + react-use: + specifier: ^17.4.1 + version: 17.4.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + yup: + specifier: ^1.3.2 + version: 1.3.3 + devDependencies: + '@iconify/react': + specifier: ^4.1.1 + version: 4.1.1(react@18.2.0) + '@next/bundle-analyzer': + specifier: ^14.0.4 + version: 14.0.4 + '@types/lodash-es': + specifier: ^4.17.12 + version: 4.17.12 + '@types/node': + specifier: ^20.10.5 + version: 20.10.7 + '@types/qs': + specifier: ^6.9.11 + version: 6.9.11 + '@types/react': + specifier: ^18.2.46 + version: 18.2.47 + autoprefixer: + specifier: ^10.4.16 + version: 10.4.16(postcss@8.4.33) + env-cmd: + specifier: ^10.1.0 + version: 10.1.0 + eslint: + specifier: ^8 + version: 8.56.0 + eslint-config-next: + specifier: 14.0.3 + version: 14.0.3(eslint@8.56.0)(typescript@5.3.3) + eslint-config-prettier: + specifier: ^9.0.0 + version: 9.1.0(eslint@8.56.0) + postcss: + specifier: ^8 + version: 8.4.33 + prettier: + specifier: ^3.1.0 + version: 3.1.1 + tailwindcss: + specifier: ^3.3.5 + version: 3.4.1 + typescript: + specifier: ^5.3.3 + version: 5.3.3 packages: - /@aashutoshrathi/word-wrap@1.2.6: + '@aashutoshrathi/word-wrap@1.2.6': resolution: {integrity: sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==} engines: {node: '>=0.10.0'} - dev: true - /@alloc/quick-lru@5.2.0: + '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} - dev: true - /@ant-design/colors@7.0.2: + '@ant-design/colors@7.0.2': resolution: {integrity: sha512-7KJkhTiPiLHSu+LmMJnehfJ6242OCxSlR3xHVBecYxnMW8MS/878NXct1GqYARyL59fyeFdKRxXTfvR9SnDgJg==} - dependencies: - '@ctrl/tinycolor': 3.6.1 - dev: false - /@ant-design/cssinjs@1.18.4(react-dom@18.2.0)(react@18.2.0): + '@ant-design/cssinjs@1.18.4': resolution: {integrity: sha512-IrUAOj5TYuMG556C9gdbFuOrigyhzhU5ZYpWb3gYTxAwymVqRbvLzFCZg6OsjLBR6GhzcxYF3AhxKmjB+rA2xA==} peerDependencies: react: '>=16.0.0' react-dom: '>=16.0.0' - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/hash': 0.8.0 - '@emotion/unitless': 0.7.5 - classnames: 2.5.1 - csstype: 3.1.3 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - stylis: 4.3.1 - dev: false - /@ant-design/icons-svg@4.4.1: + '@ant-design/icons-svg@4.4.1': resolution: {integrity: sha512-dIEsZNOeikwCsgSLpwE0Vsgiaj10bKcgut+CvGD5Dck4kOA+TuFcTCM+l2uEs4cMUANkfjrt3hDg1PzlDNhXqg==} - dev: false - /@ant-design/icons@5.3.0(react-dom@18.2.0)(react@18.2.0): + '@ant-design/icons@5.3.0': resolution: {integrity: sha512-69FgBsIkeCjw72ZU3fJpqjhmLCPrzKGEllbrAZK7MUdt1BrKsyG6A8YDCBPKea27UQ0tRXi33PcjR4tp/tEXMg==} engines: {node: '>=8'} peerDependencies: react: '>=16.0.0' react-dom: '>=16.0.0' - dependencies: - '@ant-design/colors': 7.0.2 - '@ant-design/icons-svg': 4.4.1 - '@babel/runtime': 7.23.8 - classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@ant-design/react-slick@1.0.2(react@18.2.0): + '@ant-design/react-slick@1.0.2': resolution: {integrity: sha512-Wj8onxL/T8KQLFFiCA4t8eIRGpRR+UPgOdac2sYzonv+i0n3kXHmvHLLiOYL655DQx2Umii9Y9nNgL7ssu5haQ==} peerDependencies: react: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - classnames: 2.5.1 - json2mq: 0.2.0 - react: 18.2.0 - resize-observer-polyfill: 1.5.1 - throttle-debounce: 5.0.0 - dev: false - /@babel/code-frame@7.23.5: + '@babel/code-frame@7.23.5': resolution: {integrity: sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/highlight': 7.23.4 - chalk: 2.4.2 - dev: false - /@babel/helper-module-imports@7.22.15: + '@babel/helper-module-imports@7.22.15': resolution: {integrity: sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/types': 7.23.6 - dev: false - /@babel/helper-string-parser@7.23.4: + '@babel/helper-string-parser@7.23.4': resolution: {integrity: sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==} engines: {node: '>=6.9.0'} - dev: false - /@babel/helper-validator-identifier@7.22.20: + '@babel/helper-validator-identifier@7.22.20': resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==} engines: {node: '>=6.9.0'} - dev: false - /@babel/highlight@7.23.4: + '@babel/highlight@7.23.4': resolution: {integrity: sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/helper-validator-identifier': 7.22.20 - chalk: 2.4.2 - js-tokens: 4.0.0 - dev: false - /@babel/runtime@7.23.8: + '@babel/runtime@7.23.8': resolution: {integrity: sha512-Y7KbAP984rn1VGMbGqKmBLio9V7y5Je9GvU4rQPCPinCyNfUcToxIXl06d59URp/F3LwinvODxab5N/G6qggkw==} engines: {node: '>=6.9.0'} - dependencies: - regenerator-runtime: 0.14.1 - /@babel/runtime@7.24.0: + '@babel/runtime@7.24.0': resolution: {integrity: sha512-Chk32uHMg6TnQdvw2e9IlqPpFX/6NLuK0Ys2PqLb7/gL5uFn9mXvK715FGLlOLQrcO4qIkNHkvPGktzzXexsFw==} engines: {node: '>=6.9.0'} - dependencies: - regenerator-runtime: 0.14.1 - dev: false - /@babel/types@7.23.6: + '@babel/types@7.23.6': resolution: {integrity: sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/helper-string-parser': 7.23.4 - '@babel/helper-validator-identifier': 7.22.20 - to-fast-properties: 2.0.0 - dev: false - /@ctrl/tinycolor@3.6.1: + '@ctrl/tinycolor@3.6.1': resolution: {integrity: sha512-SITSV6aIXsuVNV3f3O0f2n/cgyEDWoSqtZMYiAmcsYHydcKrOz3gUxB/iXd/Qf08+IZX4KpgNbvUdMBmWz+kcA==} engines: {node: '>=10'} - dev: false - /@emotion/babel-plugin@11.11.0: + '@emotion/babel-plugin@11.11.0': resolution: {integrity: sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==} - dependencies: - '@babel/helper-module-imports': 7.22.15 - '@babel/runtime': 7.23.8 - '@emotion/hash': 0.9.1 - '@emotion/memoize': 0.8.1 - '@emotion/serialize': 1.1.3 - babel-plugin-macros: 3.1.0 - convert-source-map: 1.9.0 - escape-string-regexp: 4.0.0 - find-root: 1.1.0 - source-map: 0.5.7 - stylis: 4.2.0 - dev: false - /@emotion/cache@11.11.0: + '@emotion/cache@11.11.0': resolution: {integrity: sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==} - dependencies: - '@emotion/memoize': 0.8.1 - '@emotion/sheet': 1.2.2 - '@emotion/utils': 1.2.1 - '@emotion/weak-memoize': 0.3.1 - stylis: 4.2.0 - dev: false - /@emotion/hash@0.8.0: + '@emotion/hash@0.8.0': resolution: {integrity: sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==} - dev: false - /@emotion/hash@0.9.1: + '@emotion/hash@0.9.1': resolution: {integrity: sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ==} - dev: false - /@emotion/is-prop-valid@1.2.1: + '@emotion/is-prop-valid@1.2.1': resolution: {integrity: sha512-61Mf7Ufx4aDxx1xlDeOm8aFFigGHE4z+0sKCa+IHCeZKiyP9RLD0Mmx7m8b9/Cf37f7NAvQOOJAbQQGVr5uERw==} - dependencies: - '@emotion/memoize': 0.8.1 - dev: false - /@emotion/memoize@0.8.1: + '@emotion/memoize@0.8.1': resolution: {integrity: sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA==} - dev: false - /@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0): + '@emotion/react@11.11.3': resolution: {integrity: sha512-Cnn0kuq4DoONOMcnoVsTOR8E+AdnKFf//6kUWc4LCdnxj31pZWn7rIULd6Y7/Js1PiPHzn7SKCM9vB/jBni8eA==} peerDependencies: '@types/react': '*' @@ -312,34 +229,14 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/babel-plugin': 11.11.0 - '@emotion/cache': 11.11.0 - '@emotion/serialize': 1.1.3 - '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) - '@emotion/utils': 1.2.1 - '@emotion/weak-memoize': 0.3.1 - '@types/react': 18.2.47 - hoist-non-react-statics: 3.3.2 - react: 18.2.0 - dev: false - /@emotion/serialize@1.1.3: + '@emotion/serialize@1.1.3': resolution: {integrity: sha512-iD4D6QVZFDhcbH0RAG1uVu1CwVLMWUkCvAqqlewO/rxf8+87yIBAlt4+AxMiiKPLs5hFc0owNk/sLLAOROw3cA==} - dependencies: - '@emotion/hash': 0.9.1 - '@emotion/memoize': 0.8.1 - '@emotion/unitless': 0.8.1 - '@emotion/utils': 1.2.1 - csstype: 3.1.3 - dev: false - /@emotion/sheet@1.2.2: + '@emotion/sheet@1.2.2': resolution: {integrity: sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA==} - dev: false - /@emotion/styled@11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0): + '@emotion/styled@11.11.0': resolution: {integrity: sha512-hM5Nnvu9P3midq5aaXj4I+lnSfNi7Pmd4EWk1fOZ3pxookaQTNew6bp4JaCBYM4HVFZF9g7UjJmsUmC2JlxOng==} peerDependencies: '@emotion/react': ^11.0.0-rc.0 @@ -348,190 +245,104 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/babel-plugin': 11.11.0 - '@emotion/is-prop-valid': 1.2.1 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/serialize': 1.1.3 - '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) - '@emotion/utils': 1.2.1 - '@types/react': 18.2.47 - react: 18.2.0 - dev: false - /@emotion/unitless@0.7.5: + '@emotion/unitless@0.7.5': resolution: {integrity: sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg==} - dev: false - /@emotion/unitless@0.8.1: + '@emotion/unitless@0.8.1': resolution: {integrity: sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ==} - dev: false - /@emotion/use-insertion-effect-with-fallbacks@1.0.1(react@18.2.0): + '@emotion/use-insertion-effect-with-fallbacks@1.0.1': resolution: {integrity: sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==} peerDependencies: react: '>=16.8.0' - dependencies: - react: 18.2.0 - dev: false - /@emotion/utils@1.2.1: + '@emotion/utils@1.2.1': resolution: {integrity: sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg==} - dev: false - /@emotion/weak-memoize@0.3.1: + '@emotion/weak-memoize@0.3.1': resolution: {integrity: sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww==} - dev: false - /@eslint-community/eslint-utils@4.4.0(eslint@8.56.0): + '@eslint-community/eslint-utils@4.4.0': resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - dependencies: - eslint: 8.56.0 - eslint-visitor-keys: 3.4.3 - dev: true - /@eslint-community/regexpp@4.10.0: + '@eslint-community/regexpp@4.10.0': resolution: {integrity: sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} - dev: true - /@eslint/eslintrc@2.1.4: + '@eslint/eslintrc@2.1.4': resolution: {integrity: sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dependencies: - ajv: 6.12.6 - debug: 4.3.4 - espree: 9.6.1 - globals: 13.24.0 - ignore: 5.3.0 - import-fresh: 3.3.0 - js-yaml: 4.1.0 - minimatch: 3.1.2 - strip-json-comments: 3.1.1 - transitivePeerDependencies: - - supports-color - dev: true - /@eslint/js@8.56.0: + '@eslint/js@8.56.0': resolution: {integrity: sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dev: true - /@floating-ui/core@1.5.3: + '@floating-ui/core@1.5.3': resolution: {integrity: sha512-O0WKDOo0yhJuugCx6trZQj5jVJ9yR0ystG2JaNAemYUWce+pmM6WUEFIibnWyEJKdrDxhm75NoSRME35FNaM/Q==} - dependencies: - '@floating-ui/utils': 0.2.1 - dev: false - /@floating-ui/dom@1.5.4: + '@floating-ui/dom@1.5.4': resolution: {integrity: sha512-jByEsHIY+eEdCjnTVu+E3ephzTOzkQ8hgUfGwos+bg7NlH33Zc5uO+QHz1mrQUOgIKKDD1RtS201P9NvAfq3XQ==} - dependencies: - '@floating-ui/core': 1.5.3 - '@floating-ui/utils': 0.2.1 - dev: false - /@floating-ui/react-dom@2.0.5(react-dom@18.2.0)(react@18.2.0): + '@floating-ui/react-dom@2.0.5': resolution: {integrity: sha512-UsBK30Bg+s6+nsgblXtZmwHhgS2vmbuQK22qgt2pTQM6M3X6H1+cQcLXqgRY3ihVLcZJE6IvqDQozhsnIVqK/Q==} peerDependencies: react: '>=16.8.0' react-dom: '>=16.8.0' - dependencies: - '@floating-ui/dom': 1.5.4 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@floating-ui/utils@0.2.1: + '@floating-ui/utils@0.2.1': resolution: {integrity: sha512-9TANp6GPoMtYzQdt54kfAyMmz1+osLlXdg2ENroU7zzrtflTLrrC/lgrIfaSe+Wu0b89GKccT7vxXA0MoAIO+Q==} - dev: false - /@hookform/resolvers@3.3.4(react-hook-form@7.49.3): + '@hookform/resolvers@3.3.4': resolution: {integrity: sha512-o5cgpGOuJYrd+iMKvkttOclgwRW86EsWJZZRC23prf0uU2i48Htq4PuT73AVb9ionFyZrwYEITuOFGF+BydEtQ==} peerDependencies: react-hook-form: ^7.0.0 - dependencies: - react-hook-form: 7.49.3(react@18.2.0) - dev: false - /@humanwhocodes/config-array@0.11.13: + '@humanwhocodes/config-array@0.11.13': resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==} engines: {node: '>=10.10.0'} - dependencies: - '@humanwhocodes/object-schema': 2.0.1 - debug: 4.3.4 - minimatch: 3.1.2 - transitivePeerDependencies: - - supports-color - dev: true - /@humanwhocodes/module-importer@1.0.1: + '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} engines: {node: '>=12.22'} - dev: true - /@humanwhocodes/object-schema@2.0.1: + '@humanwhocodes/object-schema@2.0.1': resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} - dev: true - /@iconify/react@4.1.1(react@18.2.0): + '@iconify/react@4.1.1': resolution: {integrity: sha512-jed14EjvKjee8mc0eoscGxlg7mSQRkwQG3iX3cPBCO7UlOjz0DtlvTqxqEcHUJGh+z1VJ31Yhu5B9PxfO0zbdg==} peerDependencies: react: '>=16' - dependencies: - '@iconify/types': 2.0.0 - react: 18.2.0 - dev: true - /@iconify/types@2.0.0: + '@iconify/types@2.0.0': resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} - dev: true - /@isaacs/cliui@8.0.2: + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} - dependencies: - string-width: 5.1.2 - string-width-cjs: /string-width@4.2.3 - strip-ansi: 7.1.0 - strip-ansi-cjs: /strip-ansi@6.0.1 - wrap-ansi: 8.1.0 - wrap-ansi-cjs: /wrap-ansi@7.0.0 - dev: true - /@jridgewell/gen-mapping@0.3.3: + '@jridgewell/gen-mapping@0.3.3': resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==} engines: {node: '>=6.0.0'} - dependencies: - '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 - '@jridgewell/trace-mapping': 0.3.20 - dev: true - /@jridgewell/resolve-uri@3.1.1: + '@jridgewell/resolve-uri@3.1.1': resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} engines: {node: '>=6.0.0'} - dev: true - /@jridgewell/set-array@1.1.2: + '@jridgewell/set-array@1.1.2': resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} engines: {node: '>=6.0.0'} - dev: true - /@jridgewell/sourcemap-codec@1.4.15: + '@jridgewell/sourcemap-codec@1.4.15': resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} - /@jridgewell/trace-mapping@0.3.20: + '@jridgewell/trace-mapping@0.3.20': resolution: {integrity: sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==} - dependencies: - '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.4.15 - dev: true - /@mui/base@5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0): + '@mui/base@5.0.0-beta.30': resolution: {integrity: sha512-dc38W4W3K42atE9nSaOeoJ7/x9wGIfawdwC/UmMxMLlZ1iSsITQ8dQJaTATCbn98YvYPINK/EH541YA5enQIPQ==} engines: {node: '>=12.0.0'} peerDependencies: @@ -541,24 +352,11 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@floating-ui/react-dom': 2.0.5(react-dom@18.2.0)(react@18.2.0) - '@mui/types': 7.2.12(@types/react@18.2.47) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@popperjs/core': 2.11.8 - '@types/react': 18.2.47 - clsx: 2.1.0 - prop-types: 15.8.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@mui/core-downloads-tracker@5.15.3: + '@mui/core-downloads-tracker@5.15.3': resolution: {integrity: sha512-sWeihiVyxdJjpLkp8SHkTy9kt2M/o11M60G1MzwljGL2BXdM3Ktzqv5QaQHdi00y7Y1ulvtI3GOSxP2xU8mQJw==} - dev: false - /@mui/icons-material@5.15.11(@mui/material@5.15.3)(@types/react@18.2.47)(react@18.2.0): + '@mui/icons-material@5.15.11': resolution: {integrity: sha512-R5ZoQqnKpd+5Ew7mBygTFLxgYsQHPhgR3TDXSgIHYIjGzYuyPLmGLSdcPUoMdi6kxiYqHlpPj4NJxlbaFD0UHA==} engines: {node: '>=12.0.0'} peerDependencies: @@ -568,14 +366,8 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.24.0 - '@mui/material': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@types/react': 18.2.47 - react: 18.2.0 - dev: false - /@mui/lab@5.0.0-alpha.159(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@mui/material@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0): + '@mui/lab@5.0.0-alpha.159': resolution: {integrity: sha512-42Y8nf2/mDgYSLOw6PhOfHNV6P7tPcQkQEL0DTbY7a+gc+hXDsyVEzBMYST1MrV64EHTH68msfQm+k3CvLON/g==} engines: {node: '>=12.0.0'} peerDependencies: @@ -592,23 +384,8 @@ packages: optional: true '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/material': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/system': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react@18.2.0) - '@mui/types': 7.2.12(@types/react@18.2.47) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@types/react': 18.2.47 - clsx: 2.1.0 - prop-types: 15.8.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@mui/material@5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0): + '@mui/material@5.15.3': resolution: {integrity: sha512-DODBBMouyq1B5f3YkEWL9vO8pGCxuEGqtfpltF6peMJzz/78tJFyLQsDas9MNLC/8AdFu2BQdkK7wox5UBPTAA==} engines: {node: '>=12.0.0'} peerDependencies: @@ -624,27 +401,8 @@ packages: optional: true '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/core-downloads-tracker': 5.15.3 - '@mui/system': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react@18.2.0) - '@mui/types': 7.2.12(@types/react@18.2.47) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@types/react': 18.2.47 - '@types/react-transition-group': 4.4.10 - clsx: 2.1.0 - csstype: 3.1.3 - prop-types: 15.8.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - react-is: 18.2.0 - react-transition-group: 4.4.5(react-dom@18.2.0)(react@18.2.0) - dev: false - /@mui/private-theming@5.15.3(@types/react@18.2.47)(react@18.2.0): + '@mui/private-theming@5.15.3': resolution: {integrity: sha512-Q79MhVMmywC1l5bMsMZq5PsIudr1MNPJnx9/EqdMP0vpz5iNvFpnLmxsD7d8/hqTWgFAljI+LH3jX8MxlZH9Gw==} engines: {node: '>=12.0.0'} peerDependencies: @@ -653,15 +411,8 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@types/react': 18.2.47 - prop-types: 15.8.1 - react: 18.2.0 - dev: false - /@mui/styled-engine@5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(react@18.2.0): + '@mui/styled-engine@5.15.3': resolution: {integrity: sha512-+d5XZCTeemOO/vBfWGEeHgTm8fjU1Psdgm+xAw+uegycO2EnoA/EfGSaG5UwZ6g3b66y48Mkxi35AggShMr88w==} engines: {node: '>=12.0.0'} peerDependencies: @@ -673,17 +424,8 @@ packages: optional: true '@emotion/styled': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/cache': 11.11.0 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - csstype: 3.1.3 - prop-types: 15.8.1 - react: 18.2.0 - dev: false - /@mui/system@5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react@18.2.0): + '@mui/system@5.15.3': resolution: {integrity: sha512-ewVU4eRgo4VfNMGpO61cKlfWmH7l9s6rA8EknRzuMX3DbSLfmtW2WJJg6qPwragvpPIir0Pp/AdWVSDhyNy5Tw==} engines: {node: '>=12.0.0'} peerDependencies: @@ -698,33 +440,16 @@ packages: optional: true '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - '@mui/private-theming': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@mui/styled-engine': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(react@18.2.0) - '@mui/types': 7.2.12(@types/react@18.2.47) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@types/react': 18.2.47 - clsx: 2.1.0 - csstype: 3.1.3 - prop-types: 15.8.1 - react: 18.2.0 - dev: false - /@mui/types@7.2.12(@types/react@18.2.47): + '@mui/types@7.2.12': resolution: {integrity: sha512-3kaHiNm9khCAo0pVe0RenketDSFoZGAlVZ4zDjB/QNZV0XiCj+sh1zkX0VVhQPgYJDlBEzAag+MHJ1tU3vf0Zw==} peerDependencies: '@types/react': ^17.0.0 || ^18.0.0 peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@types/react': 18.2.47 - dev: false - /@mui/utils@5.15.3(@types/react@18.2.47)(react@18.2.0): + '@mui/utils@5.15.3': resolution: {integrity: sha512-mT3LiSt9tZWCdx1pl7q4Q5tNo6gdZbvJel286ZHGuj6LQQXjWNAh8qiF9d+LogvNUI+D7eLkTnj605d1zoazfg==} engines: {node: '>=12.0.0'} peerDependencies: @@ -733,16 +458,8 @@ packages: peerDependenciesMeta: '@types/react': optional: true - dependencies: - '@babel/runtime': 7.23.8 - '@types/prop-types': 15.7.11 - '@types/react': 18.2.47 - prop-types: 15.8.1 - react: 18.2.0 - react-is: 18.2.0 - dev: false - /@mui/x-data-grid@6.18.7(@mui/material@5.15.3)(@mui/system@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0): + '@mui/x-data-grid@6.18.7': resolution: {integrity: sha512-K1A3pMUPxI4/Mt5A4vrK45fBBQK5rZvBVqRMrB5n8zX++Bj+WLWKvLTtfCmlriUtzuadr/Hl7Z+FDRXUJAx6qg==} engines: {node: '>=14.0.0'} peerDependencies: @@ -750,21 +467,8 @@ packages: '@mui/system': ^5.4.1 react: ^17.0.0 || ^18.0.0 react-dom: ^17.0.0 || ^18.0.0 - dependencies: - '@babel/runtime': 7.23.8 - '@mui/material': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/system': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react@18.2.0) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - clsx: 2.1.0 - prop-types: 15.8.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - reselect: 4.1.8 - transitivePeerDependencies: - - '@types/react' - dev: false - /@mui/x-tree-view@6.17.0(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@mui/material@5.15.3)(@mui/system@5.15.3)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0): + '@mui/x-tree-view@6.17.0': resolution: {integrity: sha512-09dc2D+Rjg2z8KOaxbUXyPi0aw7fm2jurEtV8Xw48xJ00joLWd5QJm1/v4CarEvaiyhTQzHImNqdgeJW8ZQB6g==} engines: {node: '>=14.0.0'} peerDependencies: @@ -774,255 +478,137 @@ packages: '@mui/system': ^5.8.0 react: ^17.0.0 || ^18.0.0 react-dom: ^17.0.0 || ^18.0.0 - dependencies: - '@babel/runtime': 7.23.8 - '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) - '@emotion/styled': 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.47)(react@18.2.0) - '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/material': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0) - '@mui/system': 5.15.3(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.47)(react@18.2.0) - '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) - '@types/react-transition-group': 4.4.10 - clsx: 2.1.0 - prop-types: 15.8.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - react-transition-group: 4.4.5(react-dom@18.2.0)(react@18.2.0) - transitivePeerDependencies: - - '@types/react' - dev: false - /@next/bundle-analyzer@14.0.4: + '@next/bundle-analyzer@14.0.4': resolution: {integrity: sha512-Nn2PiCkFBJBlVmpSGVNItpISws0fuc9E8AkCafBz/moRv1cfASOpFBBVzSRfWLP9BPdAhfDkb6TafN0rvs2IJQ==} - dependencies: - webpack-bundle-analyzer: 4.7.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - dev: true - /@next/env@14.0.3: + '@next/env@14.0.3': resolution: {integrity: sha512-7xRqh9nMvP5xrW4/+L0jgRRX+HoNRGnfJpD+5Wq6/13j3dsdzxO3BCXn7D3hMqsDb+vjZnJq+vI7+EtgrYZTeA==} - dev: false - /@next/eslint-plugin-next@14.0.3: + '@next/eslint-plugin-next@14.0.3': resolution: {integrity: sha512-j4K0n+DcmQYCVnSAM+UByTVfIHnYQy2ODozfQP+4RdwtRDfobrIvKq1K4Exb2koJ79HSSa7s6B2SA8T/1YR3RA==} - dependencies: - glob: 7.1.7 - dev: true - /@next/swc-darwin-arm64@14.0.3: + '@next/swc-darwin-arm64@14.0.3': resolution: {integrity: sha512-64JbSvi3nbbcEtyitNn2LEDS/hcleAFpHdykpcnrstITFlzFgB/bW0ER5/SJJwUPj+ZPY+z3e+1jAfcczRLVGw==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - requiresBuild: true - dev: false - optional: true - /@next/swc-darwin-x64@14.0.3: + '@next/swc-darwin-x64@14.0.3': resolution: {integrity: sha512-RkTf+KbAD0SgYdVn1XzqE/+sIxYGB7NLMZRn9I4Z24afrhUpVJx6L8hsRnIwxz3ERE2NFURNliPjJ2QNfnWicQ==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - requiresBuild: true - dev: false - optional: true - /@next/swc-linux-arm64-gnu@14.0.3: + '@next/swc-linux-arm64-gnu@14.0.3': resolution: {integrity: sha512-3tBWGgz7M9RKLO6sPWC6c4pAw4geujSwQ7q7Si4d6bo0l6cLs4tmO+lnSwFp1Tm3lxwfMk0SgkJT7EdwYSJvcg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - requiresBuild: true - dev: false - optional: true - /@next/swc-linux-arm64-musl@14.0.3: + '@next/swc-linux-arm64-musl@14.0.3': resolution: {integrity: sha512-v0v8Kb8j8T23jvVUWZeA2D8+izWspeyeDGNaT2/mTHWp7+37fiNfL8bmBWiOmeumXkacM/AB0XOUQvEbncSnHA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - requiresBuild: true - dev: false - optional: true - /@next/swc-linux-x64-gnu@14.0.3: + '@next/swc-linux-x64-gnu@14.0.3': resolution: {integrity: sha512-VM1aE1tJKLBwMGtyBR21yy+STfl0MapMQnNrXkxeyLs0GFv/kZqXS5Jw/TQ3TSUnbv0QPDf/X8sDXuMtSgG6eg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - requiresBuild: true - dev: false - optional: true - /@next/swc-linux-x64-musl@14.0.3: + '@next/swc-linux-x64-musl@14.0.3': resolution: {integrity: sha512-64EnmKy18MYFL5CzLaSuUn561hbO1Gk16jM/KHznYP3iCIfF9e3yULtHaMy0D8zbHfxset9LTOv6cuYKJgcOxg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - requiresBuild: true - dev: false - optional: true - /@next/swc-win32-arm64-msvc@14.0.3: + '@next/swc-win32-arm64-msvc@14.0.3': resolution: {integrity: sha512-WRDp8QrmsL1bbGtsh5GqQ/KWulmrnMBgbnb+59qNTW1kVi1nG/2ndZLkcbs2GX7NpFLlToLRMWSQXmPzQm4tog==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - requiresBuild: true - dev: false - optional: true - /@next/swc-win32-ia32-msvc@14.0.3: + '@next/swc-win32-ia32-msvc@14.0.3': resolution: {integrity: sha512-EKffQeqCrj+t6qFFhIFTRoqb2QwX1mU7iTOvMyLbYw3QtqTw9sMwjykyiMlZlrfm2a4fA84+/aeW+PMg1MjuTg==} engines: {node: '>= 10'} cpu: [ia32] os: [win32] - requiresBuild: true - dev: false - optional: true - /@next/swc-win32-x64-msvc@14.0.3: + '@next/swc-win32-x64-msvc@14.0.3': resolution: {integrity: sha512-ERhKPSJ1vQrPiwrs15Pjz/rvDHZmkmvbf/BjPN/UCOI++ODftT0GtasDPi0j+y6PPJi5HsXw+dpRaXUaw4vjuQ==} engines: {node: '>= 10'} cpu: [x64] os: [win32] - requiresBuild: true - dev: false - optional: true - /@nodelib/fs.scandir@2.1.5: + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} - dependencies: - '@nodelib/fs.stat': 2.0.5 - run-parallel: 1.2.0 - dev: true - /@nodelib/fs.stat@2.0.5: + '@nodelib/fs.stat@2.0.5': resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} engines: {node: '>= 8'} - dev: true - /@nodelib/fs.walk@1.2.8: + '@nodelib/fs.walk@1.2.8': resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} - dependencies: - '@nodelib/fs.scandir': 2.1.5 - fastq: 1.16.0 - dev: true - /@pkgjs/parseargs@0.11.0: + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} - requiresBuild: true - dev: true - optional: true - /@polka/url@1.0.0-next.24: + '@polka/url@1.0.0-next.24': resolution: {integrity: sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==} - dev: true - /@popperjs/core@2.11.8: + '@popperjs/core@2.11.8': resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==} - dev: false - /@rc-component/color-picker@1.5.1(react-dom@18.2.0)(react@18.2.0): + '@rc-component/color-picker@1.5.1': resolution: {integrity: sha512-onyAFhWKXuG4P162xE+7IgaJkPkwM94XlOYnQuu69XdXWMfxpeFi6tpJBsieIMV7EnyLV5J3lDzdLiFeK0iEBA==} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - '@ctrl/tinycolor': 3.6.1 - classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@rc-component/context@1.4.0(react-dom@18.2.0)(react@18.2.0): + '@rc-component/context@1.4.0': resolution: {integrity: sha512-kFcNxg9oLRMoL3qki0OMxK+7g5mypjgaaJp/pkOis/6rVxma9nJBF/8kCIuTYHUQNr0ii7MxqE33wirPZLJQ2w==} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@rc-component/mini-decimal@1.1.0: + '@rc-component/mini-decimal@1.1.0': resolution: {integrity: sha512-jS4E7T9Li2GuYwI6PyiVXmxTiM6b07rlD9Ge8uGZSCz3WlzcG5ZK7g5bbuKNeZ9pgUuPK/5guV781ujdVpm4HQ==} engines: {node: '>=8.x'} - dependencies: - '@babel/runtime': 7.23.8 - dev: false - /@rc-component/mutate-observer@1.1.0(react-dom@18.2.0)(react@18.2.0): + '@rc-component/mutate-observer@1.1.0': resolution: {integrity: sha512-QjrOsDXQusNwGZPf4/qRQasg7UFEj06XiCJ8iuiq/Io7CrHrgVi6Uuetw60WAMG1799v+aM8kyc+1L/GBbHSlw==} engines: {node: '>=8.x'} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@rc-component/portal@1.1.2(react-dom@18.2.0)(react@18.2.0): + '@rc-component/portal@1.1.2': resolution: {integrity: sha512-6f813C0IsasTZms08kfA8kPAGxbbkYToa8ALaiDIGGECU4i9hj8Plgbx0sNJDrey3EtHO30hmdaxtT0138xZcg==} engines: {node: '>=8.x'} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@rc-component/tour@1.12.3(react-dom@18.2.0)(react@18.2.0): + '@rc-component/tour@1.12.3': resolution: {integrity: sha512-U4mf1FiUxGCwrX4ed8op77Y8VKur+8Y/61ylxtqGbcSoh1EBC7bWd/DkLu0ClTUrKZInqEi1FL7YgFtnT90vHA==} engines: {node: '>=8.x'} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - '@rc-component/portal': 1.1.2(react-dom@18.2.0)(react@18.2.0) - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) - classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@rc-component/trigger@1.18.3(react-dom@18.2.0)(react@18.2.0): + '@rc-component/trigger@1.18.3': resolution: {integrity: sha512-Ksr25pXreYe1gX6ayZ1jLrOrl9OAUHUqnuhEx6MeHnNa1zVM5Y2Aj3Q35UrER0ns8D2cJYtmJtVli+i+4eKrvA==} engines: {node: '>=8.x'} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@babel/runtime': 7.23.8 - '@rc-component/portal': 1.1.2(react-dom@18.2.0)(react@18.2.0) - classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - dev: false - /@reduxjs/toolkit@1.9.7(react-redux@8.1.3)(react@18.2.0): + '@reduxjs/toolkit@1.9.7': resolution: {integrity: sha512-t7v8ZPxhhKgOKtU+uyJT13lu4vL7az5aFi4IdoDs/eS548edn2M8Ik9h8fxgvMjGoAUVFSt6ZC1P5cWmQ014QQ==} peerDependencies: react: ^16.9.0 || ^17.0.0 || ^18 @@ -1032,88 +618,53 @@ packages: optional: true react-redux: optional: true - dependencies: - immer: 9.0.21 - react: 18.2.0 - react-redux: 8.1.3(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0)(redux@4.2.1) - redux: 4.2.1 - redux-thunk: 2.4.2(redux@4.2.1) - reselect: 4.1.8 - dev: false - /@rushstack/eslint-patch@1.6.1: + '@rushstack/eslint-patch@1.6.1': resolution: {integrity: sha512-UY+FGM/2jjMkzQLn8pxcHGMaVLh9aEitG3zY2CiY7XHdLiz3bZOwa6oDxNqEMv7zZkV+cj5DOdz0cQ1BP5Hjgw==} - dev: true - /@swc/helpers@0.5.2: + '@swc/helpers@0.5.2': resolution: {integrity: sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==} - dependencies: - tslib: 2.6.2 - dev: false - /@types/hoist-non-react-statics@3.3.5: + '@types/hoist-non-react-statics@3.3.5': resolution: {integrity: sha512-SbcrWzkKBw2cdwRTwQAswfpB9g9LJWfjtUeW/jvNwbhC8cpmmNYVePa+ncbUe0rGTQ7G3Ff6mYUN2VMfLVr+Sg==} - dependencies: - '@types/react': 18.2.47 - hoist-non-react-statics: 3.3.2 - dev: false - /@types/js-cookie@2.2.7: + '@types/js-cookie@2.2.7': resolution: {integrity: sha512-aLkWa0C0vO5b4Sr798E26QgOkss68Un0bLjs7u9qxzPT5CG+8DuNTffWES58YzJs3hrVAOs1wonycqEBqNJubA==} - dev: false - /@types/json5@0.0.29: + '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} - dev: true - /@types/lodash-es@4.17.12: + '@types/lodash-es@4.17.12': resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} - dependencies: - '@types/lodash': 4.14.202 - dev: true - /@types/lodash@4.14.202: + '@types/lodash@4.14.202': resolution: {integrity: sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==} - dev: true - /@types/node@20.10.7: + '@types/node@20.10.7': resolution: {integrity: sha512-fRbIKb8C/Y2lXxB5eVMj4IU7xpdox0Lh8bUPEdtLysaylsml1hOOx1+STloRs/B9nf7C6kPRmmg/V7aQW7usNg==} - dependencies: - undici-types: 5.26.5 - dev: true - /@types/parse-json@4.0.2: + '@types/parse-json@4.0.2': resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} - dev: false - /@types/prop-types@15.7.11: + '@types/prop-types@15.7.11': resolution: {integrity: sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==} - /@types/qs@6.9.11: + '@types/qs@6.9.11': resolution: {integrity: sha512-oGk0gmhnEJK4Yyk+oI7EfXsLayXatCWPHary1MtcmbAifkobT9cM9yutG/hZKIseOU0MqbIwQ/u2nn/Gb+ltuQ==} - dev: true - /@types/react-transition-group@4.4.10: + '@types/react-transition-group@4.4.10': resolution: {integrity: sha512-hT/+s0VQs2ojCX823m60m5f0sL5idt9SO6Tj6Dg+rdphGPIeJbJ6CxvBYkgkGKrYeDjvIpKTR38UzmtHJOGW3Q==} - dependencies: - '@types/react': 18.2.47 - dev: false - /@types/react@18.2.47: + '@types/react@18.2.47': resolution: {integrity: sha512-xquNkkOirwyCgoClNk85BjP+aqnIS+ckAJ8i37gAbDs14jfW/J23f2GItAf33oiUPQnqNMALiFeoM9Y5mbjpVQ==} - dependencies: - '@types/prop-types': 15.7.11 - '@types/scheduler': 0.16.8 - csstype: 3.1.3 - /@types/scheduler@0.16.8: + '@types/scheduler@0.16.8': resolution: {integrity: sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==} - /@types/use-sync-external-store@0.0.3: + '@types/use-sync-external-store@0.0.3': resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==} - dev: false - /@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3): + '@typescript-eslint/parser@6.18.1': resolution: {integrity: sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA==} engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: @@ -1122,32 +673,16 @@ packages: peerDependenciesMeta: typescript: optional: true - dependencies: - '@typescript-eslint/scope-manager': 6.18.1 - '@typescript-eslint/types': 6.18.1 - '@typescript-eslint/typescript-estree': 6.18.1(typescript@5.3.3) - '@typescript-eslint/visitor-keys': 6.18.1 - debug: 4.3.4 - eslint: 8.56.0 - typescript: 5.3.3 - transitivePeerDependencies: - - supports-color - dev: true - /@typescript-eslint/scope-manager@6.18.1: + '@typescript-eslint/scope-manager@6.18.1': resolution: {integrity: sha512-BgdBwXPFmZzaZUuw6wKiHKIovms97a7eTImjkXCZE04TGHysG+0hDQPmygyvgtkoB/aOQwSM/nWv3LzrOIQOBw==} engines: {node: ^16.0.0 || >=18.0.0} - dependencies: - '@typescript-eslint/types': 6.18.1 - '@typescript-eslint/visitor-keys': 6.18.1 - dev: true - /@typescript-eslint/types@6.18.1: + '@typescript-eslint/types@6.18.1': resolution: {integrity: sha512-4TuMAe+tc5oA7wwfqMtB0Y5OrREPF1GeJBAjqwgZh1lEMH5PJQgWgHGfYufVB51LtjD+peZylmeyxUXPfENLCw==} engines: {node: ^16.0.0 || >=18.0.0} - dev: true - /@typescript-eslint/typescript-estree@6.18.1(typescript@5.3.3): + '@typescript-eslint/typescript-estree@6.18.1': resolution: {integrity: sha512-fv9B94UAhywPRhUeeV/v+3SBDvcPiLxRZJw/xZeeGgRLQZ6rLMG+8krrJUyIf6s1ecWTzlsbp0rlw7n9sjufHA==} engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: @@ -1155,564 +690,286 @@ packages: peerDependenciesMeta: typescript: optional: true - dependencies: - '@typescript-eslint/types': 6.18.1 - '@typescript-eslint/visitor-keys': 6.18.1 - debug: 4.3.4 - globby: 11.1.0 - is-glob: 4.0.3 - minimatch: 9.0.3 - semver: 7.5.4 - ts-api-utils: 1.0.3(typescript@5.3.3) - typescript: 5.3.3 - transitivePeerDependencies: - - supports-color - dev: true - /@typescript-eslint/visitor-keys@6.18.1: + '@typescript-eslint/visitor-keys@6.18.1': resolution: {integrity: sha512-/kvt0C5lRqGoCfsbmm7/CwMqoSkY3zzHLIjdhHZQW3VFrnz7ATecOHR7nb7V+xn4286MBxfnQfQhAmCI0u+bJA==} engines: {node: ^16.0.0 || >=18.0.0} - dependencies: - '@typescript-eslint/types': 6.18.1 - eslint-visitor-keys: 3.4.3 - dev: true - /@ungap/structured-clone@1.2.0: + '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} - dev: true - /@xobotyi/scrollbar-width@1.9.5: + '@xobotyi/scrollbar-width@1.9.5': resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} - dev: false - /acorn-jsx@5.3.2(acorn@8.11.3): + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 - dependencies: - acorn: 8.11.3 - dev: true - /acorn-walk@8.3.1: + acorn-walk@8.3.1: resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} engines: {node: '>=0.4.0'} - dev: true - /acorn@8.11.3: + acorn@8.11.3: resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} engines: {node: '>=0.4.0'} hasBin: true - dev: true - /ajv@6.12.6: + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - dependencies: - fast-deep-equal: 3.1.3 - fast-json-stable-stringify: 2.1.0 - json-schema-traverse: 0.4.1 - uri-js: 4.4.1 - dev: true - /ansi-regex@5.0.1: + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} - dev: true - /ansi-regex@6.0.1: + ansi-regex@6.0.1: resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} engines: {node: '>=12'} - dev: true - /ansi-styles@3.2.1: + ansi-styles@3.2.1: resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} engines: {node: '>=4'} - dependencies: - color-convert: 1.9.3 - dev: false - /ansi-styles@4.3.0: + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} - dependencies: - color-convert: 2.0.1 - dev: true - /ansi-styles@6.2.1: + ansi-styles@6.2.1: resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} engines: {node: '>=12'} - dev: true - /antd@5.13.3(react-dom@18.2.0)(react@18.2.0): + antd@5.13.3: resolution: {integrity: sha512-phQJa4ezs6e2AnWRxbKVan9fvmURwntAfI+wDRRSP7spPY6t3afjvWfAcVp0Ekb1EPzvF/jUr64j3RMQQYWHVw==} peerDependencies: react: '>=16.9.0' react-dom: '>=16.9.0' - dependencies: - '@ant-design/colors': 7.0.2 - '@ant-design/cssinjs': 1.18.4(react-dom@18.2.0)(react@18.2.0) - '@ant-design/icons': 5.3.0(react-dom@18.2.0)(react@18.2.0) - '@ant-design/react-slick': 1.0.2(react@18.2.0) - '@ctrl/tinycolor': 3.6.1 - '@rc-component/color-picker': 1.5.1(react-dom@18.2.0)(react@18.2.0) - '@rc-component/mutate-observer': 1.1.0(react-dom@18.2.0)(react@18.2.0) - '@rc-component/tour': 1.12.3(react-dom@18.2.0)(react@18.2.0) - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) - classnames: 2.5.1 - copy-to-clipboard: 3.3.3 - dayjs: 1.11.10 - qrcode.react: 3.1.0(react@18.2.0) - rc-cascader: 3.21.2(react-dom@18.2.0)(react@18.2.0) - rc-checkbox: 3.1.0(react-dom@18.2.0)(react@18.2.0) - rc-collapse: 3.7.2(react-dom@18.2.0)(react@18.2.0) - rc-dialog: 9.3.4(react-dom@18.2.0)(react@18.2.0) - rc-drawer: 7.0.0(react-dom@18.2.0)(react@18.2.0) - rc-dropdown: 4.1.0(react-dom@18.2.0)(react@18.2.0) - rc-field-form: 1.41.0(react-dom@18.2.0)(react@18.2.0) - rc-image: 7.5.1(react-dom@18.2.0)(react@18.2.0) - rc-input: 1.4.3(react-dom@18.2.0)(react@18.2.0) - rc-input-number: 8.6.1(react-dom@18.2.0)(react@18.2.0) - rc-mentions: 2.10.1(react-dom@18.2.0)(react@18.2.0) - rc-menu: 9.12.4(react-dom@18.2.0)(react@18.2.0) - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-notification: 5.3.0(react-dom@18.2.0)(react@18.2.0) - rc-pagination: 4.0.4(react-dom@18.2.0)(react@18.2.0) - rc-picker: 3.14.6(dayjs@1.11.10)(react-dom@18.2.0)(react@18.2.0) - rc-progress: 3.5.1(react-dom@18.2.0)(react@18.2.0) - rc-rate: 2.12.0(react-dom@18.2.0)(react@18.2.0) - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-segmented: 2.2.2(react-dom@18.2.0)(react@18.2.0) - rc-select: 14.11.0(react-dom@18.2.0)(react@18.2.0) - rc-slider: 10.5.0(react-dom@18.2.0)(react@18.2.0) - rc-steps: 6.0.1(react-dom@18.2.0)(react@18.2.0) - rc-switch: 4.1.0(react-dom@18.2.0)(react@18.2.0) - rc-table: 7.37.0(react-dom@18.2.0)(react@18.2.0) - rc-tabs: 14.0.0(react-dom@18.2.0)(react@18.2.0) - rc-textarea: 1.6.3(react-dom@18.2.0)(react@18.2.0) - rc-tooltip: 6.1.3(react-dom@18.2.0)(react@18.2.0) - rc-tree: 5.8.5(react-dom@18.2.0)(react@18.2.0) - rc-tree-select: 5.17.0(react-dom@18.2.0)(react@18.2.0) - rc-upload: 4.5.2(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - scroll-into-view-if-needed: 3.1.0 - throttle-debounce: 5.0.0 - transitivePeerDependencies: - - date-fns - - luxon - - moment - dev: false - /any-promise@1.3.0: + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} - dev: true - /anymatch@3.1.3: + anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} - dependencies: - normalize-path: 3.0.0 - picomatch: 2.3.1 - dev: true - /arg@5.0.2: + arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} - dev: true - /argparse@2.0.1: + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} - dev: true - /aria-query@5.3.0: + aria-query@5.3.0: resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==} - dependencies: - dequal: 2.0.3 - dev: true - /array-buffer-byte-length@1.0.0: + array-buffer-byte-length@1.0.0: resolution: {integrity: sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==} - dependencies: - call-bind: 1.0.5 - is-array-buffer: 3.0.2 - dev: true - /array-includes@3.1.7: + array-includes@3.1.7: resolution: {integrity: sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==} engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - get-intrinsic: 1.2.2 - is-string: 1.0.7 - dev: true - /array-tree-filter@2.1.0: + array-tree-filter@2.1.0: resolution: {integrity: sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==} - dev: false - /array-union@2.1.0: + array-union@2.1.0: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} - dev: true - /array.prototype.findlastindex@1.2.3: + array.prototype.findlastindex@1.2.3: resolution: {integrity: sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA==} engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - es-shim-unscopables: 1.0.2 - get-intrinsic: 1.2.2 - dev: true - /array.prototype.flat@1.3.2: + array.prototype.flat@1.3.2: resolution: {integrity: sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==} engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - es-shim-unscopables: 1.0.2 - dev: true - /array.prototype.flatmap@1.3.2: + array.prototype.flatmap@1.3.2: resolution: {integrity: sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==} engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - es-shim-unscopables: 1.0.2 - dev: true - /array.prototype.tosorted@1.1.2: + array.prototype.tosorted@1.1.2: resolution: {integrity: sha512-HuQCHOlk1Weat5jzStICBCd83NxiIMwqDg/dHEsoefabn/hJRj5pVdWcPUSpRrwhwxZOsQassMpgN/xRYFBMIg==} - dependencies: - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - es-shim-unscopables: 1.0.2 - get-intrinsic: 1.2.2 - dev: true - /arraybuffer.prototype.slice@1.0.2: + arraybuffer.prototype.slice@1.0.2: resolution: {integrity: sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==} engines: {node: '>= 0.4'} - dependencies: - array-buffer-byte-length: 1.0.0 - call-bind: 1.0.5 - define-properties: 1.2.1 - es-abstract: 1.22.3 - get-intrinsic: 1.2.2 - is-array-buffer: 3.0.2 - is-shared-array-buffer: 1.0.2 - dev: true - /ast-types-flow@0.0.8: + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} - dev: true - /async-validator@4.2.5: + async-validator@4.2.5: resolution: {integrity: sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==} - dev: false - /asynciterator.prototype@1.0.0: + asynciterator.prototype@1.0.0: resolution: {integrity: sha512-wwHYEIS0Q80f5mosx3L/dfG5t5rjEa9Ft51GTaNt862EnpyGHpgz2RkZvLPp1oF5TnAiTohkEKVEu8pQPJI7Vg==} - dependencies: - has-symbols: 1.0.3 - dev: true - /asynckit@0.4.0: + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: false - /autoprefixer@10.4.16(postcss@8.4.33): + autoprefixer@10.4.16: resolution: {integrity: sha512-7vd3UC6xKp0HLfua5IjZlcXvGAGy7cBAXTg2lyQ/8WpNhd6SiZ8Be+xm3FyBSYJx5GKcpRCzBh7RH4/0dnY+uQ==} engines: {node: ^10 || ^12 || >=14} hasBin: true peerDependencies: postcss: ^8.1.0 - dependencies: - browserslist: 4.22.2 - caniuse-lite: 1.0.30001576 - fraction.js: 4.3.7 - normalize-range: 0.1.2 - picocolors: 1.0.0 - postcss: 8.4.33 - postcss-value-parser: 4.2.0 - dev: true - /available-typed-arrays@1.0.5: + available-typed-arrays@1.0.5: resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} engines: {node: '>= 0.4'} - dev: true - /axe-core@4.7.0: + axe-core@4.7.0: resolution: {integrity: sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==} engines: {node: '>=4'} - dev: true - /axios@1.6.8: + axios@1.6.8: resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - dev: false - /axobject-query@3.2.1: + axobject-query@3.2.1: resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} - dependencies: - dequal: 2.0.3 - dev: true - /babel-plugin-macros@3.1.0: + babel-plugin-macros@3.1.0: resolution: {integrity: sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==} engines: {node: '>=10', npm: '>=6'} - dependencies: - '@babel/runtime': 7.23.8 - cosmiconfig: 7.1.0 - resolve: 1.22.8 - dev: false - /balanced-match@1.0.2: + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - dev: true - /binary-extensions@2.2.0: + binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} - dev: true - /brace-expansion@1.1.11: + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} - dependencies: - balanced-match: 1.0.2 - concat-map: 0.0.1 - dev: true - /brace-expansion@2.0.1: + brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} - dependencies: - balanced-match: 1.0.2 - dev: true - /braces@3.0.2: + braces@3.0.2: resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} engines: {node: '>=8'} - dependencies: - fill-range: 7.0.1 - dev: true - /browserslist@4.22.2: + browserslist@4.22.2: resolution: {integrity: sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true - dependencies: - caniuse-lite: 1.0.30001576 - electron-to-chromium: 1.4.625 - node-releases: 2.0.14 - update-browserslist-db: 1.0.13(browserslist@4.22.2) - dev: true - /busboy@1.6.0: + busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} - dependencies: - streamsearch: 1.1.0 - dev: false - /call-bind@1.0.5: + call-bind@1.0.5: resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==} - dependencies: - function-bind: 1.1.2 - get-intrinsic: 1.2.2 - set-function-length: 1.1.1 - /callsites@3.1.0: + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - /camelcase-css@2.0.1: + camelcase-css@2.0.1: resolution: {integrity: sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==} engines: {node: '>= 6'} - dev: true - /caniuse-lite@1.0.30001576: + caniuse-lite@1.0.30001576: resolution: {integrity: sha512-ff5BdakGe2P3SQsMsiqmt1Lc8221NR1VzHj5jXN5vBny9A6fpze94HiVV/n7XRosOlsShJcvMv5mdnpjOGCEgg==} - /chalk@2.4.2: + chalk@2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - dev: false - /chalk@4.1.2: + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} - dependencies: - ansi-styles: 4.3.0 - supports-color: 7.2.0 - dev: true - /chokidar@3.5.3: + chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} - dependencies: - anymatch: 3.1.3 - braces: 3.0.2 - glob-parent: 5.1.2 - is-binary-path: 2.1.0 - is-glob: 4.0.3 - normalize-path: 3.0.0 - readdirp: 3.6.0 - optionalDependencies: - fsevents: 2.3.3 - dev: true - /chroma-js@2.4.2: + chroma-js@2.4.2: resolution: {integrity: sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A==} - dev: false - /classnames@2.5.1: + classnames@2.5.1: resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} - dev: false - /client-only@0.0.1: + client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} - dev: false - /clsx@2.1.0: + clsx@2.1.0: resolution: {integrity: sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==} engines: {node: '>=6'} - dev: false - /color-convert@1.9.3: + color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - dependencies: - color-name: 1.1.3 - dev: false - /color-convert@2.0.1: + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} - dependencies: - color-name: 1.1.4 - dev: true - /color-name@1.1.3: + color-name@1.1.3: resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - dev: false - /color-name@1.1.4: + color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - dev: true - /combined-stream@1.0.8: + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: false - /commander@4.1.1: + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} - dev: true - /commander@7.2.0: + commander@7.2.0: resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} engines: {node: '>= 10'} - dev: true - /compute-scroll-into-view@3.1.0: + compute-scroll-into-view@3.1.0: resolution: {integrity: sha512-rj8l8pD4bJ1nx+dAkMhV1xB5RuZEyVysfxJqB1pRchh1KVvwOv9b7CGB8ZfjTImVv2oF+sYMUkMZq6Na5Ftmbg==} - dev: false - /concat-map@0.0.1: + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - dev: true - /convert-source-map@1.9.0: + convert-source-map@1.9.0: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} - dev: false - /copy-to-clipboard@3.3.3: + copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} - dependencies: - toggle-selection: 1.0.6 - dev: false - /cosmiconfig@7.1.0: + cosmiconfig@7.1.0: resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==} engines: {node: '>=10'} - dependencies: - '@types/parse-json': 4.0.2 - import-fresh: 3.3.0 - parse-json: 5.2.0 - path-type: 4.0.0 - yaml: 1.10.2 - dev: false - /cross-spawn@7.0.3: + cross-spawn@7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} - dependencies: - path-key: 3.1.1 - shebang-command: 2.0.0 - which: 2.0.2 - dev: true - /css-in-js-utils@3.1.0: + css-in-js-utils@3.1.0: resolution: {integrity: sha512-fJAcud6B3rRu+KHYk+Bwf+WFL2MDCJJ1XG9x137tJQ0xYxor7XziQtuGFbWNdqrvF4Tk26O3H73nfVqXt/fW1A==} - dependencies: - hyphenate-style-name: 1.0.4 - dev: false - /css-tree@1.1.3: + css-tree@1.1.3: resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==} engines: {node: '>=8.0.0'} - dependencies: - mdn-data: 2.0.14 - source-map: 0.6.1 - dev: false - /cssesc@3.0.0: + cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} hasBin: true - dev: true - /csstype@3.1.3: + csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} - /damerau-levenshtein@1.0.8: + damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} - dev: true - /dayjs@1.11.10: + dayjs@1.11.10: resolution: {integrity: sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==} - dev: false - /debug@3.2.7: + debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: supports-color: '*' peerDependenciesMeta: supports-color: optional: true - dependencies: - ms: 2.1.3 - dev: true - /debug@4.3.4: + debug@4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} peerDependencies: @@ -1720,129 +977,2628 @@ packages: peerDependenciesMeta: supports-color: optional: true - dependencies: - ms: 2.1.2 - dev: true - /deep-is@0.1.4: + deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} - dev: true - /define-data-property@1.1.1: + define-data-property@1.1.1: resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==} engines: {node: '>= 0.4'} - dependencies: - get-intrinsic: 1.2.2 - gopd: 1.0.1 - has-property-descriptors: 1.0.1 - /define-properties@1.2.1: + define-properties@1.2.1: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - dependencies: - define-data-property: 1.1.1 - has-property-descriptors: 1.0.1 - object-keys: 1.1.1 - dev: true - /delayed-stream@1.0.0: + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: false - /dequal@2.0.3: + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} - dev: true - /didyoumean@1.2.2: + didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} - dev: true - /dir-glob@3.0.1: + dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} - dependencies: - path-type: 4.0.0 - dev: true - /dlv@1.1.3: + dlv@1.1.3: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} - dev: true - /doctrine@2.1.0: + doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} - dependencies: - esutils: 2.0.3 - dev: true - /doctrine@3.0.0: + doctrine@3.0.0: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} - dependencies: + + dom-helpers@5.2.1: + resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} + + duplexer@0.1.2: + resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + electron-to-chromium@1.4.625: + resolution: {integrity: sha512-DENMhh3MFgaPDoXWrVIqSPInQoLImywfCwrSmVl3cf9QHzoZSiutHwGaB/Ql3VkqcQV30rzgdM+BjKqBAJxo5Q==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + enhanced-resolve@5.15.0: + resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==} + engines: {node: '>=10.13.0'} + + env-cmd@10.1.0: + resolution: {integrity: sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==} + engines: {node: '>=8.0.0'} + hasBin: true + + error-ex@1.3.2: + resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + + error-stack-parser@2.1.4: + resolution: {integrity: sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==} + + es-abstract@1.22.3: + resolution: {integrity: sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==} + engines: {node: '>= 0.4'} + + es-iterator-helpers@1.0.15: + resolution: {integrity: sha512-GhoY8uYqd6iwUl2kgjTm4CZAf6oo5mHK7BPqx3rKgx893YSsy0LGHV6gfqqQvZt/8xM8xeOnfXBCfqclMKkJ5g==} + + es-set-tostringtag@2.0.2: + resolution: {integrity: sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==} + engines: {node: '>= 0.4'} + + es-shim-unscopables@1.0.2: + resolution: {integrity: sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==} + + es-to-primitive@1.2.1: + resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} + engines: {node: '>= 0.4'} + + escalade@3.1.1: + resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} + engines: {node: '>=6'} + + escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-config-next@14.0.3: + resolution: {integrity: sha512-IKPhpLdpSUyKofmsXUfrvBC49JMUTdeaD8ZIH4v9Vk0sC1X6URTuTJCLtA0Vwuj7V/CQh0oISuSTvNn5//Buew==} + peerDependencies: + eslint: ^7.23.0 || ^8.0.0 + typescript: '>=3.3.1' + peerDependenciesMeta: + typescript: + optional: true + + eslint-config-prettier@9.1.0: + resolution: {integrity: sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==} + hasBin: true + peerDependencies: + eslint: '>=7.0.0' + + eslint-import-resolver-node@0.3.9: + resolution: {integrity: sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==} + + eslint-import-resolver-typescript@3.6.1: + resolution: {integrity: sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==} + engines: {node: ^14.18.0 || >=16.0.0} + peerDependencies: + eslint: '*' + eslint-plugin-import: '*' + + eslint-module-utils@2.8.0: + resolution: {integrity: sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==} + engines: {node: '>=4'} + peerDependencies: + '@typescript-eslint/parser': '*' + eslint: '*' + eslint-import-resolver-node: '*' + eslint-import-resolver-typescript: '*' + eslint-import-resolver-webpack: '*' + peerDependenciesMeta: + '@typescript-eslint/parser': + optional: true + eslint: + optional: true + eslint-import-resolver-node: + optional: true + eslint-import-resolver-typescript: + optional: true + eslint-import-resolver-webpack: + optional: true + + eslint-plugin-import@2.29.1: + resolution: {integrity: sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==} + engines: {node: '>=4'} + peerDependencies: + '@typescript-eslint/parser': '*' + eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 + peerDependenciesMeta: + '@typescript-eslint/parser': + optional: true + + eslint-plugin-jsx-a11y@6.8.0: + resolution: {integrity: sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA==} + engines: {node: '>=4.0'} + peerDependencies: + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + + eslint-plugin-react-hooks@4.6.0: + resolution: {integrity: sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==} + engines: {node: '>=10'} + peerDependencies: + eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 + + eslint-plugin-react@7.33.2: + resolution: {integrity: sha512-73QQMKALArI8/7xGLNI/3LylrEYrlKZSb5C9+q3OtOewTnMQi5cT+aE9E41sLCmli3I9PGGmD1yiZydyo4FEPw==} + engines: {node: '>=4'} + peerDependencies: + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + + eslint-scope@7.2.2: + resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint@8.56.0: + resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + hasBin: true + + espree@9.6.1: + resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + esquery@1.5.0: + resolution: {integrity: sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-glob@3.3.2: + resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} + engines: {node: '>=8.6.0'} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fast-loops@1.1.3: + resolution: {integrity: sha512-8EZzEP0eKkEEVX+drtd9mtuQ+/QrlfW/5MlwcwK5Nds6EkZ/tRzEexkzUY2mIssnAyVLT+TKHuRXmFNNXYUd6g==} + + fast-shallow-equal@1.0.0: + resolution: {integrity: sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==} + + fastest-stable-stringify@2.0.2: + resolution: {integrity: sha512-bijHueCGd0LqqNK9b5oCMHc0MluJAx0cwqASgbWMvkO01lCYgIhacVRLcaDz3QnyYIRNJRDwMb41VuT6pHJ91Q==} + + fastq@1.16.0: + resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} + + file-entry-cache@6.0.1: + resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} + engines: {node: ^10.12.0 || >=12.0.0} + + fill-range@7.0.1: + resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} + engines: {node: '>=8'} + + find-root@1.1.0: + resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@3.2.0: + resolution: {integrity: sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==} + engines: {node: ^10.12.0 || >=12.0.0} + + flatted@3.2.9: + resolution: {integrity: sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==} + + follow-redirects@1.15.6: + resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + for-each@0.3.3: + resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + + foreground-child@3.1.1: + resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} + engines: {node: '>=14'} + + form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + + fraction.js@4.3.7: + resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + + fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + function.prototype.name@1.1.6: + resolution: {integrity: sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==} + engines: {node: '>= 0.4'} + + functions-have-names@1.2.3: + resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + + get-intrinsic@1.2.2: + resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + + get-symbol-description@1.0.0: + resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} + engines: {node: '>= 0.4'} + + get-tsconfig@4.7.2: + resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + + glob-parent@5.1.2: + resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} + engines: {node: '>= 6'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + glob-to-regexp@0.4.1: + resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} + + glob@10.3.10: + resolution: {integrity: sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + + glob@7.1.7: + resolution: {integrity: sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==} + + glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + + globals@13.24.0: + resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} + engines: {node: '>=8'} + + globalthis@1.0.3: + resolution: {integrity: sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==} + engines: {node: '>= 0.4'} + + globby@11.1.0: + resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} + engines: {node: '>=10'} + + goober@2.1.13: + resolution: {integrity: sha512-jFj3BQeleOoy7t93E9rZ2de+ScC4lQICLwiAQmKMg9F6roKGaLSHoCDYKkWlSafg138jejvq/mTdvmnwDQgqoQ==} + peerDependencies: + csstype: ^3.0.10 + + gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + graphemer@1.4.0: + resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + + gzip-size@6.0.0: + resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} + engines: {node: '>=10'} + + has-bigints@1.0.2: + resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==} + + has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-property-descriptors@1.0.1: + resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} + + has-proto@1.0.1: + resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} + engines: {node: '>= 0.4'} + + has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.0: + resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} + engines: {node: '>= 0.4'} + + hasown@2.0.0: + resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} + engines: {node: '>= 0.4'} + + hoist-non-react-statics@3.3.2: + resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + + hyphenate-style-name@1.0.4: + resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==} + + ignore@5.3.0: + resolution: {integrity: sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==} + engines: {node: '>= 4'} + + immer@9.0.21: + resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} + + import-fresh@3.3.0: + resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} + engines: {node: '>=6'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + inline-style-prefixer@7.0.0: + resolution: {integrity: sha512-I7GEdScunP1dQ6IM2mQWh6v0mOYdYmH3Bp31UecKdrcUgcURTcctSe1IECdUznSHKSmsHtjrT3CwCPI1pyxfUQ==} + + internal-slot@1.0.6: + resolution: {integrity: sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==} + engines: {node: '>= 0.4'} + + is-array-buffer@3.0.2: + resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==} + + is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + + is-async-function@2.0.0: + resolution: {integrity: sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==} + engines: {node: '>= 0.4'} + + is-bigint@1.0.4: + resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} + + is-binary-path@2.1.0: + resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} + engines: {node: '>=8'} + + is-boolean-object@1.1.2: + resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} + engines: {node: '>= 0.4'} + + is-callable@1.2.7: + resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} + engines: {node: '>= 0.4'} + + is-core-module@2.13.1: + resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} + + is-date-object@1.0.5: + resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==} + engines: {node: '>= 0.4'} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-finalizationregistry@1.0.2: + resolution: {integrity: sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-generator-function@1.0.10: + resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} + engines: {node: '>= 0.4'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-map@2.0.2: + resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} + + is-negative-zero@2.0.2: + resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} + engines: {node: '>= 0.4'} + + is-number-object@1.0.7: + resolution: {integrity: sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==} + engines: {node: '>= 0.4'} + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-path-inside@3.0.3: + resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} + engines: {node: '>=8'} + + is-regex@1.1.4: + resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} + engines: {node: '>= 0.4'} + + is-set@2.0.2: + resolution: {integrity: sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==} + + is-shared-array-buffer@1.0.2: + resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==} + + is-string@1.0.7: + resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} + engines: {node: '>= 0.4'} + + is-symbol@1.0.4: + resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} + engines: {node: '>= 0.4'} + + is-typed-array@1.1.12: + resolution: {integrity: sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==} + engines: {node: '>= 0.4'} + + is-weakmap@2.0.1: + resolution: {integrity: sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==} + + is-weakref@1.0.2: + resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} + + is-weakset@2.0.2: + resolution: {integrity: sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==} + + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + iterator.prototype@1.1.2: + resolution: {integrity: sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==} + + jackspeak@2.3.6: + resolution: {integrity: sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==} + engines: {node: '>=14'} + + jiti@1.21.0: + resolution: {integrity: sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==} + hasBin: true + + js-cookie@2.2.1: + resolution: {integrity: sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + json2mq@0.2.0: + resolution: {integrity: sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==} + + json5@1.0.2: + resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} + hasBin: true + + jsx-ast-utils@3.3.5: + resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} + engines: {node: '>=4.0'} + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + language-subtag-registry@0.3.22: + resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} + + language-tags@1.0.9: + resolution: {integrity: sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==} + engines: {node: '>=0.10'} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lilconfig@2.1.0: + resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} + engines: {node: '>=10'} + + lilconfig@3.0.0: + resolution: {integrity: sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==} + engines: {node: '>=14'} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lodash-es@4.17.21: + resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + + lru-cache@10.1.0: + resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==} + engines: {node: 14 || >=16.14} + + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + mdn-data@2.0.14: + resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} + + merge2@1.4.1: + resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} + engines: {node: '>= 8'} + + micromatch@4.0.5: + resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} + engines: {node: '>=8.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + minimatch@9.0.3: + resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} + engines: {node: '>=16 || 14 >=14.17'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minipass@7.0.4: + resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==} + engines: {node: '>=16 || 14 >=14.17'} + + mrmime@1.0.1: + resolution: {integrity: sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==} + engines: {node: '>=10'} + + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mz@2.7.0: + resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + + nano-css@5.6.1: + resolution: {integrity: sha512-T2Mhc//CepkTa3X4pUhKgbEheJHYAxD0VptuqFhDbGMUWVV2m+lkNiW/Ieuj35wrfC8Zm0l7HvssQh7zcEttSw==} + peerDependencies: + react: '*' + react-dom: '*' + + nanoid@3.3.7: + resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + next@14.0.3: + resolution: {integrity: sha512-AbYdRNfImBr3XGtvnwOxq8ekVCwbFTv/UJoLwmaX89nk9i051AEY4/HAWzU0YpaTDw8IofUpmuIlvzWF13jxIw==} + engines: {node: '>=18.17.0'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.1.0 + react: ^18.2.0 + react-dom: ^18.2.0 + sass: ^1.3.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + sass: + optional: true + + node-releases@2.0.14: + resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + normalize-range@0.1.2: + resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} + engines: {node: '>=0.10.0'} + + nprogress@0.2.0: + resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-hash@3.0.0: + resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} + engines: {node: '>= 6'} + + object-inspect@1.13.1: + resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + + object.assign@4.1.5: + resolution: {integrity: sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==} + engines: {node: '>= 0.4'} + + object.entries@1.1.7: + resolution: {integrity: sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==} + engines: {node: '>= 0.4'} + + object.fromentries@2.0.7: + resolution: {integrity: sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==} + engines: {node: '>= 0.4'} + + object.groupby@1.0.1: + resolution: {integrity: sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ==} + + object.hasown@1.1.3: + resolution: {integrity: sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA==} + + object.values@1.1.7: + resolution: {integrity: sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==} + engines: {node: '>= 0.4'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + opener@1.5.2: + resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} + hasBin: true + + optionator@0.9.3: + resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==} + engines: {node: '>= 0.8.0'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + + path-scurry@1.10.1: + resolution: {integrity: sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==} + engines: {node: '>=16 || 14 >=14.17'} + + path-type@4.0.0: + resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} + engines: {node: '>=8'} + + picocolors@1.0.0: + resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} + + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + + pify@2.3.0: + resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} + engines: {node: '>=0.10.0'} + + pirates@4.0.6: + resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} + engines: {node: '>= 6'} + + postcss-import@15.1.0: + resolution: {integrity: sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==} + engines: {node: '>=14.0.0'} + peerDependencies: + postcss: ^8.0.0 + + postcss-js@4.0.1: + resolution: {integrity: sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==} + engines: {node: ^12 || ^14 || >= 16} + peerDependencies: + postcss: ^8.4.21 + + postcss-load-config@4.0.2: + resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} + engines: {node: '>= 14'} + peerDependencies: + postcss: '>=8.0.9' + ts-node: '>=9.0.0' + peerDependenciesMeta: + postcss: + optional: true + ts-node: + optional: true + + postcss-nested@6.0.1: + resolution: {integrity: sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==} + engines: {node: '>=12.0'} + peerDependencies: + postcss: ^8.2.14 + + postcss-selector-parser@6.0.15: + resolution: {integrity: sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==} + engines: {node: '>=4'} + + postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + + postcss@8.4.31: + resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==} + engines: {node: ^10 || ^12 || >=14} + + postcss@8.4.33: + resolution: {integrity: sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + prettier@3.1.1: + resolution: {integrity: sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==} + engines: {node: '>=14'} + hasBin: true + + prop-types@15.8.1: + resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + + property-expr@2.0.6: + resolution: {integrity: sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA==} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + qrcode.react@3.1.0: + resolution: {integrity: sha512-oyF+Urr3oAMUG/OiOuONL3HXM+53wvuH3mtIWQrYmsXoAq0DkvZp2RYUWFSMFtbdOpuS++9v+WAkzNVkMlNW6Q==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + + qs@6.11.2: + resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + engines: {node: '>=0.6'} + + queue-microtask@1.2.3: + resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + + rc-cascader@3.21.2: + resolution: {integrity: sha512-J7GozpgsLaOtzfIHFJFuh4oFY0ePb1w10twqK6is3pAkqHkca/PsokbDr822KIRZ8/CK8CqevxohuPDVZ1RO/A==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-checkbox@3.1.0: + resolution: {integrity: sha512-PAwpJFnBa3Ei+5pyqMMXdcKYKNBMS+TvSDiLdDnARnMJHC8ESxwPfm4Ao1gJiKtWLdmGfigascnCpwrHFgoOBQ==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-collapse@3.7.2: + resolution: {integrity: sha512-ZRw6ipDyOnfLFySxAiCMdbHtb5ePAsB9mT17PA6y1mRD/W6KHRaZeb5qK/X9xDV1CqgyxMpzw0VdS74PCcUk4A==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-dialog@9.3.4: + resolution: {integrity: sha512-975X3018GhR+EjZFbxA2Z57SX5rnu0G0/OxFgMMvZK4/hQWEm3MHaNvP4wXpxYDoJsp+xUvVW+GB9CMMCm81jA==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-drawer@7.0.0: + resolution: {integrity: sha512-ePcS4KtQnn57bCbVXazHN2iC8nTPCXlWEIA/Pft87Pd9U7ZeDkdRzG47jWG2/TAFXFlFltRAMcslqmUM8NPCGA==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-dropdown@4.1.0: + resolution: {integrity: sha512-VZjMunpBdlVzYpEdJSaV7WM7O0jf8uyDjirxXLZRNZ+tAC+NzD3PXPEtliFwGzVwBBdCmGuSqiS9DWcOLxQ9tw==} + peerDependencies: + react: '>=16.11.0' + react-dom: '>=16.11.0' + + rc-field-form@1.41.0: + resolution: {integrity: sha512-k9AS0wmxfJfusWDP/YXWTpteDNaQ4isJx9UKxx4/e8Dub4spFeZ54/EuN2sYrMRID/+hUznPgVZeg+Gf7XSYCw==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-image@7.5.1: + resolution: {integrity: sha512-Z9loECh92SQp0nSipc0MBuf5+yVC05H/pzC+Nf8xw1BKDFUJzUeehYBjaWlxly8VGBZJcTHYri61Fz9ng1G3Ag==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-input-number@8.6.1: + resolution: {integrity: sha512-gaAMUKtUKLktJ3Yx93tjgYY1M0HunnoqzPEqkb9//Ydup4DcG0TFL9yHBA3pgVdNIt5f0UWyHCgFBj//JxeD6A==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-input@1.4.3: + resolution: {integrity: sha512-aHyQUAIRmTlOnvk5EcNqEpJ+XMtfMpYRAJayIlJfsvvH9cAKUWboh4egm23vgMA7E+c/qm4BZcnrDcA960GC1w==} + peerDependencies: + react: '>=16.0.0' + react-dom: '>=16.0.0' + + rc-mentions@2.10.1: + resolution: {integrity: sha512-72qsEcr/7su+a07ndJ1j8rI9n0Ka/ngWOLYnWMMv0p2mi/5zPwPrEDTt6Uqpe8FWjWhueDJx/vzunL6IdKDYMg==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-menu@9.12.4: + resolution: {integrity: sha512-t2NcvPLV1mFJzw4F21ojOoRVofK2rWhpKPx69q2raUsiHPDP6DDevsBILEYdsIegqBeSXoWs2bf6CueBKg3BFg==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-motion@2.9.0: + resolution: {integrity: sha512-XIU2+xLkdIr1/h6ohPZXyPBMvOmuyFZQ/T0xnawz+Rh+gh4FINcnZmMT5UTIj6hgI0VLDjTaPeRd+smJeSPqiQ==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-notification@5.3.0: + resolution: {integrity: sha512-WCf0uCOkZ3HGfF0p1H4Sgt7aWfipxORWTPp7o6prA3vxwtWhtug3GfpYls1pnBp4WA+j8vGIi5c2/hQRpGzPcQ==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-overflow@1.3.2: + resolution: {integrity: sha512-nsUm78jkYAoPygDAcGZeC2VwIg/IBGSodtOY3pMof4W3M9qRJgqaDYm03ZayHlde3I6ipliAxbN0RUcGf5KOzw==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-pagination@4.0.4: + resolution: {integrity: sha512-GGrLT4NgG6wgJpT/hHIpL9nELv27A1XbSZzECIuQBQTVSf4xGKxWr6I/jhpRPauYEWEbWVw22ObG6tJQqwJqWQ==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-picker@3.14.6: + resolution: {integrity: sha512-AdKKW0AqMwZsKvIpwUWDUnpuGKZVrbxVTZTNjcO+pViGkjC1EBcjMgxVe8tomOEaIHJL5Gd13vS8Rr3zzxWmag==} + engines: {node: '>=8.x'} + peerDependencies: + date-fns: '>= 2.x' + dayjs: '>= 1.x' + luxon: '>= 3.x' + moment: '>= 2.x' + react: '>=16.9.0' + react-dom: '>=16.9.0' + peerDependenciesMeta: + date-fns: + optional: true + dayjs: + optional: true + luxon: + optional: true + moment: + optional: true + + rc-progress@3.5.1: + resolution: {integrity: sha512-V6Amx6SbLRwPin/oD+k1vbPrO8+9Qf8zW1T8A7o83HdNafEVvAxPV5YsgtKFP+Ud5HghLj33zKOcEHrcrUGkfw==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-rate@2.12.0: + resolution: {integrity: sha512-g092v5iZCdVzbjdn28FzvWebK2IutoVoiTeqoLTj9WM7SjA/gOJIw5/JFZMRyJYYVe1jLAU2UhAfstIpCNRozg==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-resize-observer@1.4.0: + resolution: {integrity: sha512-PnMVyRid9JLxFavTjeDXEXo65HCRqbmLBw9xX9gfC4BZiSzbLXKzW3jPz+J0P71pLbD5tBMTT+mkstV5gD0c9Q==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-segmented@2.2.2: + resolution: {integrity: sha512-Mq52M96QdHMsNdE/042ibT5vkcGcD5jxKp7HgPC2SRofpia99P5fkfHy1pEaajLMF/kj0+2Lkq1UZRvqzo9mSA==} + peerDependencies: + react: '>=16.0.0' + react-dom: '>=16.0.0' + + rc-select@14.11.0: + resolution: {integrity: sha512-8J8G/7duaGjFiTXCBLWfh5P+KDWyA3KTlZDfV3xj/asMPqB2cmxfM+lH50wRiPIRsCQ6EbkCFBccPuaje3DHIg==} + engines: {node: '>=8.x'} + peerDependencies: + react: '*' + react-dom: '*' + + rc-slider@10.5.0: + resolution: {integrity: sha512-xiYght50cvoODZYI43v3Ylsqiw14+D7ELsgzR40boDZaya1HFa1Etnv9MDkQE8X/UrXAffwv2AcNAhslgYuDTw==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-steps@6.0.1: + resolution: {integrity: sha512-lKHL+Sny0SeHkQKKDJlAjV5oZ8DwCdS2hFhAkIjuQt1/pB81M0cA0ErVFdHq9+jmPmFw1vJB2F5NBzFXLJxV+g==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-switch@4.1.0: + resolution: {integrity: sha512-TI8ufP2Az9oEbvyCeVE4+90PDSljGyuwix3fV58p7HV2o4wBnVToEyomJRVyTaZeqNPAp+vqeo4Wnj5u0ZZQBg==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-table@7.37.0: + resolution: {integrity: sha512-hEB17ktLRVfVmdo+U8MjGr+PuIgdQ8Cxj/N5lwMvP/Az7TOrQxwTMLVEDoj207tyPYLTWifHIF9EJREWwyk67g==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-tabs@14.0.0: + resolution: {integrity: sha512-lp1YWkaPnjlyhOZCPrAWxK6/P6nMGX/BAZcAC3nuVwKz0Byfp+vNnQKK8BRCP2g/fzu+SeB5dm9aUigRu3tRkQ==} + engines: {node: '>=8.x'} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-textarea@1.6.3: + resolution: {integrity: sha512-8k7+8Y2GJ/cQLiClFMg8kUXOOdvcFQrnGeSchOvI2ZMIVvX5a3zQpLxoODL0HTrvU63fPkRmMuqaEcOF9dQemA==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-tooltip@6.1.3: + resolution: {integrity: sha512-HMSbSs5oieZ7XddtINUddBLSVgsnlaSb3bZrzzGWjXa7/B7nNedmsuz72s7EWFEro9mNa7RyF3gOXKYqvJiTcQ==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-tree-select@5.17.0: + resolution: {integrity: sha512-7sRGafswBhf7n6IuHyCEFCildwQIgyKiV8zfYyUoWfZEFdhuk7lCH+DN0aHt+oJrdiY9+6Io/LDXloGe01O8XQ==} + peerDependencies: + react: '*' + react-dom: '*' + + rc-tree@5.8.5: + resolution: {integrity: sha512-PRfcZtVDNkR7oh26RuNe1hpw11c1wfgzwmPFL0lnxGnYefe9lDAO6cg5wJKIAwyXFVt5zHgpjYmaz0CPy1ZtKg==} + engines: {node: '>=10.x'} + peerDependencies: + react: '*' + react-dom: '*' + + rc-upload@4.5.2: + resolution: {integrity: sha512-QO3ne77DwnAPKFn0bA5qJM81QBjQi0e0NHdkvpFyY73Bea2NfITiotqJqVjHgeYPOJu5lLVR32TNGP084aSoXA==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-util@5.38.1: + resolution: {integrity: sha512-e4ZMs7q9XqwTuhIK7zBIVFltUtMSjphuPPQXHoHlzRzNdOwUxDejo0Zls5HYaJfRKNURcsS/ceKVULlhjBrxng==} + peerDependencies: + react: '>=16.9.0' + react-dom: '>=16.9.0' + + rc-virtual-list@3.11.3: + resolution: {integrity: sha512-tu5UtrMk/AXonHwHxUogdXAWynaXsrx1i6dsgg+lOo/KJSF8oBAcprh1z5J3xgnPJD5hXxTL58F8s8onokdt0Q==} + engines: {node: '>=8.x'} + peerDependencies: + react: '*' + react-dom: '*' + + react-dom@18.2.0: + resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} + peerDependencies: + react: ^18.2.0 + + react-hook-form@7.51.3: + resolution: {integrity: sha512-cvJ/wbHdhYx8aviSWh28w9ImjmVsb5Y05n1+FW786vEZQJV5STNM0pW6ujS+oiBecb0ARBxJFyAnXj9+GHXACQ==} + engines: {node: '>=12.22.0'} + peerDependencies: + react: ^16.8.0 || ^17 || ^18 + + react-hot-toast@2.4.1: + resolution: {integrity: sha512-j8z+cQbWIM5LY37pR6uZR6D4LfseplqnuAO4co4u8917hBUvXlEqyP1ZzqVLcqoyUesZZv/ImreoCeHVDpE5pQ==} + engines: {node: '>=10'} + peerDependencies: + react: '>=16' + react-dom: '>=16' + + react-is@16.13.1: + resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + + react-is@18.2.0: + resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} + + react-redux@8.1.3: + resolution: {integrity: sha512-n0ZrutD7DaX/j9VscF+uTALI3oUPa/pO4Z3soOBIjuRn/FzVu6aehhysxZCLi6y7duMf52WNZGMl7CtuK5EnRw==} + peerDependencies: + '@types/react': ^16.8 || ^17.0 || ^18.0 + '@types/react-dom': ^16.8 || ^17.0 || ^18.0 + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + react-native: '>=0.59' + redux: ^4 || ^5.0.0-beta.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + react-dom: + optional: true + react-native: + optional: true + redux: + optional: true + + react-transition-group@4.4.5: + resolution: {integrity: sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==} + peerDependencies: + react: '>=16.6.0' + react-dom: '>=16.6.0' + + react-universal-interface@0.6.2: + resolution: {integrity: sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw==} + peerDependencies: + react: '*' + tslib: '*' + + react-use@17.4.2: + resolution: {integrity: sha512-1jPtmWLD8OJJNYCdYLJEH/HM+bPDfJuyGwCYeJFgPmWY8ttwpgZnW5QnzgM55CYUByUiTjHxsGOnEpLl6yQaoQ==} + peerDependencies: + react: '*' + react-dom: '*' + + react@18.2.0: + resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==} + engines: {node: '>=0.10.0'} + + read-cache@1.0.0: + resolution: {integrity: sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==} + + readdirp@3.6.0: + resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} + engines: {node: '>=8.10.0'} + + redux-thunk@2.4.2: + resolution: {integrity: sha512-+P3TjtnP0k/FEjcBL5FZpoovtvrTNT/UXd4/sluaSyrURlSlhLSzEdfsTBW7WsKB6yPvgd7q/iZPICFjW4o57Q==} + peerDependencies: + redux: ^4 + + redux@4.2.1: + resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} + + reflect.getprototypeof@1.0.4: + resolution: {integrity: sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw==} + engines: {node: '>= 0.4'} + + regenerator-runtime@0.14.1: + resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} + + regexp.prototype.flags@1.5.1: + resolution: {integrity: sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==} + engines: {node: '>= 0.4'} + + reselect@4.1.8: + resolution: {integrity: sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ==} + + resize-observer-polyfill@1.5.1: + resolution: {integrity: sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + + resolve@1.22.8: + resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} + hasBin: true + + resolve@2.0.0-next.5: + resolution: {integrity: sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==} + hasBin: true + + reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rimraf@3.0.2: + resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + hasBin: true + + rtl-css-js@1.16.1: + resolution: {integrity: sha512-lRQgou1mu19e+Ya0LsTvKrVJ5TYUbqCVPAiImX3UfLTenarvPUl1QFdvu5Z3PYmHT9RCcwIfbjRQBntExyj3Zg==} + + run-parallel@1.2.0: + resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + + safe-array-concat@1.0.1: + resolution: {integrity: sha512-6XbUAseYE2KtOuGueyeobCySj9L4+66Tn6KQMOPQJrAJEowYKW/YR/MGJZl7FdydUdaFu4LYyDZjxf4/Nmo23Q==} + engines: {node: '>=0.4'} + + safe-regex-test@1.0.0: + resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} + + scheduler@0.23.0: + resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} + + screenfull@5.2.0: + resolution: {integrity: sha512-9BakfsO2aUQN2K9Fdbj87RJIEZ82Q9IGim7FqM5OsebfoFC6ZHXgDq/KvniuLTPdeM8wY2o6Dj3WQ7KeQCj3cA==} + engines: {node: '>=0.10.0'} + + scroll-into-view-if-needed@3.1.0: + resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + semver@7.5.4: + resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} + engines: {node: '>=10'} + hasBin: true + + set-function-length@1.1.1: + resolution: {integrity: sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==} + engines: {node: '>= 0.4'} + + set-function-name@2.0.1: + resolution: {integrity: sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==} + engines: {node: '>= 0.4'} + + set-harmonic-interval@1.0.1: + resolution: {integrity: sha512-AhICkFV84tBP1aWqPwLZqFvAwqEoVA9kxNMniGEUvzOlm4vLmOFLiTT3UZ6bziJTy4bOVpzWGTfSCbmaayGx8g==} + engines: {node: '>=6.9'} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel@1.0.4: + resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sirv@1.0.19: + resolution: {integrity: sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ==} + engines: {node: '>= 10'} + + slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + + source-map-js@1.0.2: + resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} + engines: {node: '>=0.10.0'} + + source-map@0.5.6: + resolution: {integrity: sha512-MjZkVp0NHr5+TPihLcadqnlVoGIoWo4IBHptutGh9wI3ttUYvCG26HkSuDi+K6lsZ25syXJXcctwgyVCt//xqA==} + engines: {node: '>=0.10.0'} + + source-map@0.5.7: + resolution: {integrity: sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==} + engines: {node: '>=0.10.0'} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + stack-generator@2.0.10: + resolution: {integrity: sha512-mwnua/hkqM6pF4k8SnmZ2zfETsRUpWXREfA/goT8SLCV4iOFa4bzOX2nDipWAZFPTjLvQB82f5yaodMVhK0yJQ==} + + stackframe@1.3.4: + resolution: {integrity: sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==} + + stacktrace-gps@3.1.2: + resolution: {integrity: sha512-GcUgbO4Jsqqg6RxfyTHFiPxdPqF+3LFmQhm7MgCuYQOYuWyqxo5pwRPz5d/u6/WYJdEnWfK4r+jGbyD8TSggXQ==} + + stacktrace-js@2.0.2: + resolution: {integrity: sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==} + + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + + string-convert@0.2.1: + resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + string.prototype.matchall@4.0.10: + resolution: {integrity: sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ==} + + string.prototype.trim@1.2.8: + resolution: {integrity: sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ==} + engines: {node: '>= 0.4'} + + string.prototype.trimend@1.0.7: + resolution: {integrity: sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA==} + + string.prototype.trimstart@1.0.7: + resolution: {integrity: sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg==} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + + strip-bom@3.0.0: + resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} + engines: {node: '>=4'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + styled-jsx@5.1.1: + resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} + engines: {node: '>= 12.0.0'} + peerDependencies: + '@babel/core': '*' + babel-plugin-macros: '*' + react: '>= 16.8.0 || 17.x.x || ^18.0.0-0' + peerDependenciesMeta: + '@babel/core': + optional: true + babel-plugin-macros: + optional: true + + stylis@4.2.0: + resolution: {integrity: sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==} + + stylis@4.3.1: + resolution: {integrity: sha512-EQepAV+wMsIaGVGX1RECzgrcqRRU/0sYOHkeLsZ3fzHaHXZy4DaOOX0vOlGQdlsjkh3mFHAIlVimpwAs4dslyQ==} + + sucrase@3.35.0: + resolution: {integrity: sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + + supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + + tailwindcss@3.4.1: + resolution: {integrity: sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==} + engines: {node: '>=14.0.0'} + hasBin: true + + tapable@2.2.1: + resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} + engines: {node: '>=6'} + + text-table@0.2.0: + resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} + + thenify-all@1.6.0: + resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} + engines: {node: '>=0.8'} + + thenify@3.3.1: + resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + + throttle-debounce@3.0.1: + resolution: {integrity: sha512-dTEWWNu6JmeVXY0ZYoPuH5cRIwc0MeGbJwah9KUNYSJwommQpCzTySTpEe8Gs1J23aeWEuAobe4Ag7EHVt/LOg==} + engines: {node: '>=10'} + + throttle-debounce@5.0.0: + resolution: {integrity: sha512-2iQTSgkkc1Zyk0MeVrt/3BvuOXYPl/R8Z0U2xxo9rjwNciaHDG3R+Lm6dh4EeUci49DanvBnuqI6jshoQQRGEg==} + engines: {node: '>=12.22'} + + tiny-case@1.0.3: + resolution: {integrity: sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==} + + to-fast-properties@2.0.0: + resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} + engines: {node: '>=4'} + + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + toggle-selection@1.0.6: + resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} + + toposort@2.0.2: + resolution: {integrity: sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==} + + totalist@1.1.0: + resolution: {integrity: sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==} + engines: {node: '>=6'} + + ts-api-utils@1.0.3: + resolution: {integrity: sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==} + engines: {node: '>=16.13.0'} + peerDependencies: + typescript: '>=4.2.0' + + ts-easing@0.2.0: + resolution: {integrity: sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ==} + + ts-interface-checker@0.1.13: + resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + + tsconfig-paths@3.15.0: + resolution: {integrity: sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==} + + tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-fest@0.20.2: + resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} + engines: {node: '>=10'} + + type-fest@2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + + typed-array-buffer@1.0.0: + resolution: {integrity: sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==} + engines: {node: '>= 0.4'} + + typed-array-byte-length@1.0.0: + resolution: {integrity: sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==} + engines: {node: '>= 0.4'} + + typed-array-byte-offset@1.0.0: + resolution: {integrity: sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==} + engines: {node: '>= 0.4'} + + typed-array-length@1.0.4: + resolution: {integrity: sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==} + + typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + + unbox-primitive@1.0.2: + resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + update-browserslist-db@1.0.13: + resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + use-sync-external-store@1.2.0: + resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + watchpack@2.4.0: + resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} + engines: {node: '>=10.13.0'} + + webpack-bundle-analyzer@4.7.0: + resolution: {integrity: sha512-j9b8ynpJS4K+zfO5GGwsAcQX4ZHpWV+yRiHDiL+bE0XHJ8NiPYLTNVQdlFYWxtpg9lfAQNlwJg16J9AJtFSXRg==} + engines: {node: '>= 10.13.0'} + hasBin: true + + which-boxed-primitive@1.0.2: + resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} + + which-builtin-type@1.1.3: + resolution: {integrity: sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw==} + engines: {node: '>= 0.4'} + + which-collection@1.0.1: + resolution: {integrity: sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==} + + which-typed-array@1.1.13: + resolution: {integrity: sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==} + engines: {node: '>= 0.4'} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + ws@7.5.9: + resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==} + engines: {node: '>=8.3.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + + yaml@1.10.2: + resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} + engines: {node: '>= 6'} + + yaml@2.3.4: + resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} + engines: {node: '>= 14'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + + yup@1.3.3: + resolution: {integrity: sha512-v8QwZSsHH2K3/G9WSkp6mZKO+hugKT1EmnMqLNUcfu51HU9MDyhlETT/JgtzprnrnQHPWsjc6MUDMBp/l9fNnw==} + +snapshots: + + '@aashutoshrathi/word-wrap@1.2.6': {} + + '@alloc/quick-lru@5.2.0': {} + + '@ant-design/colors@7.0.2': + dependencies: + '@ctrl/tinycolor': 3.6.1 + + '@ant-design/cssinjs@1.18.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@emotion/hash': 0.8.0 + '@emotion/unitless': 0.7.5 + classnames: 2.5.1 + csstype: 3.1.3 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + stylis: 4.3.1 + + '@ant-design/icons-svg@4.4.1': {} + + '@ant-design/icons@5.3.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@ant-design/colors': 7.0.2 + '@ant-design/icons-svg': 4.4.1 + '@babel/runtime': 7.23.8 + classnames: 2.5.1 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@ant-design/react-slick@1.0.2(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + classnames: 2.5.1 + json2mq: 0.2.0 + react: 18.2.0 + resize-observer-polyfill: 1.5.1 + throttle-debounce: 5.0.0 + + '@babel/code-frame@7.23.5': + dependencies: + '@babel/highlight': 7.23.4 + chalk: 2.4.2 + + '@babel/helper-module-imports@7.22.15': + dependencies: + '@babel/types': 7.23.6 + + '@babel/helper-string-parser@7.23.4': {} + + '@babel/helper-validator-identifier@7.22.20': {} + + '@babel/highlight@7.23.4': + dependencies: + '@babel/helper-validator-identifier': 7.22.20 + chalk: 2.4.2 + js-tokens: 4.0.0 + + '@babel/runtime@7.23.8': + dependencies: + regenerator-runtime: 0.14.1 + + '@babel/runtime@7.24.0': + dependencies: + regenerator-runtime: 0.14.1 + + '@babel/types@7.23.6': + dependencies: + '@babel/helper-string-parser': 7.23.4 + '@babel/helper-validator-identifier': 7.22.20 + to-fast-properties: 2.0.0 + + '@ctrl/tinycolor@3.6.1': {} + + '@emotion/babel-plugin@11.11.0': + dependencies: + '@babel/helper-module-imports': 7.22.15 + '@babel/runtime': 7.23.8 + '@emotion/hash': 0.9.1 + '@emotion/memoize': 0.8.1 + '@emotion/serialize': 1.1.3 + babel-plugin-macros: 3.1.0 + convert-source-map: 1.9.0 + escape-string-regexp: 4.0.0 + find-root: 1.1.0 + source-map: 0.5.7 + stylis: 4.2.0 + + '@emotion/cache@11.11.0': + dependencies: + '@emotion/memoize': 0.8.1 + '@emotion/sheet': 1.2.2 + '@emotion/utils': 1.2.1 + '@emotion/weak-memoize': 0.3.1 + stylis: 4.2.0 + + '@emotion/hash@0.8.0': {} + + '@emotion/hash@0.9.1': {} + + '@emotion/is-prop-valid@1.2.1': + dependencies: + '@emotion/memoize': 0.8.1 + + '@emotion/memoize@0.8.1': {} + + '@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@emotion/babel-plugin': 11.11.0 + '@emotion/cache': 11.11.0 + '@emotion/serialize': 1.1.3 + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) + '@emotion/utils': 1.2.1 + '@emotion/weak-memoize': 0.3.1 + hoist-non-react-statics: 3.3.2 + react: 18.2.0 + optionalDependencies: + '@types/react': 18.2.47 + + '@emotion/serialize@1.1.3': + dependencies: + '@emotion/hash': 0.9.1 + '@emotion/memoize': 0.8.1 + '@emotion/unitless': 0.8.1 + '@emotion/utils': 1.2.1 + csstype: 3.1.3 + + '@emotion/sheet@1.2.2': {} + + '@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@emotion/babel-plugin': 11.11.0 + '@emotion/is-prop-valid': 1.2.1 + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/serialize': 1.1.3 + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) + '@emotion/utils': 1.2.1 + react: 18.2.0 + optionalDependencies: + '@types/react': 18.2.47 + + '@emotion/unitless@0.7.5': {} + + '@emotion/unitless@0.8.1': {} + + '@emotion/use-insertion-effect-with-fallbacks@1.0.1(react@18.2.0)': + dependencies: + react: 18.2.0 + + '@emotion/utils@1.2.1': {} + + '@emotion/weak-memoize@0.3.1': {} + + '@eslint-community/eslint-utils@4.4.0(eslint@8.56.0)': + dependencies: + eslint: 8.56.0 + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.10.0': {} + + '@eslint/eslintrc@2.1.4': + dependencies: + ajv: 6.12.6 + debug: 4.3.4 + espree: 9.6.1 + globals: 13.24.0 + ignore: 5.3.0 + import-fresh: 3.3.0 + js-yaml: 4.1.0 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@8.56.0': {} + + '@floating-ui/core@1.5.3': + dependencies: + '@floating-ui/utils': 0.2.1 + + '@floating-ui/dom@1.5.4': + dependencies: + '@floating-ui/core': 1.5.3 + '@floating-ui/utils': 0.2.1 + + '@floating-ui/react-dom@2.0.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@floating-ui/dom': 1.5.4 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@floating-ui/utils@0.2.1': {} + + '@hookform/resolvers@3.3.4(react-hook-form@7.51.3(react@18.2.0))': + dependencies: + react-hook-form: 7.51.3(react@18.2.0) + + '@humanwhocodes/config-array@0.11.13': + dependencies: + '@humanwhocodes/object-schema': 2.0.1 + debug: 4.3.4 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/object-schema@2.0.1': {} + + '@iconify/react@4.1.1(react@18.2.0)': + dependencies: + '@iconify/types': 2.0.0 + react: 18.2.0 + + '@iconify/types@2.0.0': {} + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@jridgewell/gen-mapping@0.3.3': + dependencies: + '@jridgewell/set-array': 1.1.2 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.20 + + '@jridgewell/resolve-uri@3.1.1': {} + + '@jridgewell/set-array@1.1.2': {} + + '@jridgewell/sourcemap-codec@1.4.15': {} + + '@jridgewell/trace-mapping@0.3.20': + dependencies: + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 + + '@mui/base@5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@floating-ui/react-dom': 2.0.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/types': 7.2.12(@types/react@18.2.47) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + '@popperjs/core': 2.11.8 + clsx: 2.1.0 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + optionalDependencies: + '@types/react': 18.2.47 + + '@mui/core-downloads-tracker@5.15.3': {} + + '@mui/icons-material@5.15.11(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.24.0 + '@mui/material': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + optionalDependencies: + '@types/react': 18.2.47 + + '@mui/lab@5.0.0-alpha.159(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/material': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/system': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/types': 7.2.12(@types/react@18.2.47) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + clsx: 2.1.0 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + optionalDependencies: + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@types/react': 18.2.47 + + '@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/core-downloads-tracker': 5.15.3 + '@mui/system': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/types': 7.2.12(@types/react@18.2.47) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + '@types/react-transition-group': 4.4.10 + clsx: 2.1.0 + csstype: 3.1.3 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-is: 18.2.0 + react-transition-group: 4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + optionalDependencies: + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@types/react': 18.2.47 + + '@mui/private-theming@5.15.3(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + prop-types: 15.8.1 + react: 18.2.0 + optionalDependencies: + '@types/react': 18.2.47 + + '@mui/styled-engine@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@emotion/cache': 11.11.0 + csstype: 3.1.3 + prop-types: 15.8.1 + react: 18.2.0 + optionalDependencies: + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + + '@mui/system@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@mui/private-theming': 5.15.3(@types/react@18.2.47)(react@18.2.0) + '@mui/styled-engine': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(react@18.2.0) + '@mui/types': 7.2.12(@types/react@18.2.47) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + clsx: 2.1.0 + csstype: 3.1.3 + prop-types: 15.8.1 + react: 18.2.0 + optionalDependencies: + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@types/react': 18.2.47 + + '@mui/types@7.2.12(@types/react@18.2.47)': + optionalDependencies: + '@types/react': 18.2.47 + + '@mui/utils@5.15.3(@types/react@18.2.47)(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@types/prop-types': 15.7.11 + prop-types: 15.8.1 + react: 18.2.0 + react-is: 18.2.0 + optionalDependencies: + '@types/react': 18.2.47 + + '@mui/x-data-grid@6.18.7(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mui/system@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@mui/material': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/system': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + clsx: 2.1.0 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + reselect: 4.1.8 + transitivePeerDependencies: + - '@types/react' + + '@mui/x-tree-view@6.17.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@mui/material@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mui/system@5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@emotion/react': 11.11.3(@types/react@18.2.47)(react@18.2.0) + '@emotion/styled': 11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/base': 5.0.0-beta.30(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/material': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@mui/system': 5.15.3(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@emotion/styled@11.11.0(@emotion/react@11.11.3(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0))(@types/react@18.2.47)(react@18.2.0) + '@mui/utils': 5.15.3(@types/react@18.2.47)(react@18.2.0) + '@types/react-transition-group': 4.4.10 + clsx: 2.1.0 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-transition-group: 4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + transitivePeerDependencies: + - '@types/react' + + '@next/bundle-analyzer@14.0.4': + dependencies: + webpack-bundle-analyzer: 4.7.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@next/env@14.0.3': {} + + '@next/eslint-plugin-next@14.0.3': + dependencies: + glob: 7.1.7 + + '@next/swc-darwin-arm64@14.0.3': + optional: true + + '@next/swc-darwin-x64@14.0.3': + optional: true + + '@next/swc-linux-arm64-gnu@14.0.3': + optional: true + + '@next/swc-linux-arm64-musl@14.0.3': + optional: true + + '@next/swc-linux-x64-gnu@14.0.3': + optional: true + + '@next/swc-linux-x64-musl@14.0.3': + optional: true + + '@next/swc-win32-arm64-msvc@14.0.3': + optional: true + + '@next/swc-win32-ia32-msvc@14.0.3': + optional: true + + '@next/swc-win32-x64-msvc@14.0.3': + optional: true + + '@nodelib/fs.scandir@2.1.5': + dependencies: + '@nodelib/fs.stat': 2.0.5 + run-parallel: 1.2.0 + + '@nodelib/fs.stat@2.0.5': {} + + '@nodelib/fs.walk@1.2.8': + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.16.0 + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@polka/url@1.0.0-next.24': {} + + '@popperjs/core@2.11.8': {} + + '@rc-component/color-picker@1.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@ctrl/tinycolor': 3.6.1 + classnames: 2.5.1 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@rc-component/context@1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@rc-component/mini-decimal@1.1.0': + dependencies: + '@babel/runtime': 7.23.8 + + '@rc-component/mutate-observer@1.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + classnames: 2.5.1 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@rc-component/portal@1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + classnames: 2.5.1 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@rc-component/tour@1.12.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@rc-component/portal': 1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + classnames: 2.5.1 + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@rc-component/trigger@1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': + dependencies: + '@babel/runtime': 7.23.8 + '@rc-component/portal': 1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + classnames: 2.5.1 + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + + '@reduxjs/toolkit@1.9.7(react-redux@8.1.3(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(redux@4.2.1))(react@18.2.0)': + dependencies: + immer: 9.0.21 + redux: 4.2.1 + redux-thunk: 2.4.2(redux@4.2.1) + reselect: 4.1.8 + optionalDependencies: + react: 18.2.0 + react-redux: 8.1.3(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(redux@4.2.1) + + '@rushstack/eslint-patch@1.6.1': {} + + '@swc/helpers@0.5.2': + dependencies: + tslib: 2.6.2 + + '@types/hoist-non-react-statics@3.3.5': + dependencies: + '@types/react': 18.2.47 + hoist-non-react-statics: 3.3.2 + + '@types/js-cookie@2.2.7': {} + + '@types/json5@0.0.29': {} + + '@types/lodash-es@4.17.12': + dependencies: + '@types/lodash': 4.14.202 + + '@types/lodash@4.14.202': {} + + '@types/node@20.10.7': + dependencies: + undici-types: 5.26.5 + + '@types/parse-json@4.0.2': {} + + '@types/prop-types@15.7.11': {} + + '@types/qs@6.9.11': {} + + '@types/react-transition-group@4.4.10': + dependencies: + '@types/react': 18.2.47 + + '@types/react@18.2.47': + dependencies: + '@types/prop-types': 15.7.11 + '@types/scheduler': 0.16.8 + csstype: 3.1.3 + + '@types/scheduler@0.16.8': {} + + '@types/use-sync-external-store@0.0.3': {} + + '@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3)': + dependencies: + '@typescript-eslint/scope-manager': 6.18.1 + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/typescript-estree': 6.18.1(typescript@5.3.3) + '@typescript-eslint/visitor-keys': 6.18.1 + debug: 4.3.4 + eslint: 8.56.0 + optionalDependencies: + typescript: 5.3.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@6.18.1': + dependencies: + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/visitor-keys': 6.18.1 + + '@typescript-eslint/types@6.18.1': {} + + '@typescript-eslint/typescript-estree@6.18.1(typescript@5.3.3)': + dependencies: + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/visitor-keys': 6.18.1 + debug: 4.3.4 + globby: 11.1.0 + is-glob: 4.0.3 + minimatch: 9.0.3 + semver: 7.5.4 + ts-api-utils: 1.0.3(typescript@5.3.3) + optionalDependencies: + typescript: 5.3.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@6.18.1': + dependencies: + '@typescript-eslint/types': 6.18.1 + eslint-visitor-keys: 3.4.3 + + '@ungap/structured-clone@1.2.0': {} + + '@xobotyi/scrollbar-width@1.9.5': {} + + acorn-jsx@5.3.2(acorn@8.11.3): + dependencies: + acorn: 8.11.3 + + acorn-walk@8.3.1: {} + + acorn@8.11.3: {} + + ajv@6.12.6: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ansi-regex@5.0.1: {} + + ansi-regex@6.0.1: {} + + ansi-styles@3.2.1: + dependencies: + color-convert: 1.9.3 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.1: {} + + antd@5.13.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): + dependencies: + '@ant-design/colors': 7.0.2 + '@ant-design/cssinjs': 1.18.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@ant-design/icons': 5.3.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@ant-design/react-slick': 1.0.2(react@18.2.0) + '@ctrl/tinycolor': 3.6.1 + '@rc-component/color-picker': 1.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@rc-component/mutate-observer': 1.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@rc-component/tour': 1.12.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + classnames: 2.5.1 + copy-to-clipboard: 3.3.3 + dayjs: 1.11.10 + qrcode.react: 3.1.0(react@18.2.0) + rc-cascader: 3.21.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-checkbox: 3.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-collapse: 3.7.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-dialog: 9.3.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-drawer: 7.0.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-dropdown: 4.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-field-form: 1.41.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-image: 7.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-input: 1.4.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-input-number: 8.6.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-mentions: 2.10.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-menu: 9.12.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-notification: 5.3.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-pagination: 4.0.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-picker: 3.14.6(dayjs@1.11.10)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-progress: 3.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-rate: 2.12.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-segmented: 2.2.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-select: 14.11.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-slider: 10.5.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-steps: 6.0.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-switch: 4.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-table: 7.37.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tabs: 14.0.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-textarea: 1.6.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tooltip: 6.1.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tree: 5.8.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tree-select: 5.17.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-upload: 4.5.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + scroll-into-view-if-needed: 3.1.0 + throttle-debounce: 5.0.0 + transitivePeerDependencies: + - date-fns + - luxon + - moment + + any-promise@1.3.0: {} + + anymatch@3.1.3: + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.1 + + arg@5.0.2: {} + + argparse@2.0.1: {} + + aria-query@5.3.0: + dependencies: + dequal: 2.0.3 + + array-buffer-byte-length@1.0.0: + dependencies: + call-bind: 1.0.5 + is-array-buffer: 3.0.2 + + array-includes@3.1.7: + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + get-intrinsic: 1.2.2 + is-string: 1.0.7 + + array-tree-filter@2.1.0: {} + + array-union@2.1.0: {} + + array.prototype.findlastindex@1.2.3: + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + es-shim-unscopables: 1.0.2 + get-intrinsic: 1.2.2 + + array.prototype.flat@1.3.2: + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + es-shim-unscopables: 1.0.2 + + array.prototype.flatmap@1.3.2: + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + es-shim-unscopables: 1.0.2 + + array.prototype.tosorted@1.1.2: + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + es-shim-unscopables: 1.0.2 + get-intrinsic: 1.2.2 + + arraybuffer.prototype.slice@1.0.2: + dependencies: + array-buffer-byte-length: 1.0.0 + call-bind: 1.0.5 + define-properties: 1.2.1 + es-abstract: 1.22.3 + get-intrinsic: 1.2.2 + is-array-buffer: 3.0.2 + is-shared-array-buffer: 1.0.2 + + ast-types-flow@0.0.8: {} + + async-validator@4.2.5: {} + + asynciterator.prototype@1.0.0: + dependencies: + has-symbols: 1.0.3 + + asynckit@0.4.0: {} + + autoprefixer@10.4.16(postcss@8.4.33): + dependencies: + browserslist: 4.22.2 + caniuse-lite: 1.0.30001576 + fraction.js: 4.3.7 + normalize-range: 0.1.2 + picocolors: 1.0.0 + postcss: 8.4.33 + postcss-value-parser: 4.2.0 + + available-typed-arrays@1.0.5: {} + + axe-core@4.7.0: {} + + axios@1.6.8: + dependencies: + follow-redirects: 1.15.6 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + axobject-query@3.2.1: + dependencies: + dequal: 2.0.3 + + babel-plugin-macros@3.1.0: + dependencies: + '@babel/runtime': 7.23.8 + cosmiconfig: 7.1.0 + resolve: 1.22.8 + + balanced-match@1.0.2: {} + + binary-extensions@2.2.0: {} + + brace-expansion@1.1.11: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.0.1: + dependencies: + balanced-match: 1.0.2 + + braces@3.0.2: + dependencies: + fill-range: 7.0.1 + + browserslist@4.22.2: + dependencies: + caniuse-lite: 1.0.30001576 + electron-to-chromium: 1.4.625 + node-releases: 2.0.14 + update-browserslist-db: 1.0.13(browserslist@4.22.2) + + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + + call-bind@1.0.5: + dependencies: + function-bind: 1.1.2 + get-intrinsic: 1.2.2 + set-function-length: 1.1.1 + + callsites@3.1.0: {} + + camelcase-css@2.0.1: {} + + caniuse-lite@1.0.30001576: {} + + chalk@2.4.2: + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + chokidar@3.5.3: + dependencies: + anymatch: 3.1.3 + braces: 3.0.2 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.6.0 + optionalDependencies: + fsevents: 2.3.3 + + chroma-js@2.4.2: {} + + classnames@2.5.1: {} + + client-only@0.0.1: {} + + clsx@2.1.0: {} + + color-convert@1.9.3: + dependencies: + color-name: 1.1.3 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.3: {} + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + commander@4.1.1: {} + + commander@7.2.0: {} + + compute-scroll-into-view@3.1.0: {} + + concat-map@0.0.1: {} + + convert-source-map@1.9.0: {} + + copy-to-clipboard@3.3.3: + dependencies: + toggle-selection: 1.0.6 + + cosmiconfig@7.1.0: + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.0 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 1.10.2 + + cross-spawn@7.0.3: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + css-in-js-utils@3.1.0: + dependencies: + hyphenate-style-name: 1.0.4 + + css-tree@1.1.3: + dependencies: + mdn-data: 2.0.14 + source-map: 0.6.1 + + cssesc@3.0.0: {} + + csstype@3.1.3: {} + + damerau-levenshtein@1.0.8: {} + + dayjs@1.11.10: {} + + debug@3.2.7: + dependencies: + ms: 2.1.3 + + debug@4.3.4: + dependencies: + ms: 2.1.2 + + deep-is@0.1.4: {} + + define-data-property@1.1.1: + dependencies: + get-intrinsic: 1.2.2 + gopd: 1.0.1 + has-property-descriptors: 1.0.1 + + define-properties@1.2.1: + dependencies: + define-data-property: 1.1.1 + has-property-descriptors: 1.0.1 + object-keys: 1.1.1 + + delayed-stream@1.0.0: {} + + dequal@2.0.3: {} + + didyoumean@1.2.2: {} + + dir-glob@3.0.1: + dependencies: + path-type: 4.0.0 + + dlv@1.1.3: {} + + doctrine@2.1.0: + dependencies: esutils: 2.0.3 - dev: true - /dom-helpers@5.2.1: - resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} + doctrine@3.0.0: + dependencies: + esutils: 2.0.3 + + dom-helpers@5.2.1: dependencies: '@babel/runtime': 7.23.8 csstype: 3.1.3 - dev: false - /duplexer@0.1.2: - resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} - dev: true + duplexer@0.1.2: {} - /eastasianwidth@0.2.0: - resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - dev: true + eastasianwidth@0.2.0: {} - /electron-to-chromium@1.4.625: - resolution: {integrity: sha512-DENMhh3MFgaPDoXWrVIqSPInQoLImywfCwrSmVl3cf9QHzoZSiutHwGaB/Ql3VkqcQV30rzgdM+BjKqBAJxo5Q==} - dev: true + electron-to-chromium@1.4.625: {} - /emoji-regex@8.0.0: - resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} - dev: true + emoji-regex@8.0.0: {} - /emoji-regex@9.2.2: - resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} - dev: true + emoji-regex@9.2.2: {} - /enhanced-resolve@5.15.0: - resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==} - engines: {node: '>=10.13.0'} + enhanced-resolve@5.15.0: dependencies: graceful-fs: 4.2.11 tapable: 2.2.1 - dev: true - /env-cmd@10.1.0: - resolution: {integrity: sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==} - engines: {node: '>=8.0.0'} - hasBin: true + env-cmd@10.1.0: dependencies: commander: 4.1.1 cross-spawn: 7.0.3 - dev: true - /error-ex@1.3.2: - resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + error-ex@1.3.2: dependencies: is-arrayish: 0.2.1 - dev: false - /error-stack-parser@2.1.4: - resolution: {integrity: sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==} + error-stack-parser@2.1.4: dependencies: stackframe: 1.3.4 - dev: false - /es-abstract@1.22.3: - resolution: {integrity: sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==} - engines: {node: '>= 0.4'} + es-abstract@1.22.3: dependencies: array-buffer-byte-length: 1.0.0 arraybuffer.prototype.slice: 1.0.2 @@ -1883,10 +3639,8 @@ packages: typed-array-length: 1.0.4 unbox-primitive: 1.0.2 which-typed-array: 1.1.13 - dev: true - /es-iterator-helpers@1.0.15: - resolution: {integrity: sha512-GhoY8uYqd6iwUl2kgjTm4CZAf6oo5mHK7BPqx3rKgx893YSsy0LGHV6gfqqQvZt/8xM8xeOnfXBCfqclMKkJ5g==} + es-iterator-helpers@1.0.15: dependencies: asynciterator.prototype: 1.0.0 call-bind: 1.0.5 @@ -1902,102 +3656,66 @@ packages: internal-slot: 1.0.6 iterator.prototype: 1.1.2 safe-array-concat: 1.0.1 - dev: true - /es-set-tostringtag@2.0.2: - resolution: {integrity: sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==} - engines: {node: '>= 0.4'} + es-set-tostringtag@2.0.2: dependencies: get-intrinsic: 1.2.2 has-tostringtag: 1.0.0 hasown: 2.0.0 - dev: true - /es-shim-unscopables@1.0.2: - resolution: {integrity: sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==} + es-shim-unscopables@1.0.2: dependencies: hasown: 2.0.0 - dev: true - /es-to-primitive@1.2.1: - resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} - engines: {node: '>= 0.4'} + es-to-primitive@1.2.1: dependencies: is-callable: 1.2.7 is-date-object: 1.0.5 is-symbol: 1.0.4 - dev: true - /escalade@3.1.1: - resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} - engines: {node: '>=6'} - dev: true + escalade@3.1.1: {} - /escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - dev: false + escape-string-regexp@1.0.5: {} - /escape-string-regexp@4.0.0: - resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} - engines: {node: '>=10'} + escape-string-regexp@4.0.0: {} - /eslint-config-next@14.0.3(eslint@8.56.0)(typescript@5.3.3): - resolution: {integrity: sha512-IKPhpLdpSUyKofmsXUfrvBC49JMUTdeaD8ZIH4v9Vk0sC1X6URTuTJCLtA0Vwuj7V/CQh0oISuSTvNn5//Buew==} - peerDependencies: - eslint: ^7.23.0 || ^8.0.0 - typescript: '>=3.3.1' - peerDependenciesMeta: - typescript: - optional: true + eslint-config-next@14.0.3(eslint@8.56.0)(typescript@5.3.3): dependencies: '@next/eslint-plugin-next': 14.0.3 '@rushstack/eslint-patch': 1.6.1 '@typescript-eslint/parser': 6.18.1(eslint@8.56.0)(typescript@5.3.3) eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.56.0) eslint-plugin-react: 7.33.2(eslint@8.56.0) eslint-plugin-react-hooks: 4.6.0(eslint@8.56.0) + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - eslint-import-resolver-webpack - supports-color - dev: true - /eslint-config-prettier@9.1.0(eslint@8.56.0): - resolution: {integrity: sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==} - hasBin: true - peerDependencies: - eslint: '>=7.0.0' + eslint-config-prettier@9.1.0(eslint@8.56.0): dependencies: eslint: 8.56.0 - dev: true - /eslint-import-resolver-node@0.3.9: - resolution: {integrity: sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==} + eslint-import-resolver-node@0.3.9: dependencies: debug: 3.2.7 is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: - supports-color - dev: true - /eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0): - resolution: {integrity: sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==} - engines: {node: ^14.18.0 || >=16.0.0} - peerDependencies: - eslint: '*' - eslint-plugin-import: '*' + eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0): dependencies: debug: 4.3.4 enhanced-resolve: 5.15.0 eslint: 8.56.0 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0))(eslint@8.56.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) fast-glob: 3.3.2 get-tsconfig: 4.7.2 is-core-module: 2.13.1 @@ -2007,49 +3725,20 @@ packages: - eslint-import-resolver-node - eslint-import-resolver-webpack - supports-color - dev: true - /eslint-module-utils@2.8.0(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0): - resolution: {integrity: sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==} - engines: {node: '>=4'} - peerDependencies: - '@typescript-eslint/parser': '*' - eslint: '*' - eslint-import-resolver-node: '*' - eslint-import-resolver-typescript: '*' - eslint-import-resolver-webpack: '*' - peerDependenciesMeta: - '@typescript-eslint/parser': - optional: true - eslint: - optional: true - eslint-import-resolver-node: - optional: true - eslint-import-resolver-typescript: - optional: true - eslint-import-resolver-webpack: - optional: true + eslint-module-utils@2.8.0(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0))(eslint@8.56.0): dependencies: - '@typescript-eslint/parser': 6.18.1(eslint@8.56.0)(typescript@5.3.3) debug: 3.2.7 + optionalDependencies: + '@typescript-eslint/parser': 6.18.1(eslint@8.56.0)(typescript@5.3.3) eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0) transitivePeerDependencies: - supports-color - dev: true - /eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0): - resolution: {integrity: sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==} - engines: {node: '>=4'} - peerDependencies: - '@typescript-eslint/parser': '*' - eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 - peerDependenciesMeta: - '@typescript-eslint/parser': - optional: true + eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0): dependencies: - '@typescript-eslint/parser': 6.18.1(eslint@8.56.0)(typescript@5.3.3) array-includes: 3.1.7 array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 @@ -2058,7 +3747,7 @@ packages: doctrine: 2.1.0 eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.18.1)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.56.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.56.0))(eslint@8.56.0) hasown: 2.0.0 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -2068,17 +3757,14 @@ packages: object.values: 1.1.7 semver: 6.3.1 tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 6.18.1(eslint@8.56.0)(typescript@5.3.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack - supports-color - dev: true - /eslint-plugin-jsx-a11y@6.8.0(eslint@8.56.0): - resolution: {integrity: sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA==} - engines: {node: '>=4.0'} - peerDependencies: - eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + eslint-plugin-jsx-a11y@6.8.0(eslint@8.56.0): dependencies: '@babel/runtime': 7.23.8 aria-query: 5.3.0 @@ -2097,22 +3783,12 @@ packages: minimatch: 3.1.2 object.entries: 1.1.7 object.fromentries: 2.0.7 - dev: true - /eslint-plugin-react-hooks@4.6.0(eslint@8.56.0): - resolution: {integrity: sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==} - engines: {node: '>=10'} - peerDependencies: - eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 + eslint-plugin-react-hooks@4.6.0(eslint@8.56.0): dependencies: eslint: 8.56.0 - dev: true - /eslint-plugin-react@7.33.2(eslint@8.56.0): - resolution: {integrity: sha512-73QQMKALArI8/7xGLNI/3LylrEYrlKZSb5C9+q3OtOewTnMQi5cT+aE9E41sLCmli3I9PGGmD1yiZydyo4FEPw==} - engines: {node: '>=4'} - peerDependencies: - eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + eslint-plugin-react@7.33.2(eslint@8.56.0): dependencies: array-includes: 3.1.7 array.prototype.flatmap: 1.3.2 @@ -2131,25 +3807,15 @@ packages: resolve: 2.0.0-next.5 semver: 6.3.1 string.prototype.matchall: 4.0.10 - dev: true - /eslint-scope@7.2.2: - resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + eslint-scope@7.2.2: dependencies: esrecurse: 4.3.0 estraverse: 5.3.0 - dev: true - /eslint-visitor-keys@3.4.3: - resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dev: true + eslint-visitor-keys@3.4.3: {} - /eslint@8.56.0: - resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - hasBin: true + eslint@8.56.0: dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0) '@eslint-community/regexpp': 4.10.0 @@ -2191,240 +3857,142 @@ packages: text-table: 0.2.0 transitivePeerDependencies: - supports-color - dev: true - /espree@9.6.1: - resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + espree@9.6.1: dependencies: acorn: 8.11.3 acorn-jsx: 5.3.2(acorn@8.11.3) eslint-visitor-keys: 3.4.3 - dev: true - /esquery@1.5.0: - resolution: {integrity: sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==} - engines: {node: '>=0.10'} + esquery@1.5.0: dependencies: estraverse: 5.3.0 - dev: true - /esrecurse@4.3.0: - resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} - engines: {node: '>=4.0'} + esrecurse@4.3.0: dependencies: estraverse: 5.3.0 - dev: true - /estraverse@5.3.0: - resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} - engines: {node: '>=4.0'} - dev: true + estraverse@5.3.0: {} - /esutils@2.0.3: - resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} - engines: {node: '>=0.10.0'} - dev: true + esutils@2.0.3: {} - /fast-deep-equal@3.1.3: - resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-deep-equal@3.1.3: {} - /fast-glob@3.3.2: - resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} - engines: {node: '>=8.6.0'} + fast-glob@3.3.2: dependencies: '@nodelib/fs.stat': 2.0.5 '@nodelib/fs.walk': 1.2.8 glob-parent: 5.1.2 merge2: 1.4.1 micromatch: 4.0.5 - dev: true - /fast-json-stable-stringify@2.1.0: - resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} - dev: true + fast-json-stable-stringify@2.1.0: {} - /fast-levenshtein@2.0.6: - resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} - dev: true + fast-levenshtein@2.0.6: {} - /fast-loops@1.1.3: - resolution: {integrity: sha512-8EZzEP0eKkEEVX+drtd9mtuQ+/QrlfW/5MlwcwK5Nds6EkZ/tRzEexkzUY2mIssnAyVLT+TKHuRXmFNNXYUd6g==} - dev: false + fast-loops@1.1.3: {} - /fast-shallow-equal@1.0.0: - resolution: {integrity: sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==} - dev: false + fast-shallow-equal@1.0.0: {} - /fastest-stable-stringify@2.0.2: - resolution: {integrity: sha512-bijHueCGd0LqqNK9b5oCMHc0MluJAx0cwqASgbWMvkO01lCYgIhacVRLcaDz3QnyYIRNJRDwMb41VuT6pHJ91Q==} - dev: false + fastest-stable-stringify@2.0.2: {} - /fastq@1.16.0: - resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} + fastq@1.16.0: dependencies: reusify: 1.0.4 - dev: true - /file-entry-cache@6.0.1: - resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} - engines: {node: ^10.12.0 || >=12.0.0} + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0 - dev: true - /fill-range@7.0.1: - resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} - engines: {node: '>=8'} + fill-range@7.0.1: dependencies: to-regex-range: 5.0.1 - dev: true - /find-root@1.1.0: - resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} - dev: false + find-root@1.1.0: {} - /find-up@5.0.0: - resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} - engines: {node: '>=10'} + find-up@5.0.0: dependencies: locate-path: 6.0.0 path-exists: 4.0.0 - dev: true - /flat-cache@3.2.0: - resolution: {integrity: sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==} - engines: {node: ^10.12.0 || >=12.0.0} + flat-cache@3.2.0: dependencies: flatted: 3.2.9 keyv: 4.5.4 rimraf: 3.0.2 - dev: true - /flatted@3.2.9: - resolution: {integrity: sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==} - dev: true + flatted@3.2.9: {} - /follow-redirects@1.15.6: - resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false + follow-redirects@1.15.6: {} - /for-each@0.3.3: - resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + for-each@0.3.3: dependencies: is-callable: 1.2.7 - dev: true - /foreground-child@3.1.1: - resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} - engines: {node: '>=14'} + foreground-child@3.1.1: dependencies: cross-spawn: 7.0.3 signal-exit: 4.1.0 - dev: true - /form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} - engines: {node: '>= 6'} + form-data@4.0.0: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: false - /fraction.js@4.3.7: - resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} - dev: true + fraction.js@4.3.7: {} - /fs.realpath@1.0.0: - resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} - dev: true + fs.realpath@1.0.0: {} - /fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - requiresBuild: true - dev: true + fsevents@2.3.3: optional: true - /function-bind@1.1.2: - resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + function-bind@1.1.2: {} - /function.prototype.name@1.1.6: - resolution: {integrity: sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==} - engines: {node: '>= 0.4'} + function.prototype.name@1.1.6: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 functions-have-names: 1.2.3 - dev: true - /functions-have-names@1.2.3: - resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} - dev: true + functions-have-names@1.2.3: {} - /get-intrinsic@1.2.2: - resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + get-intrinsic@1.2.2: dependencies: function-bind: 1.1.2 has-proto: 1.0.1 has-symbols: 1.0.3 hasown: 2.0.0 - /get-symbol-description@1.0.0: - resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} - engines: {node: '>= 0.4'} + get-symbol-description@1.0.0: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 - dev: true - /get-tsconfig@4.7.2: - resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + get-tsconfig@4.7.2: dependencies: resolve-pkg-maps: 1.0.0 - dev: true - /glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 - dev: true - /glob-parent@6.0.2: - resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} - engines: {node: '>=10.13.0'} + glob-parent@6.0.2: dependencies: is-glob: 4.0.3 - dev: true - /glob-to-regexp@0.4.1: - resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} - dev: false + glob-to-regexp@0.4.1: {} - /glob@10.3.10: - resolution: {integrity: sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==} - engines: {node: '>=16 || 14 >=14.17'} - hasBin: true + glob@10.3.10: dependencies: foreground-child: 3.1.1 jackspeak: 2.3.6 minimatch: 9.0.3 minipass: 7.0.4 path-scurry: 1.10.1 - dev: true - /glob@7.1.7: - resolution: {integrity: sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==} + glob@7.1.7: dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -2432,10 +4000,8 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 - dev: true - /glob@7.2.3: - resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + glob@7.2.3: dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -2443,25 +4009,16 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 - dev: true - /globals@13.24.0: - resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} - engines: {node: '>=8'} + globals@13.24.0: dependencies: type-fest: 0.20.2 - dev: true - /globalthis@1.0.3: - resolution: {integrity: sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==} - engines: {node: '>= 0.4'} + globalthis@1.0.3: dependencies: define-properties: 1.2.1 - dev: true - /globby@11.1.0: - resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} - engines: {node: '>=10'} + globby@11.1.0: dependencies: array-union: 2.1.0 dir-glob: 3.0.1 @@ -2469,539 +4026,309 @@ packages: ignore: 5.3.0 merge2: 1.4.1 slash: 3.0.0 - dev: true - /goober@2.1.13(csstype@3.1.3): - resolution: {integrity: sha512-jFj3BQeleOoy7t93E9rZ2de+ScC4lQICLwiAQmKMg9F6roKGaLSHoCDYKkWlSafg138jejvq/mTdvmnwDQgqoQ==} - peerDependencies: - csstype: ^3.0.10 + goober@2.1.13(csstype@3.1.3): dependencies: csstype: 3.1.3 - dev: false - /gopd@1.0.1: - resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + gopd@1.0.1: dependencies: get-intrinsic: 1.2.2 - /graceful-fs@4.2.11: - resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + graceful-fs@4.2.11: {} - /graphemer@1.4.0: - resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - dev: true + graphemer@1.4.0: {} - /gzip-size@6.0.0: - resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} - engines: {node: '>=10'} + gzip-size@6.0.0: dependencies: duplexer: 0.1.2 - dev: true - /has-bigints@1.0.2: - resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==} - dev: true + has-bigints@1.0.2: {} - /has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - dev: false + has-flag@3.0.0: {} - /has-flag@4.0.0: - resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} - engines: {node: '>=8'} - dev: true + has-flag@4.0.0: {} - /has-property-descriptors@1.0.1: - resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} + has-property-descriptors@1.0.1: dependencies: get-intrinsic: 1.2.2 - /has-proto@1.0.1: - resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} - engines: {node: '>= 0.4'} + has-proto@1.0.1: {} - /has-symbols@1.0.3: - resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} - engines: {node: '>= 0.4'} + has-symbols@1.0.3: {} - /has-tostringtag@1.0.0: - resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} - engines: {node: '>= 0.4'} + has-tostringtag@1.0.0: dependencies: has-symbols: 1.0.3 - dev: true - /hasown@2.0.0: - resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} - engines: {node: '>= 0.4'} + hasown@2.0.0: dependencies: function-bind: 1.1.2 - /hoist-non-react-statics@3.3.2: - resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1 - dev: false - /hyphenate-style-name@1.0.4: - resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==} - dev: false + hyphenate-style-name@1.0.4: {} - /ignore@5.3.0: - resolution: {integrity: sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==} - engines: {node: '>= 4'} - dev: true + ignore@5.3.0: {} - /immer@9.0.21: - resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} - dev: false + immer@9.0.21: {} - /import-fresh@3.3.0: - resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} - engines: {node: '>=6'} + import-fresh@3.3.0: dependencies: parent-module: 1.0.1 resolve-from: 4.0.0 - /imurmurhash@0.1.4: - resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} - engines: {node: '>=0.8.19'} - dev: true + imurmurhash@0.1.4: {} - /inflight@1.0.6: - resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + inflight@1.0.6: dependencies: once: 1.4.0 wrappy: 1.0.2 - dev: true - /inherits@2.0.4: - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - dev: true + inherits@2.0.4: {} - /inline-style-prefixer@7.0.0: - resolution: {integrity: sha512-I7GEdScunP1dQ6IM2mQWh6v0mOYdYmH3Bp31UecKdrcUgcURTcctSe1IECdUznSHKSmsHtjrT3CwCPI1pyxfUQ==} + inline-style-prefixer@7.0.0: dependencies: css-in-js-utils: 3.1.0 fast-loops: 1.1.3 - dev: false - /internal-slot@1.0.6: - resolution: {integrity: sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==} - engines: {node: '>= 0.4'} + internal-slot@1.0.6: dependencies: get-intrinsic: 1.2.2 hasown: 2.0.0 side-channel: 1.0.4 - dev: true - /is-array-buffer@3.0.2: - resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==} + is-array-buffer@3.0.2: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 is-typed-array: 1.1.12 - dev: true - /is-arrayish@0.2.1: - resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} - dev: false + is-arrayish@0.2.1: {} - /is-async-function@2.0.0: - resolution: {integrity: sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==} - engines: {node: '>= 0.4'} + is-async-function@2.0.0: dependencies: has-tostringtag: 1.0.0 - dev: true - /is-bigint@1.0.4: - resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} + is-bigint@1.0.4: dependencies: has-bigints: 1.0.2 - dev: true - /is-binary-path@2.1.0: - resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} - engines: {node: '>=8'} + is-binary-path@2.1.0: dependencies: binary-extensions: 2.2.0 - dev: true - /is-boolean-object@1.1.2: - resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} - engines: {node: '>= 0.4'} + is-boolean-object@1.1.2: dependencies: call-bind: 1.0.5 has-tostringtag: 1.0.0 - dev: true - /is-callable@1.2.7: - resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} - engines: {node: '>= 0.4'} - dev: true + is-callable@1.2.7: {} - /is-core-module@2.13.1: - resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} + is-core-module@2.13.1: dependencies: hasown: 2.0.0 - /is-date-object@1.0.5: - resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==} - engines: {node: '>= 0.4'} + is-date-object@1.0.5: dependencies: has-tostringtag: 1.0.0 - dev: true - /is-extglob@2.1.1: - resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} - engines: {node: '>=0.10.0'} - dev: true + is-extglob@2.1.1: {} - /is-finalizationregistry@1.0.2: - resolution: {integrity: sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==} + is-finalizationregistry@1.0.2: dependencies: call-bind: 1.0.5 - dev: true - /is-fullwidth-code-point@3.0.0: - resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} - engines: {node: '>=8'} - dev: true + is-fullwidth-code-point@3.0.0: {} - /is-generator-function@1.0.10: - resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} - engines: {node: '>= 0.4'} + is-generator-function@1.0.10: dependencies: has-tostringtag: 1.0.0 - dev: true - /is-glob@4.0.3: - resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} - engines: {node: '>=0.10.0'} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 - dev: true - /is-map@2.0.2: - resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} - dev: true + is-map@2.0.2: {} - /is-negative-zero@2.0.2: - resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} - engines: {node: '>= 0.4'} - dev: true + is-negative-zero@2.0.2: {} - /is-number-object@1.0.7: - resolution: {integrity: sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==} - engines: {node: '>= 0.4'} + is-number-object@1.0.7: dependencies: has-tostringtag: 1.0.0 - dev: true - /is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - dev: true + is-number@7.0.0: {} - /is-path-inside@3.0.3: - resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} - engines: {node: '>=8'} - dev: true + is-path-inside@3.0.3: {} - /is-regex@1.1.4: - resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} - engines: {node: '>= 0.4'} + is-regex@1.1.4: dependencies: call-bind: 1.0.5 has-tostringtag: 1.0.0 - dev: true - /is-set@2.0.2: - resolution: {integrity: sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==} - dev: true + is-set@2.0.2: {} - /is-shared-array-buffer@1.0.2: - resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==} + is-shared-array-buffer@1.0.2: dependencies: call-bind: 1.0.5 - dev: true - /is-string@1.0.7: - resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} - engines: {node: '>= 0.4'} + is-string@1.0.7: dependencies: has-tostringtag: 1.0.0 - dev: true - /is-symbol@1.0.4: - resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} - engines: {node: '>= 0.4'} + is-symbol@1.0.4: dependencies: has-symbols: 1.0.3 - dev: true - /is-typed-array@1.1.12: - resolution: {integrity: sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==} - engines: {node: '>= 0.4'} + is-typed-array@1.1.12: dependencies: which-typed-array: 1.1.13 - dev: true - /is-weakmap@2.0.1: - resolution: {integrity: sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==} - dev: true + is-weakmap@2.0.1: {} - /is-weakref@1.0.2: - resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} + is-weakref@1.0.2: dependencies: call-bind: 1.0.5 - dev: true - /is-weakset@2.0.2: - resolution: {integrity: sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==} + is-weakset@2.0.2: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 - dev: true - /isarray@2.0.5: - resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} - dev: true + isarray@2.0.5: {} - /isexe@2.0.0: - resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - dev: true + isexe@2.0.0: {} - /iterator.prototype@1.1.2: - resolution: {integrity: sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==} + iterator.prototype@1.1.2: dependencies: define-properties: 1.2.1 get-intrinsic: 1.2.2 has-symbols: 1.0.3 reflect.getprototypeof: 1.0.4 set-function-name: 2.0.1 - dev: true - /jackspeak@2.3.6: - resolution: {integrity: sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==} - engines: {node: '>=14'} + jackspeak@2.3.6: dependencies: '@isaacs/cliui': 8.0.2 optionalDependencies: '@pkgjs/parseargs': 0.11.0 - dev: true - /jiti@1.21.0: - resolution: {integrity: sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==} - hasBin: true - dev: true + jiti@1.21.0: {} - /js-cookie@2.2.1: - resolution: {integrity: sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==} - dev: false + js-cookie@2.2.1: {} - /js-tokens@4.0.0: - resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-tokens@4.0.0: {} - /js-yaml@4.1.0: - resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} - hasBin: true + js-yaml@4.1.0: dependencies: argparse: 2.0.1 - dev: true - /json-buffer@3.0.1: - resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} - dev: true + json-buffer@3.0.1: {} - /json-parse-even-better-errors@2.3.1: - resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} - dev: false + json-parse-even-better-errors@2.3.1: {} - /json-schema-traverse@0.4.1: - resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - dev: true + json-schema-traverse@0.4.1: {} - /json-stable-stringify-without-jsonify@1.0.1: - resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} - dev: true + json-stable-stringify-without-jsonify@1.0.1: {} - /json2mq@0.2.0: - resolution: {integrity: sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==} + json2mq@0.2.0: dependencies: string-convert: 0.2.1 - dev: false - /json5@1.0.2: - resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} - hasBin: true + json5@1.0.2: dependencies: minimist: 1.2.8 - dev: true - /jsx-ast-utils@3.3.5: - resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} - engines: {node: '>=4.0'} + jsx-ast-utils@3.3.5: dependencies: array-includes: 3.1.7 array.prototype.flat: 1.3.2 object.assign: 4.1.5 object.values: 1.1.7 - dev: true - /keyv@4.5.4: - resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 - dev: true - /language-subtag-registry@0.3.22: - resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} - dev: true + language-subtag-registry@0.3.22: {} - /language-tags@1.0.9: - resolution: {integrity: sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==} - engines: {node: '>=0.10'} + language-tags@1.0.9: dependencies: language-subtag-registry: 0.3.22 - dev: true - /levn@0.4.1: - resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} - engines: {node: '>= 0.8.0'} + levn@0.4.1: dependencies: prelude-ls: 1.2.1 type-check: 0.4.0 - dev: true - /lilconfig@2.1.0: - resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} - engines: {node: '>=10'} - dev: true + lilconfig@2.1.0: {} - /lilconfig@3.0.0: - resolution: {integrity: sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==} - engines: {node: '>=14'} - dev: true + lilconfig@3.0.0: {} - /lines-and-columns@1.2.4: - resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + lines-and-columns@1.2.4: {} - /locate-path@6.0.0: - resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} - engines: {node: '>=10'} + locate-path@6.0.0: dependencies: p-locate: 5.0.0 - dev: true - /lodash-es@4.17.21: - resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} - dev: false + lodash-es@4.17.21: {} - /lodash.merge@4.6.2: - resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} - dev: true + lodash.merge@4.6.2: {} - /lodash@4.17.21: - resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} - dev: true + lodash@4.17.21: {} - /loose-envify@1.4.0: - resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} - hasBin: true + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 - /lru-cache@10.1.0: - resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==} - engines: {node: 14 || >=16.14} - dev: true + lru-cache@10.1.0: {} - /lru-cache@6.0.0: - resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} - engines: {node: '>=10'} + lru-cache@6.0.0: dependencies: yallist: 4.0.0 - dev: true - /mdn-data@2.0.14: - resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} - dev: false + mdn-data@2.0.14: {} - /merge2@1.4.1: - resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} - engines: {node: '>= 8'} - dev: true + merge2@1.4.1: {} - /micromatch@4.0.5: - resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} - engines: {node: '>=8.6'} + micromatch@4.0.5: dependencies: braces: 3.0.2 picomatch: 2.3.1 - dev: true - /mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: false + mime-db@1.52.0: {} - /mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} + mime-types@2.1.35: dependencies: mime-db: 1.52.0 - dev: false - /minimatch@3.1.2: - resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 - dev: true - /minimatch@9.0.3: - resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} - engines: {node: '>=16 || 14 >=14.17'} + minimatch@9.0.3: dependencies: brace-expansion: 2.0.1 - dev: true - /minimist@1.2.8: - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: true + minimist@1.2.8: {} - /minipass@7.0.4: - resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==} - engines: {node: '>=16 || 14 >=14.17'} - dev: true + minipass@7.0.4: {} - /mrmime@1.0.1: - resolution: {integrity: sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==} - engines: {node: '>=10'} - dev: true + mrmime@1.0.1: {} - /ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true + ms@2.1.2: {} - /ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - dev: true + ms@2.1.3: {} - /mz@2.7.0: - resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + mz@2.7.0: dependencies: any-promise: 1.3.0 object-assign: 4.1.1 thenify-all: 1.6.0 - dev: true - /nano-css@5.6.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-T2Mhc//CepkTa3X4pUhKgbEheJHYAxD0VptuqFhDbGMUWVV2m+lkNiW/Ieuj35wrfC8Zm0l7HvssQh7zcEttSw==} - peerDependencies: - react: '*' - react-dom: '*' + nano-css@5.6.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@jridgewell/sourcemap-codec': 1.4.15 css-tree: 1.1.3 @@ -3013,31 +4340,12 @@ packages: rtl-css-js: 1.16.1 stacktrace-js: 2.0.2 stylis: 4.3.1 - dev: false - /nanoid@3.3.7: - resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true + nanoid@3.3.7: {} - /natural-compare@1.4.0: - resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} - dev: true + natural-compare@1.4.0: {} - /next@14.0.3(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-AbYdRNfImBr3XGtvnwOxq8ekVCwbFTv/UJoLwmaX89nk9i051AEY4/HAWzU0YpaTDw8IofUpmuIlvzWF13jxIw==} - engines: {node: '>=18.17.0'} - hasBin: true - peerDependencies: - '@opentelemetry/api': ^1.1.0 - react: ^18.2.0 - react-dom: ^18.2.0 - sass: ^1.3.0 - peerDependenciesMeta: - '@opentelemetry/api': - optional: true - sass: - optional: true + next@14.0.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@next/env': 14.0.3 '@swc/helpers': 0.5.2 @@ -3061,110 +4369,67 @@ packages: transitivePeerDependencies: - '@babel/core' - babel-plugin-macros - dev: false - - /node-releases@2.0.14: - resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} - dev: true - /normalize-path@3.0.0: - resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} - engines: {node: '>=0.10.0'} - dev: true + node-releases@2.0.14: {} - /normalize-range@0.1.2: - resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} - engines: {node: '>=0.10.0'} - dev: true + normalize-path@3.0.0: {} - /nprogress@0.2.0: - resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} - dev: false + normalize-range@0.1.2: {} - /object-assign@4.1.1: - resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} - engines: {node: '>=0.10.0'} + nprogress@0.2.0: {} - /object-hash@3.0.0: - resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} - engines: {node: '>= 6'} - dev: true + object-assign@4.1.1: {} - /object-inspect@1.13.1: - resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + object-hash@3.0.0: {} - /object-keys@1.1.1: - resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} - engines: {node: '>= 0.4'} - dev: true + object-inspect@1.13.1: {} - /object.assign@4.1.5: - resolution: {integrity: sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==} - engines: {node: '>= 0.4'} + object-keys@1.1.1: {} + + object.assign@4.1.5: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 has-symbols: 1.0.3 object-keys: 1.1.1 - dev: true - /object.entries@1.1.7: - resolution: {integrity: sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==} - engines: {node: '>= 0.4'} + object.entries@1.1.7: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /object.fromentries@2.0.7: - resolution: {integrity: sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==} - engines: {node: '>= 0.4'} + object.fromentries@2.0.7: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /object.groupby@1.0.1: - resolution: {integrity: sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ==} + object.groupby@1.0.1: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 get-intrinsic: 1.2.2 - dev: true - /object.hasown@1.1.3: - resolution: {integrity: sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA==} + object.hasown@1.1.3: dependencies: define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /object.values@1.1.7: - resolution: {integrity: sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==} - engines: {node: '>= 0.4'} + object.values@1.1.7: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /once@1.4.0: - resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + once@1.4.0: dependencies: wrappy: 1.0.2 - dev: true - /opener@1.5.2: - resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} - hasBin: true - dev: true + opener@1.5.2: {} - /optionator@0.9.3: - resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==} - engines: {node: '>= 0.8.0'} + optionator@0.9.3: dependencies: '@aashutoshrathi/word-wrap': 1.2.6 deep-is: 0.1.4 @@ -3172,803 +4437,472 @@ packages: levn: 0.4.1 prelude-ls: 1.2.1 type-check: 0.4.0 - dev: true - /p-limit@3.1.0: - resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} - engines: {node: '>=10'} + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 - dev: true - /p-locate@5.0.0: - resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} - engines: {node: '>=10'} + p-locate@5.0.0: dependencies: p-limit: 3.1.0 - dev: true - /parent-module@1.0.1: - resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} - engines: {node: '>=6'} + parent-module@1.0.1: dependencies: callsites: 3.1.0 - /parse-json@5.2.0: - resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} - engines: {node: '>=8'} + parse-json@5.2.0: dependencies: '@babel/code-frame': 7.23.5 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 - dev: false - /path-exists@4.0.0: - resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} - engines: {node: '>=8'} - dev: true + path-exists@4.0.0: {} - /path-is-absolute@1.0.1: - resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} - engines: {node: '>=0.10.0'} - dev: true + path-is-absolute@1.0.1: {} - /path-key@3.1.1: - resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} - engines: {node: '>=8'} - dev: true + path-key@3.1.1: {} - /path-parse@1.0.7: - resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-parse@1.0.7: {} - /path-scurry@1.10.1: - resolution: {integrity: sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==} - engines: {node: '>=16 || 14 >=14.17'} + path-scurry@1.10.1: dependencies: lru-cache: 10.1.0 minipass: 7.0.4 - dev: true - /path-type@4.0.0: - resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} - engines: {node: '>=8'} + path-type@4.0.0: {} - /picocolors@1.0.0: - resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} + picocolors@1.0.0: {} - /picomatch@2.3.1: - resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} - engines: {node: '>=8.6'} - dev: true + picomatch@2.3.1: {} - /pify@2.3.0: - resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} - engines: {node: '>=0.10.0'} - dev: true + pify@2.3.0: {} - /pirates@4.0.6: - resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} - engines: {node: '>= 6'} - dev: true + pirates@4.0.6: {} - /postcss-import@15.1.0(postcss@8.4.33): - resolution: {integrity: sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==} - engines: {node: '>=14.0.0'} - peerDependencies: - postcss: ^8.0.0 + postcss-import@15.1.0(postcss@8.4.33): dependencies: postcss: 8.4.33 postcss-value-parser: 4.2.0 read-cache: 1.0.0 resolve: 1.22.8 - dev: true - /postcss-js@4.0.1(postcss@8.4.33): - resolution: {integrity: sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==} - engines: {node: ^12 || ^14 || >= 16} - peerDependencies: - postcss: ^8.4.21 + postcss-js@4.0.1(postcss@8.4.33): dependencies: camelcase-css: 2.0.1 postcss: 8.4.33 - dev: true - /postcss-load-config@4.0.2(postcss@8.4.33): - resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} - engines: {node: '>= 14'} - peerDependencies: - postcss: '>=8.0.9' - ts-node: '>=9.0.0' - peerDependenciesMeta: - postcss: - optional: true - ts-node: - optional: true + postcss-load-config@4.0.2(postcss@8.4.33): dependencies: lilconfig: 3.0.0 - postcss: 8.4.33 yaml: 2.3.4 - dev: true + optionalDependencies: + postcss: 8.4.33 - /postcss-nested@6.0.1(postcss@8.4.33): - resolution: {integrity: sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==} - engines: {node: '>=12.0'} - peerDependencies: - postcss: ^8.2.14 + postcss-nested@6.0.1(postcss@8.4.33): dependencies: postcss: 8.4.33 postcss-selector-parser: 6.0.15 - dev: true - /postcss-selector-parser@6.0.15: - resolution: {integrity: sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==} - engines: {node: '>=4'} + postcss-selector-parser@6.0.15: dependencies: cssesc: 3.0.0 util-deprecate: 1.0.2 - dev: true - /postcss-value-parser@4.2.0: - resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} - dev: true + postcss-value-parser@4.2.0: {} - /postcss@8.4.31: - resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==} - engines: {node: ^10 || ^12 || >=14} + postcss@8.4.31: dependencies: nanoid: 3.3.7 picocolors: 1.0.0 source-map-js: 1.0.2 - dev: false - /postcss@8.4.33: - resolution: {integrity: sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==} - engines: {node: ^10 || ^12 || >=14} + postcss@8.4.33: dependencies: nanoid: 3.3.7 picocolors: 1.0.0 source-map-js: 1.0.2 - dev: true - /prelude-ls@1.2.1: - resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} - engines: {node: '>= 0.8.0'} - dev: true + prelude-ls@1.2.1: {} - /prettier@3.1.1: - resolution: {integrity: sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==} - engines: {node: '>=14'} - hasBin: true - dev: true + prettier@3.1.1: {} - /prop-types@15.8.1: - resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + prop-types@15.8.1: dependencies: loose-envify: 1.4.0 object-assign: 4.1.1 react-is: 16.13.1 - /property-expr@2.0.6: - resolution: {integrity: sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA==} - dev: false + property-expr@2.0.6: {} - /proxy-from-env@1.1.0: - resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: false + proxy-from-env@1.1.0: {} - /punycode@2.3.1: - resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} - engines: {node: '>=6'} - dev: true + punycode@2.3.1: {} - /qrcode.react@3.1.0(react@18.2.0): - resolution: {integrity: sha512-oyF+Urr3oAMUG/OiOuONL3HXM+53wvuH3mtIWQrYmsXoAq0DkvZp2RYUWFSMFtbdOpuS++9v+WAkzNVkMlNW6Q==} - peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + qrcode.react@3.1.0(react@18.2.0): dependencies: react: 18.2.0 - dev: false - /qs@6.11.2: - resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} - engines: {node: '>=0.6'} + qs@6.11.2: dependencies: side-channel: 1.0.4 - dev: false - /queue-microtask@1.2.3: - resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - dev: true + queue-microtask@1.2.3: {} - /rc-cascader@3.21.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-J7GozpgsLaOtzfIHFJFuh4oFY0ePb1w10twqK6is3pAkqHkca/PsokbDr822KIRZ8/CK8CqevxohuPDVZ1RO/A==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-cascader@3.21.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 array-tree-filter: 2.1.0 classnames: 2.5.1 - rc-select: 14.11.0(react-dom@18.2.0)(react@18.2.0) - rc-tree: 5.8.5(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-select: 14.11.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tree: 5.8.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-checkbox@3.1.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-PAwpJFnBa3Ei+5pyqMMXdcKYKNBMS+TvSDiLdDnARnMJHC8ESxwPfm4Ao1gJiKtWLdmGfigascnCpwrHFgoOBQ==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-checkbox@3.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-collapse@3.7.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-ZRw6ipDyOnfLFySxAiCMdbHtb5ePAsB9mT17PA6y1mRD/W6KHRaZeb5qK/X9xDV1CqgyxMpzw0VdS74PCcUk4A==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-collapse@3.7.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-dialog@9.3.4(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-975X3018GhR+EjZFbxA2Z57SX5rnu0G0/OxFgMMvZK4/hQWEm3MHaNvP4wXpxYDoJsp+xUvVW+GB9CMMCm81jA==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-dialog@9.3.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/portal': 1.1.2(react-dom@18.2.0)(react@18.2.0) + '@rc-component/portal': 1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-drawer@7.0.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-ePcS4KtQnn57bCbVXazHN2iC8nTPCXlWEIA/Pft87Pd9U7ZeDkdRzG47jWG2/TAFXFlFltRAMcslqmUM8NPCGA==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-drawer@7.0.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/portal': 1.1.2(react-dom@18.2.0)(react@18.2.0) + '@rc-component/portal': 1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-dropdown@4.1.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-VZjMunpBdlVzYpEdJSaV7WM7O0jf8uyDjirxXLZRNZ+tAC+NzD3PXPEtliFwGzVwBBdCmGuSqiS9DWcOLxQ9tw==} - peerDependencies: - react: '>=16.11.0' - react-dom: '>=16.11.0' + rc-dropdown@4.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-field-form@1.41.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-k9AS0wmxfJfusWDP/YXWTpteDNaQ4isJx9UKxx4/e8Dub4spFeZ54/EuN2sYrMRID/+hUznPgVZeg+Gf7XSYCw==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-field-form@1.41.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 async-validator: 4.2.5 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-image@7.5.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-Z9loECh92SQp0nSipc0MBuf5+yVC05H/pzC+Nf8xw1BKDFUJzUeehYBjaWlxly8VGBZJcTHYri61Fz9ng1G3Ag==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-image@7.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/portal': 1.1.2(react-dom@18.2.0)(react@18.2.0) + '@rc-component/portal': 1.1.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-dialog: 9.3.4(react-dom@18.2.0)(react@18.2.0) - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-dialog: 9.3.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-input-number@8.6.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-gaAMUKtUKLktJ3Yx93tjgYY1M0HunnoqzPEqkb9//Ydup4DcG0TFL9yHBA3pgVdNIt5f0UWyHCgFBj//JxeD6A==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-input-number@8.6.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 '@rc-component/mini-decimal': 1.1.0 classnames: 2.5.1 - rc-input: 1.4.3(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-input: 1.4.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-input@1.4.3(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-aHyQUAIRmTlOnvk5EcNqEpJ+XMtfMpYRAJayIlJfsvvH9cAKUWboh4egm23vgMA7E+c/qm4BZcnrDcA960GC1w==} - peerDependencies: - react: '>=16.0.0' - react-dom: '>=16.0.0' + rc-input@1.4.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-mentions@2.10.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-72qsEcr/7su+a07ndJ1j8rI9n0Ka/ngWOLYnWMMv0p2mi/5zPwPrEDTt6Uqpe8FWjWhueDJx/vzunL6IdKDYMg==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-mentions@2.10.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-input: 1.4.3(react-dom@18.2.0)(react@18.2.0) - rc-menu: 9.12.4(react-dom@18.2.0)(react@18.2.0) - rc-textarea: 1.6.3(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-input: 1.4.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-menu: 9.12.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-textarea: 1.6.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-menu@9.12.4(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-t2NcvPLV1mFJzw4F21ojOoRVofK2rWhpKPx69q2raUsiHPDP6DDevsBILEYdsIegqBeSXoWs2bf6CueBKg3BFg==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-menu@9.12.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-overflow: 1.3.2(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-overflow: 1.3.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-motion@2.9.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-XIU2+xLkdIr1/h6ohPZXyPBMvOmuyFZQ/T0xnawz+Rh+gh4FINcnZmMT5UTIj6hgI0VLDjTaPeRd+smJeSPqiQ==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-motion@2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-notification@5.3.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-WCf0uCOkZ3HGfF0p1H4Sgt7aWfipxORWTPp7o6prA3vxwtWhtug3GfpYls1pnBp4WA+j8vGIi5c2/hQRpGzPcQ==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-notification@5.3.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-overflow@1.3.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-nsUm78jkYAoPygDAcGZeC2VwIg/IBGSodtOY3pMof4W3M9qRJgqaDYm03ZayHlde3I6ipliAxbN0RUcGf5KOzw==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-overflow@1.3.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-pagination@4.0.4(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-GGrLT4NgG6wgJpT/hHIpL9nELv27A1XbSZzECIuQBQTVSf4xGKxWr6I/jhpRPauYEWEbWVw22ObG6tJQqwJqWQ==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-pagination@4.0.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-picker@3.14.6(dayjs@1.11.10)(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-AdKKW0AqMwZsKvIpwUWDUnpuGKZVrbxVTZTNjcO+pViGkjC1EBcjMgxVe8tomOEaIHJL5Gd13vS8Rr3zzxWmag==} - engines: {node: '>=8.x'} - peerDependencies: - date-fns: '>= 2.x' - dayjs: '>= 1.x' - luxon: '>= 3.x' - moment: '>= 2.x' - react: '>=16.9.0' - react-dom: '>=16.9.0' - peerDependenciesMeta: - date-fns: - optional: true - dayjs: - optional: true - luxon: - optional: true - moment: - optional: true + rc-picker@3.14.6(dayjs@1.11.10)(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - dayjs: 1.11.10 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false + optionalDependencies: + dayjs: 1.11.10 - /rc-progress@3.5.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-V6Amx6SbLRwPin/oD+k1vbPrO8+9Qf8zW1T8A7o83HdNafEVvAxPV5YsgtKFP+Ud5HghLj33zKOcEHrcrUGkfw==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-progress@3.5.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-rate@2.12.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-g092v5iZCdVzbjdn28FzvWebK2IutoVoiTeqoLTj9WM7SjA/gOJIw5/JFZMRyJYYVe1jLAU2UhAfstIpCNRozg==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-rate@2.12.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-resize-observer@1.4.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-PnMVyRid9JLxFavTjeDXEXo65HCRqbmLBw9xX9gfC4BZiSzbLXKzW3jPz+J0P71pLbD5tBMTT+mkstV5gD0c9Q==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-resize-observer@1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) resize-observer-polyfill: 1.5.1 - dev: false - /rc-segmented@2.2.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-Mq52M96QdHMsNdE/042ibT5vkcGcD5jxKp7HgPC2SRofpia99P5fkfHy1pEaajLMF/kj0+2Lkq1UZRvqzo9mSA==} - peerDependencies: - react: '>=16.0.0' - react-dom: '>=16.0.0' + rc-segmented@2.2.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-select@14.11.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-8J8G/7duaGjFiTXCBLWfh5P+KDWyA3KTlZDfV3xj/asMPqB2cmxfM+lH50wRiPIRsCQ6EbkCFBccPuaje3DHIg==} - engines: {node: '>=8.x'} - peerDependencies: - react: '*' - react-dom: '*' + rc-select@14.11.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-overflow: 1.3.2(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - rc-virtual-list: 3.11.3(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-overflow: 1.3.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-virtual-list: 3.11.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-slider@10.5.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-xiYght50cvoODZYI43v3Ylsqiw14+D7ELsgzR40boDZaya1HFa1Etnv9MDkQE8X/UrXAffwv2AcNAhslgYuDTw==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-slider@10.5.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-steps@6.0.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-lKHL+Sny0SeHkQKKDJlAjV5oZ8DwCdS2hFhAkIjuQt1/pB81M0cA0ErVFdHq9+jmPmFw1vJB2F5NBzFXLJxV+g==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-steps@6.0.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-switch@4.1.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-TI8ufP2Az9oEbvyCeVE4+90PDSljGyuwix3fV58p7HV2o4wBnVToEyomJRVyTaZeqNPAp+vqeo4Wnj5u0ZZQBg==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-switch@4.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-table@7.37.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-hEB17ktLRVfVmdo+U8MjGr+PuIgdQ8Cxj/N5lwMvP/Az7TOrQxwTMLVEDoj207tyPYLTWifHIF9EJREWwyk67g==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-table@7.37.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/context': 1.4.0(react-dom@18.2.0)(react@18.2.0) + '@rc-component/context': 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - rc-virtual-list: 3.11.3(react-dom@18.2.0)(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-virtual-list: 3.11.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-tabs@14.0.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-lp1YWkaPnjlyhOZCPrAWxK6/P6nMGX/BAZcAC3nuVwKz0Byfp+vNnQKK8BRCP2g/fzu+SeB5dm9aUigRu3tRkQ==} - engines: {node: '>=8.x'} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-tabs@14.0.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-dropdown: 4.1.0(react-dom@18.2.0)(react@18.2.0) - rc-menu: 9.12.4(react-dom@18.2.0)(react@18.2.0) - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-dropdown: 4.1.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-menu: 9.12.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-textarea@1.6.3(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-8k7+8Y2GJ/cQLiClFMg8kUXOOdvcFQrnGeSchOvI2ZMIVvX5a3zQpLxoODL0HTrvU63fPkRmMuqaEcOF9dQemA==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-textarea@1.6.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-input: 1.4.3(react-dom@18.2.0)(react@18.2.0) - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-input: 1.4.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-tooltip@6.1.3(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-HMSbSs5oieZ7XddtINUddBLSVgsnlaSb3bZrzzGWjXa7/B7nNedmsuz72s7EWFEro9mNa7RyF3gOXKYqvJiTcQ==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-tooltip@6.1.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 - '@rc-component/trigger': 1.18.3(react-dom@18.2.0)(react@18.2.0) + '@rc-component/trigger': 1.18.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) classnames: 2.5.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-tree-select@5.17.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-7sRGafswBhf7n6IuHyCEFCildwQIgyKiV8zfYyUoWfZEFdhuk7lCH+DN0aHt+oJrdiY9+6Io/LDXloGe01O8XQ==} - peerDependencies: - react: '*' - react-dom: '*' + rc-tree-select@5.17.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-select: 14.11.0(react-dom@18.2.0)(react@18.2.0) - rc-tree: 5.8.5(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-select: 14.11.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-tree: 5.8.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-tree@5.8.5(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-PRfcZtVDNkR7oh26RuNe1hpw11c1wfgzwmPFL0lnxGnYefe9lDAO6cg5wJKIAwyXFVt5zHgpjYmaz0CPy1ZtKg==} - engines: {node: '>=10.x'} - peerDependencies: - react: '*' - react-dom: '*' + rc-tree@5.8.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-motion: 2.9.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) - rc-virtual-list: 3.11.3(react-dom@18.2.0)(react@18.2.0) + rc-motion: 2.9.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-virtual-list: 3.11.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-upload@4.5.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-QO3ne77DwnAPKFn0bA5qJM81QBjQi0e0NHdkvpFyY73Bea2NfITiotqJqVjHgeYPOJu5lLVR32TNGP084aSoXA==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-upload@4.5.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /rc-util@5.38.1(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-e4ZMs7q9XqwTuhIK7zBIVFltUtMSjphuPPQXHoHlzRzNdOwUxDejo0Zls5HYaJfRKNURcsS/ceKVULlhjBrxng==} - peerDependencies: - react: '>=16.9.0' - react-dom: '>=16.9.0' + rc-util@5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) react-is: 18.2.0 - dev: false - /rc-virtual-list@3.11.3(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-tu5UtrMk/AXonHwHxUogdXAWynaXsrx1i6dsgg+lOo/KJSF8oBAcprh1z5J3xgnPJD5hXxTL58F8s8onokdt0Q==} - engines: {node: '>=8.x'} - peerDependencies: - react: '*' - react-dom: '*' + rc-virtual-list@3.11.3(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 classnames: 2.5.1 - rc-resize-observer: 1.4.0(react-dom@18.2.0)(react@18.2.0) - rc-util: 5.38.1(react-dom@18.2.0)(react@18.2.0) + rc-resize-observer: 1.4.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rc-util: 5.38.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /react-dom@18.2.0(react@18.2.0): - resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} - peerDependencies: - react: ^18.2.0 + react-dom@18.2.0(react@18.2.0): dependencies: loose-envify: 1.4.0 react: 18.2.0 scheduler: 0.23.0 - dev: false - /react-hook-form@7.49.3(react@18.2.0): - resolution: {integrity: sha512-foD6r3juidAT1cOZzpmD/gOKt7fRsDhXXZ0y28+Al1CHgX+AY1qIN9VSIIItXRq1dN68QrRwl1ORFlwjBaAqeQ==} - engines: {node: '>=18', pnpm: '8'} - peerDependencies: - react: ^16.8.0 || ^17 || ^18 + react-hook-form@7.51.3(react@18.2.0): dependencies: react: 18.2.0 - dev: false - /react-hot-toast@2.4.1(csstype@3.1.3)(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-j8z+cQbWIM5LY37pR6uZR6D4LfseplqnuAO4co4u8917hBUvXlEqyP1ZzqVLcqoyUesZZv/ImreoCeHVDpE5pQ==} - engines: {node: '>=10'} - peerDependencies: - react: '>=16' - react-dom: '>=16' + react-hot-toast@2.4.1(csstype@3.1.3)(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: goober: 2.1.13(csstype@3.1.3) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) transitivePeerDependencies: - csstype - dev: false - /react-is@16.13.1: - resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + react-is@16.13.1: {} - /react-is@18.2.0: - resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} - dev: false + react-is@18.2.0: {} - /react-redux@8.1.3(@types/react@18.2.47)(react-dom@18.2.0)(react@18.2.0)(redux@4.2.1): - resolution: {integrity: sha512-n0ZrutD7DaX/j9VscF+uTALI3oUPa/pO4Z3soOBIjuRn/FzVu6aehhysxZCLi6y7duMf52WNZGMl7CtuK5EnRw==} - peerDependencies: - '@types/react': ^16.8 || ^17.0 || ^18.0 - '@types/react-dom': ^16.8 || ^17.0 || ^18.0 - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 - react-native: '>=0.59' - redux: ^4 || ^5.0.0-beta.0 - peerDependenciesMeta: - '@types/react': - optional: true - '@types/react-dom': - optional: true - react-dom: - optional: true - react-native: - optional: true - redux: - optional: true + react-redux@8.1.3(@types/react@18.2.47)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(redux@4.2.1): dependencies: '@babel/runtime': 7.23.8 '@types/hoist-non-react-statics': 3.3.5 - '@types/react': 18.2.47 '@types/use-sync-external-store': 0.0.3 hoist-non-react-statics: 3.3.2 react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) react-is: 18.2.0 - redux: 4.2.1 use-sync-external-store: 1.2.0(react@18.2.0) - dev: false + optionalDependencies: + '@types/react': 18.2.47 + react-dom: 18.2.0(react@18.2.0) + redux: 4.2.1 - /react-transition-group@4.4.5(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==} - peerDependencies: - react: '>=16.6.0' - react-dom: '>=16.6.0' + react-transition-group@4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.23.8 dom-helpers: 5.2.1 @@ -3976,23 +4910,13 @@ packages: prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - dev: false - /react-universal-interface@0.6.2(react@18.2.0)(tslib@2.6.2): - resolution: {integrity: sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw==} - peerDependencies: - react: '*' - tslib: '*' + react-universal-interface@0.6.2(react@18.2.0)(tslib@2.6.2): dependencies: react: 18.2.0 tslib: 2.6.2 - dev: false - /react-use@17.4.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-1jPtmWLD8OJJNYCdYLJEH/HM+bPDfJuyGwCYeJFgPmWY8ttwpgZnW5QnzgM55CYUByUiTjHxsGOnEpLl6yQaoQ==} - peerDependencies: - react: '*' - react-dom: '*' + react-use@17.4.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@types/js-cookie': 2.2.7 '@xobotyi/scrollbar-width': 1.9.5 @@ -4000,7 +4924,7 @@ packages: fast-deep-equal: 3.1.3 fast-shallow-equal: 1.0.0 js-cookie: 2.2.1 - nano-css: 5.6.1(react-dom@18.2.0)(react@18.2.0) + nano-css: 5.6.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) react-universal-interface: 0.6.2(react@18.2.0)(tslib@2.6.2) @@ -4010,44 +4934,28 @@ packages: throttle-debounce: 3.0.1 ts-easing: 0.2.0 tslib: 2.6.2 - dev: false - /react@18.2.0: - resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==} - engines: {node: '>=0.10.0'} + react@18.2.0: dependencies: loose-envify: 1.4.0 - /read-cache@1.0.0: - resolution: {integrity: sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==} + read-cache@1.0.0: dependencies: pify: 2.3.0 - dev: true - /readdirp@3.6.0: - resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} - engines: {node: '>=8.10.0'} + readdirp@3.6.0: dependencies: picomatch: 2.3.1 - dev: true - /redux-thunk@2.4.2(redux@4.2.1): - resolution: {integrity: sha512-+P3TjtnP0k/FEjcBL5FZpoovtvrTNT/UXd4/sluaSyrURlSlhLSzEdfsTBW7WsKB6yPvgd7q/iZPICFjW4o57Q==} - peerDependencies: - redux: ^4 + redux-thunk@2.4.2(redux@4.2.1): dependencies: redux: 4.2.1 - dev: false - /redux@4.2.1: - resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} + redux@4.2.1: dependencies: '@babel/runtime': 7.23.8 - dev: false - /reflect.getprototypeof@1.0.4: - resolution: {integrity: sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw==} - engines: {node: '>= 0.4'} + reflect.getprototypeof@1.0.4: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 @@ -4055,259 +4963,157 @@ packages: get-intrinsic: 1.2.2 globalthis: 1.0.3 which-builtin-type: 1.1.3 - dev: true - /regenerator-runtime@0.14.1: - resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} + regenerator-runtime@0.14.1: {} - /regexp.prototype.flags@1.5.1: - resolution: {integrity: sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==} - engines: {node: '>= 0.4'} + regexp.prototype.flags@1.5.1: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 set-function-name: 2.0.1 - dev: true - /reselect@4.1.8: - resolution: {integrity: sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ==} - dev: false + reselect@4.1.8: {} - /resize-observer-polyfill@1.5.1: - resolution: {integrity: sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==} - dev: false + resize-observer-polyfill@1.5.1: {} - /resolve-from@4.0.0: - resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} - engines: {node: '>=4'} + resolve-from@4.0.0: {} - /resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - dev: true + resolve-pkg-maps@1.0.0: {} - /resolve@1.22.8: - resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} - hasBin: true + resolve@1.22.8: dependencies: is-core-module: 2.13.1 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - /resolve@2.0.0-next.5: - resolution: {integrity: sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==} - hasBin: true + resolve@2.0.0-next.5: dependencies: is-core-module: 2.13.1 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - dev: true - /reusify@1.0.4: - resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} - engines: {iojs: '>=1.0.0', node: '>=0.10.0'} - dev: true + reusify@1.0.4: {} - /rimraf@3.0.2: - resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} - hasBin: true + rimraf@3.0.2: dependencies: glob: 7.2.3 - dev: true - /rtl-css-js@1.16.1: - resolution: {integrity: sha512-lRQgou1mu19e+Ya0LsTvKrVJ5TYUbqCVPAiImX3UfLTenarvPUl1QFdvu5Z3PYmHT9RCcwIfbjRQBntExyj3Zg==} + rtl-css-js@1.16.1: dependencies: '@babel/runtime': 7.23.8 - dev: false - /run-parallel@1.2.0: - resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 - dev: true - /safe-array-concat@1.0.1: - resolution: {integrity: sha512-6XbUAseYE2KtOuGueyeobCySj9L4+66Tn6KQMOPQJrAJEowYKW/YR/MGJZl7FdydUdaFu4LYyDZjxf4/Nmo23Q==} - engines: {node: '>=0.4'} + safe-array-concat@1.0.1: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 has-symbols: 1.0.3 isarray: 2.0.5 - dev: true - /safe-regex-test@1.0.0: - resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} + safe-regex-test@1.0.0: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 is-regex: 1.1.4 - dev: true - /scheduler@0.23.0: - resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} + scheduler@0.23.0: dependencies: loose-envify: 1.4.0 - dev: false - /screenfull@5.2.0: - resolution: {integrity: sha512-9BakfsO2aUQN2K9Fdbj87RJIEZ82Q9IGim7FqM5OsebfoFC6ZHXgDq/KvniuLTPdeM8wY2o6Dj3WQ7KeQCj3cA==} - engines: {node: '>=0.10.0'} - dev: false + screenfull@5.2.0: {} - /scroll-into-view-if-needed@3.1.0: - resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + scroll-into-view-if-needed@3.1.0: dependencies: compute-scroll-into-view: 3.1.0 - dev: false - /semver@6.3.1: - resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} - hasBin: true - dev: true + semver@6.3.1: {} - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} - engines: {node: '>=10'} - hasBin: true + semver@7.5.4: dependencies: lru-cache: 6.0.0 - dev: true - /set-function-length@1.1.1: - resolution: {integrity: sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==} - engines: {node: '>= 0.4'} + set-function-length@1.1.1: dependencies: define-data-property: 1.1.1 get-intrinsic: 1.2.2 gopd: 1.0.1 has-property-descriptors: 1.0.1 - /set-function-name@2.0.1: - resolution: {integrity: sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==} - engines: {node: '>= 0.4'} + set-function-name@2.0.1: dependencies: define-data-property: 1.1.1 functions-have-names: 1.2.3 has-property-descriptors: 1.0.1 - dev: true - /set-harmonic-interval@1.0.1: - resolution: {integrity: sha512-AhICkFV84tBP1aWqPwLZqFvAwqEoVA9kxNMniGEUvzOlm4vLmOFLiTT3UZ6bziJTy4bOVpzWGTfSCbmaayGx8g==} - engines: {node: '>=6.9'} - dev: false + set-harmonic-interval@1.0.1: {} - /shebang-command@2.0.0: - resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} - engines: {node: '>=8'} + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 - dev: true - /shebang-regex@3.0.0: - resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} - engines: {node: '>=8'} - dev: true + shebang-regex@3.0.0: {} - /side-channel@1.0.4: - resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + side-channel@1.0.4: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 object-inspect: 1.13.1 - /signal-exit@4.1.0: - resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} - engines: {node: '>=14'} - dev: true + signal-exit@4.1.0: {} - /sirv@1.0.19: - resolution: {integrity: sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ==} - engines: {node: '>= 10'} + sirv@1.0.19: dependencies: '@polka/url': 1.0.0-next.24 mrmime: 1.0.1 totalist: 1.1.0 - dev: true - /slash@3.0.0: - resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} - engines: {node: '>=8'} - dev: true + slash@3.0.0: {} - /source-map-js@1.0.2: - resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} - engines: {node: '>=0.10.0'} + source-map-js@1.0.2: {} - /source-map@0.5.6: - resolution: {integrity: sha512-MjZkVp0NHr5+TPihLcadqnlVoGIoWo4IBHptutGh9wI3ttUYvCG26HkSuDi+K6lsZ25syXJXcctwgyVCt//xqA==} - engines: {node: '>=0.10.0'} - dev: false + source-map@0.5.6: {} - /source-map@0.5.7: - resolution: {integrity: sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==} - engines: {node: '>=0.10.0'} - dev: false + source-map@0.5.7: {} - /source-map@0.6.1: - resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} - engines: {node: '>=0.10.0'} - dev: false + source-map@0.6.1: {} - /stack-generator@2.0.10: - resolution: {integrity: sha512-mwnua/hkqM6pF4k8SnmZ2zfETsRUpWXREfA/goT8SLCV4iOFa4bzOX2nDipWAZFPTjLvQB82f5yaodMVhK0yJQ==} + stack-generator@2.0.10: dependencies: stackframe: 1.3.4 - dev: false - /stackframe@1.3.4: - resolution: {integrity: sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==} - dev: false + stackframe@1.3.4: {} - /stacktrace-gps@3.1.2: - resolution: {integrity: sha512-GcUgbO4Jsqqg6RxfyTHFiPxdPqF+3LFmQhm7MgCuYQOYuWyqxo5pwRPz5d/u6/WYJdEnWfK4r+jGbyD8TSggXQ==} + stacktrace-gps@3.1.2: dependencies: source-map: 0.5.6 stackframe: 1.3.4 - dev: false - /stacktrace-js@2.0.2: - resolution: {integrity: sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==} + stacktrace-js@2.0.2: dependencies: error-stack-parser: 2.1.4 stack-generator: 2.0.10 stacktrace-gps: 3.1.2 - dev: false - /streamsearch@1.1.0: - resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} - engines: {node: '>=10.0.0'} - dev: false + streamsearch@1.1.0: {} - /string-convert@0.2.1: - resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} - dev: false + string-convert@0.2.1: {} - /string-width@4.2.3: - resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} - engines: {node: '>=8'} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 is-fullwidth-code-point: 3.0.0 strip-ansi: 6.0.1 - dev: true - /string-width@5.1.2: - resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} - engines: {node: '>=12'} + string-width@5.1.2: dependencies: eastasianwidth: 0.2.0 emoji-regex: 9.2.2 strip-ansi: 7.1.0 - dev: true - /string.prototype.matchall@4.0.10: - resolution: {integrity: sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ==} + string.prototype.matchall@4.0.10: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 @@ -4318,86 +5124,47 @@ packages: regexp.prototype.flags: 1.5.1 set-function-name: 2.0.1 side-channel: 1.0.4 - dev: true - /string.prototype.trim@1.2.8: - resolution: {integrity: sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ==} - engines: {node: '>= 0.4'} + string.prototype.trim@1.2.8: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /string.prototype.trimend@1.0.7: - resolution: {integrity: sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA==} + string.prototype.trimend@1.0.7: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /string.prototype.trimstart@1.0.7: - resolution: {integrity: sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg==} + string.prototype.trimstart@1.0.7: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - dev: true - /strip-ansi@6.0.1: - resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} - engines: {node: '>=8'} + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 - dev: true - /strip-ansi@7.1.0: - resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} - engines: {node: '>=12'} + strip-ansi@7.1.0: dependencies: ansi-regex: 6.0.1 - dev: true - /strip-bom@3.0.0: - resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} - engines: {node: '>=4'} - dev: true + strip-bom@3.0.0: {} - /strip-json-comments@3.1.1: - resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} - engines: {node: '>=8'} - dev: true + strip-json-comments@3.1.1: {} - /styled-jsx@5.1.1(react@18.2.0): - resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} - engines: {node: '>= 12.0.0'} - peerDependencies: - '@babel/core': '*' - babel-plugin-macros: '*' - react: '>= 16.8.0 || 17.x.x || ^18.0.0-0' - peerDependenciesMeta: - '@babel/core': - optional: true - babel-plugin-macros: - optional: true + styled-jsx@5.1.1(react@18.2.0): dependencies: client-only: 0.0.1 react: 18.2.0 - dev: false - /stylis@4.2.0: - resolution: {integrity: sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==} - dev: false + stylis@4.2.0: {} - /stylis@4.3.1: - resolution: {integrity: sha512-EQepAV+wMsIaGVGX1RECzgrcqRRU/0sYOHkeLsZ3fzHaHXZy4DaOOX0vOlGQdlsjkh3mFHAIlVimpwAs4dslyQ==} - dev: false + stylis@4.3.1: {} - /sucrase@3.35.0: - resolution: {integrity: sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==} - engines: {node: '>=16 || 14 >=14.17'} - hasBin: true + sucrase@3.35.0: dependencies: '@jridgewell/gen-mapping': 0.3.3 commander: 4.1.1 @@ -4406,30 +5173,18 @@ packages: mz: 2.7.0 pirates: 4.0.6 ts-interface-checker: 0.1.13 - dev: true - /supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} + supports-color@5.5.0: dependencies: has-flag: 3.0.0 - dev: false - /supports-color@7.2.0: - resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} - engines: {node: '>=8'} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 - dev: true - /supports-preserve-symlinks-flag@1.0.0: - resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} - engines: {node: '>= 0.4'} + supports-preserve-symlinks-flag@1.0.0: {} - /tailwindcss@3.4.1: - resolution: {integrity: sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==} - engines: {node: '>=14.0.0'} - hasBin: true + tailwindcss@3.4.1: dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -4455,214 +5210,122 @@ packages: sucrase: 3.35.0 transitivePeerDependencies: - ts-node - dev: true - /tapable@2.2.1: - resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} - engines: {node: '>=6'} - dev: true + tapable@2.2.1: {} - /text-table@0.2.0: - resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} - dev: true + text-table@0.2.0: {} - /thenify-all@1.6.0: - resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} - engines: {node: '>=0.8'} + thenify-all@1.6.0: dependencies: thenify: 3.3.1 - dev: true - /thenify@3.3.1: - resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + thenify@3.3.1: dependencies: any-promise: 1.3.0 - dev: true - /throttle-debounce@3.0.1: - resolution: {integrity: sha512-dTEWWNu6JmeVXY0ZYoPuH5cRIwc0MeGbJwah9KUNYSJwommQpCzTySTpEe8Gs1J23aeWEuAobe4Ag7EHVt/LOg==} - engines: {node: '>=10'} - dev: false + throttle-debounce@3.0.1: {} - /throttle-debounce@5.0.0: - resolution: {integrity: sha512-2iQTSgkkc1Zyk0MeVrt/3BvuOXYPl/R8Z0U2xxo9rjwNciaHDG3R+Lm6dh4EeUci49DanvBnuqI6jshoQQRGEg==} - engines: {node: '>=12.22'} - dev: false + throttle-debounce@5.0.0: {} - /tiny-case@1.0.3: - resolution: {integrity: sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==} - dev: false + tiny-case@1.0.3: {} - /to-fast-properties@2.0.0: - resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} - engines: {node: '>=4'} - dev: false + to-fast-properties@2.0.0: {} - /to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 - dev: true - /toggle-selection@1.0.6: - resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} - dev: false + toggle-selection@1.0.6: {} - /toposort@2.0.2: - resolution: {integrity: sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==} - dev: false + toposort@2.0.2: {} - /totalist@1.1.0: - resolution: {integrity: sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==} - engines: {node: '>=6'} - dev: true + totalist@1.1.0: {} - /ts-api-utils@1.0.3(typescript@5.3.3): - resolution: {integrity: sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==} - engines: {node: '>=16.13.0'} - peerDependencies: - typescript: '>=4.2.0' + ts-api-utils@1.0.3(typescript@5.3.3): dependencies: typescript: 5.3.3 - dev: true - /ts-easing@0.2.0: - resolution: {integrity: sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ==} - dev: false + ts-easing@0.2.0: {} - /ts-interface-checker@0.1.13: - resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} - dev: true + ts-interface-checker@0.1.13: {} - /tsconfig-paths@3.15.0: - resolution: {integrity: sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==} + tsconfig-paths@3.15.0: dependencies: '@types/json5': 0.0.29 json5: 1.0.2 minimist: 1.2.8 strip-bom: 3.0.0 - dev: true - /tslib@2.6.2: - resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} - dev: false + tslib@2.6.2: {} - /type-check@0.4.0: - resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} - engines: {node: '>= 0.8.0'} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 - dev: true - /type-fest@0.20.2: - resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} - engines: {node: '>=10'} - dev: true + type-fest@0.20.2: {} - /type-fest@2.19.0: - resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} - engines: {node: '>=12.20'} - dev: false + type-fest@2.19.0: {} - /typed-array-buffer@1.0.0: - resolution: {integrity: sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==} - engines: {node: '>= 0.4'} + typed-array-buffer@1.0.0: dependencies: call-bind: 1.0.5 get-intrinsic: 1.2.2 is-typed-array: 1.1.12 - dev: true - /typed-array-byte-length@1.0.0: - resolution: {integrity: sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==} - engines: {node: '>= 0.4'} + typed-array-byte-length@1.0.0: dependencies: call-bind: 1.0.5 for-each: 0.3.3 has-proto: 1.0.1 is-typed-array: 1.1.12 - dev: true - /typed-array-byte-offset@1.0.0: - resolution: {integrity: sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==} - engines: {node: '>= 0.4'} + typed-array-byte-offset@1.0.0: dependencies: available-typed-arrays: 1.0.5 call-bind: 1.0.5 for-each: 0.3.3 has-proto: 1.0.1 is-typed-array: 1.1.12 - dev: true - /typed-array-length@1.0.4: - resolution: {integrity: sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==} + typed-array-length@1.0.4: dependencies: call-bind: 1.0.5 for-each: 0.3.3 is-typed-array: 1.1.12 - dev: true - /typescript@5.3.3: - resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} - engines: {node: '>=14.17'} - hasBin: true - dev: true + typescript@5.3.3: {} - /unbox-primitive@1.0.2: - resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + unbox-primitive@1.0.2: dependencies: call-bind: 1.0.5 has-bigints: 1.0.2 has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 - dev: true - /undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - dev: true + undici-types@5.26.5: {} - /update-browserslist-db@1.0.13(browserslist@4.22.2): - resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} - hasBin: true - peerDependencies: - browserslist: '>= 4.21.0' + update-browserslist-db@1.0.13(browserslist@4.22.2): dependencies: browserslist: 4.22.2 escalade: 3.1.1 picocolors: 1.0.0 - dev: true - /uri-js@4.4.1: - resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + uri-js@4.4.1: dependencies: punycode: 2.3.1 - dev: true - /use-sync-external-store@1.2.0(react@18.2.0): - resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} - peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + use-sync-external-store@1.2.0(react@18.2.0): dependencies: react: 18.2.0 - dev: false - /util-deprecate@1.0.2: - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - dev: true + util-deprecate@1.0.2: {} - /watchpack@2.4.0: - resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} - engines: {node: '>=10.13.0'} + watchpack@2.4.0: dependencies: glob-to-regexp: 0.4.1 graceful-fs: 4.2.11 - dev: false - /webpack-bundle-analyzer@4.7.0: - resolution: {integrity: sha512-j9b8ynpJS4K+zfO5GGwsAcQX4ZHpWV+yRiHDiL+bE0XHJ8NiPYLTNVQdlFYWxtpg9lfAQNlwJg16J9AJtFSXRg==} - engines: {node: '>= 10.13.0'} - hasBin: true + webpack-bundle-analyzer@4.7.0: dependencies: acorn: 8.11.3 acorn-walk: 8.3.1 @@ -4676,21 +5339,16 @@ packages: transitivePeerDependencies: - bufferutil - utf-8-validate - dev: true - /which-boxed-primitive@1.0.2: - resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} + which-boxed-primitive@1.0.2: dependencies: is-bigint: 1.0.4 is-boolean-object: 1.1.2 is-number-object: 1.0.7 is-string: 1.0.7 is-symbol: 1.0.4 - dev: true - /which-builtin-type@1.1.3: - resolution: {integrity: sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw==} - engines: {node: '>= 0.4'} + which-builtin-type@1.1.3: dependencies: function.prototype.name: 1.1.6 has-tostringtag: 1.0.0 @@ -4704,95 +5362,53 @@ packages: which-boxed-primitive: 1.0.2 which-collection: 1.0.1 which-typed-array: 1.1.13 - dev: true - /which-collection@1.0.1: - resolution: {integrity: sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==} + which-collection@1.0.1: dependencies: is-map: 2.0.2 is-set: 2.0.2 is-weakmap: 2.0.1 is-weakset: 2.0.2 - dev: true - /which-typed-array@1.1.13: - resolution: {integrity: sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==} - engines: {node: '>= 0.4'} + which-typed-array@1.1.13: dependencies: available-typed-arrays: 1.0.5 call-bind: 1.0.5 for-each: 0.3.3 gopd: 1.0.1 has-tostringtag: 1.0.0 - dev: true - /which@2.0.2: - resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} - engines: {node: '>= 8'} - hasBin: true + which@2.0.2: dependencies: isexe: 2.0.0 - dev: true - /wrap-ansi@7.0.0: - resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} - engines: {node: '>=10'} + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 string-width: 4.2.3 strip-ansi: 6.0.1 - dev: true - /wrap-ansi@8.1.0: - resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} - engines: {node: '>=12'} + wrap-ansi@8.1.0: dependencies: ansi-styles: 6.2.1 string-width: 5.1.2 strip-ansi: 7.1.0 - dev: true - /wrappy@1.0.2: - resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: true + wrappy@1.0.2: {} - /ws@7.5.9: - resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==} - engines: {node: '>=8.3.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - dev: true + ws@7.5.9: {} - /yallist@4.0.0: - resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: true + yallist@4.0.0: {} - /yaml@1.10.2: - resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} - engines: {node: '>= 6'} - dev: false + yaml@1.10.2: {} - /yaml@2.3.4: - resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} - engines: {node: '>= 14'} - dev: true + yaml@2.3.4: {} - /yocto-queue@0.1.0: - resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} - engines: {node: '>=10'} - dev: true + yocto-queue@0.1.0: {} - /yup@1.3.3: - resolution: {integrity: sha512-v8QwZSsHH2K3/G9WSkp6mZKO+hugKT1EmnMqLNUcfu51HU9MDyhlETT/JgtzprnrnQHPWsjc6MUDMBp/l9fNnw==} + yup@1.3.3: dependencies: property-expr: 2.0.6 tiny-case: 1.0.3 toposort: 2.0.2 type-fest: 2.19.0 - dev: false From b04efd01fcab5038ab3466a2a49dbc655a765d8f Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 16 Apr 2024 23:24:57 +0800 Subject: [PATCH 043/106] [#2966] feat(core): supports generate topic event (#2968) ### What changes were proposed in this pull request? supports generate topic event ### Why are the changes needed? Fix: #2966 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --------- Co-authored-by: Jerry Shao --- .../datastrato/gravitino/GravitinoEnv.java | 15 +- .../gravitino/catalog/TopicDispatcher.java | 16 +++ .../catalog/TopicEventDispatcher.java | 131 ++++++++++++++++++ .../catalog/TopicOperationDispatcher.java | 3 +- .../listener/api/event/AlterTopicEvent.java | 58 ++++++++ .../api/event/AlterTopicFailureEvent.java | 43 ++++++ .../listener/api/event/CreateTopicEvent.java | 40 ++++++ .../api/event/CreateTopicFailureEvent.java | 47 +++++++ .../listener/api/event/DropTopicEvent.java | 41 ++++++ .../api/event/DropTopicFailureEvent.java | 29 ++++ .../listener/api/event/ListTopicEvent.java | 36 +++++ .../api/event/ListTopicFailureEvent.java | 40 ++++++ .../listener/api/event/LoadTopicEvent.java | 37 +++++ .../api/event/LoadTopicFailureEvent.java | 25 ++++ .../listener/api/event/TopicEvent.java | 32 +++++ .../listener/api/event/TopicFailureEvent.java | 35 +++++ .../listener/api/info/TopicInfo.java | 84 +++++++++++ .../gravitino/server/GravitinoServer.java | 5 +- .../server/web/rest/TopicOperations.java | 6 +- .../server/web/rest/TestTopicOperations.java | 3 +- 20 files changed, 711 insertions(+), 15 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java create mode 100644 core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 02498d2d077..57c3ea18bc4 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -18,6 +18,8 @@ import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.catalog.TableEventDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; +import com.datastrato.gravitino.catalog.TopicDispatcher; +import com.datastrato.gravitino.catalog.TopicEventDispatcher; import com.datastrato.gravitino.catalog.TopicOperationDispatcher; import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.EventListenerManager; @@ -55,7 +57,7 @@ public class GravitinoEnv { private FilesetDispatcher filesetDispatcher; - private TopicOperationDispatcher topicOperationDispatcher; + private TopicDispatcher topicDispatcher; private MetalakeDispatcher metalakeDispatcher; @@ -149,8 +151,9 @@ public void initialize(Config config) { FilesetOperationDispatcher filesetOperationDispatcher = new FilesetOperationDispatcher(catalogManager, entityStore, idGenerator); this.filesetDispatcher = new FilesetEventDispatcher(eventBus, filesetOperationDispatcher); - this.topicOperationDispatcher = + TopicOperationDispatcher topicOperationDispatcher = new TopicOperationDispatcher(catalogManager, entityStore, idGenerator); + this.topicDispatcher = new TopicEventDispatcher(eventBus, topicOperationDispatcher); // Create and initialize access control related modules boolean enableAuthorization = config.get(Configs.ENABLE_AUTHORIZATION); @@ -225,12 +228,12 @@ public FilesetDispatcher filesetDispatcher() { } /** - * Get the TopicOperationDispatcher associated with the Gravitino environment. + * Get the TopicDispatcher associated with the Gravitino environment. * - * @return The TopicOperationDispatcher instance. + * @return The TopicDispatcher instance. */ - public TopicOperationDispatcher topicOperationDispatcher() { - return topicOperationDispatcher; + public TopicDispatcher topicDispatcher() { + return topicDispatcher; } /** diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java new file mode 100644 index 00000000000..131a600c621 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.messaging.TopicCatalog; + +/** + * {@code TopicDispatcher} interface acts as a specialization of the {@link TopicCatalog} interface. + * This interface is designed to potentially add custom behaviors or operations related to + * dispatching or handling topic-related events or actions that are not covered by the standard + * {@code TopicCatalog} operations. + */ +public interface TopicDispatcher extends TopicCatalog {} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java new file mode 100644 index 00000000000..9375cae604d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchTopicException; +import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.event.AlterTopicEvent; +import com.datastrato.gravitino.listener.api.event.AlterTopicFailureEvent; +import com.datastrato.gravitino.listener.api.event.CreateTopicEvent; +import com.datastrato.gravitino.listener.api.event.CreateTopicFailureEvent; +import com.datastrato.gravitino.listener.api.event.DropTopicEvent; +import com.datastrato.gravitino.listener.api.event.DropTopicFailureEvent; +import com.datastrato.gravitino.listener.api.event.ListTopicEvent; +import com.datastrato.gravitino.listener.api.event.ListTopicFailureEvent; +import com.datastrato.gravitino.listener.api.event.LoadTopicEvent; +import com.datastrato.gravitino.listener.api.event.LoadTopicFailureEvent; +import com.datastrato.gravitino.listener.api.info.TopicInfo; +import com.datastrato.gravitino.messaging.DataLayout; +import com.datastrato.gravitino.messaging.Topic; +import com.datastrato.gravitino.messaging.TopicChange; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.util.Map; + +/** + * {@code TopicEventDispatcher} is a decorator for {@link TopicDispatcher} that not only delegates + * topic operations to the underlying catalog dispatcher but also dispatches corresponding events to + * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or + * monitoring of topic operations. + */ +public class TopicEventDispatcher implements TopicDispatcher { + private final EventBus eventBus; + private final TopicDispatcher dispatcher; + + /** + * Constructs a TopicEventDispatcher with a specified EventBus and TopicCatalog. + * + * @param eventBus The EventBus to which events will be dispatched. + * @param dispatcher The underlying {@link TopicDispatcher} that will perform the actual topic + * operations. + */ + public TopicEventDispatcher(EventBus eventBus, TopicDispatcher dispatcher) { + this.eventBus = eventBus; + this.dispatcher = dispatcher; + } + + @Override + public Topic alterTopic(NameIdentifier ident, TopicChange... changes) + throws NoSuchTopicException, IllegalArgumentException { + try { + Topic topic = dispatcher.alterTopic(ident, changes); + eventBus.dispatchEvent( + new AlterTopicEvent( + PrincipalUtils.getCurrentUserName(), ident, changes, new TopicInfo(topic))); + return topic; + } catch (Exception e) { + eventBus.dispatchEvent( + new AlterTopicFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e, changes)); + throw e; + } + } + + @Override + public boolean dropTopic(NameIdentifier ident) { + try { + boolean isExists = dispatcher.dropTopic(ident); + eventBus.dispatchEvent( + new DropTopicEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); + return isExists; + } catch (Exception e) { + eventBus.dispatchEvent( + new DropTopicFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public NameIdentifier[] listTopics(Namespace namespace) throws NoSuchTopicException { + try { + NameIdentifier[] nameIdentifiers = dispatcher.listTopics(namespace); + eventBus.dispatchEvent(new ListTopicEvent(PrincipalUtils.getCurrentUserName(), namespace)); + return nameIdentifiers; + } catch (Exception e) { + eventBus.dispatchEvent( + new ListTopicFailureEvent(PrincipalUtils.getCurrentUserName(), namespace, e)); + throw e; + } + } + + @Override + public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { + try { + Topic topic = dispatcher.loadTopic(ident); + eventBus.dispatchEvent( + new LoadTopicEvent(PrincipalUtils.getCurrentUserName(), ident, new TopicInfo(topic))); + return topic; + } catch (Exception e) { + eventBus.dispatchEvent( + new LoadTopicFailureEvent(PrincipalUtils.getCurrentUserName(), ident, e)); + throw e; + } + } + + @Override + public boolean topicExists(NameIdentifier ident) { + return dispatcher.topicExists(ident); + } + + @Override + public Topic createTopic( + NameIdentifier ident, String comment, DataLayout dataLayout, Map properties) + throws NoSuchTopicException, TopicAlreadyExistsException { + try { + Topic topic = dispatcher.createTopic(ident, comment, dataLayout, properties); + eventBus.dispatchEvent( + new CreateTopicEvent(PrincipalUtils.getCurrentUserName(), ident, new TopicInfo(topic))); + return topic; + } catch (Exception e) { + TopicInfo createTopicRequest = new TopicInfo(ident.name(), comment, properties, null); + eventBus.dispatchEvent( + new CreateTopicFailureEvent( + PrincipalUtils.getCurrentUserName(), ident, e, createTopicRequest)); + throw e; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java index 26c59627196..8935f4f6518 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java @@ -19,7 +19,6 @@ import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; import com.datastrato.gravitino.messaging.DataLayout; import com.datastrato.gravitino.messaging.Topic; -import com.datastrato.gravitino.messaging.TopicCatalog; import com.datastrato.gravitino.messaging.TopicChange; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.TopicEntity; @@ -31,7 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TopicOperationDispatcher extends OperationDispatcher implements TopicCatalog { +public class TopicOperationDispatcher extends OperationDispatcher implements TopicDispatcher { private static final Logger LOG = LoggerFactory.getLogger(TopicOperationDispatcher.class); /** diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java new file mode 100644 index 00000000000..c6a0b6961b1 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TopicInfo; +import com.datastrato.gravitino.messaging.TopicChange; + +/** Represents an event fired when a topic is successfully altered. */ +@DeveloperApi +public final class AlterTopicEvent extends TopicEvent { + private final TopicInfo updatedTopicInfo; + private final TopicChange[] topicChanges; + + /** + * Constructs an instance of {@code AlterTopicEvent}, encapsulating the key details about the + * successful alteration of a topic. + * + * @param user The username of the individual responsible for initiating the topic alteration. + * @param identifier The unique identifier of the altered topic, serving as a clear reference + * point for the topic in question. + * @param topicChanges An array of {@link TopicChange} objects representing the specific changes + * applied to the topic during the alteration process. + * @param updatedTopicInfo The post-alteration state of the topic. + */ + public AlterTopicEvent( + String user, + NameIdentifier identifier, + TopicChange[] topicChanges, + TopicInfo updatedTopicInfo) { + super(user, identifier); + this.topicChanges = topicChanges.clone(); + this.updatedTopicInfo = updatedTopicInfo; + } + + /** + * Retrieves the updated state of the topic after the successful alteration. + * + * @return A {@link TopicInfo} instance encapsulating the details of the altered topic. + */ + public TopicInfo updatedTopicInfo() { + return updatedTopicInfo; + } + + /** + * Retrieves the specific changes that were made to the topic during the alteration process. + * + * @return An array of {@link TopicChange} objects detailing each modification applied to the + * topic. + */ + public TopicChange[] topicChanges() { + return topicChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java new file mode 100644 index 00000000000..dc207450260 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.messaging.TopicChange; + +/** + * Represents an event that is triggered when an attempt to alter a topic fails due to an exception. + */ +@DeveloperApi +public final class AlterTopicFailureEvent extends TopicFailureEvent { + private final TopicChange[] topicChanges; + + /** + * Constructs an {@code AlterTopicFailureEvent} instance, capturing detailed information about the + * failed topic alteration attempt. + * + * @param user The user who initiated the topic alteration operation. + * @param identifier The identifier of the topic that was attempted to be altered. + * @param exception The exception that was thrown during the topic alteration operation. + * @param topicChanges The changes that were attempted on the topic. + */ + public AlterTopicFailureEvent( + String user, NameIdentifier identifier, Exception exception, TopicChange[] topicChanges) { + super(user, identifier, exception); + this.topicChanges = topicChanges.clone(); + } + + /** + * Retrieves the changes that were attempted on the topic. + * + * @return An array of {@link TopicChange} objects representing the attempted modifications to the + * topic. + */ + public TopicChange[] topicChanges() { + return topicChanges; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java new file mode 100644 index 00000000000..f3418faf0f8 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TopicInfo; + +/** Represents an event triggered upon the successful creation of a topic. */ +@DeveloperApi +public final class CreateTopicEvent extends TopicEvent { + private final TopicInfo createdTopicInfo; + + /** + * Constructs an instance of {@code CreateTopicEvent}, capturing essential details about the + * successful creation of a topic. + * + * @param user The username of the individual who initiated the topic creation. + * @param identifier The unique identifier of the topic that was created. + * @param createdTopicInfo The final state of the topic post-creation. + */ + public CreateTopicEvent(String user, NameIdentifier identifier, TopicInfo createdTopicInfo) { + super(user, identifier); + this.createdTopicInfo = createdTopicInfo; + } + + /** + * Retrieves the final state of the topic as it was returned to the user after successful + * creation. + * + * @return A {@link TopicInfo} instance encapsulating the comprehensive details of the newly + * created topic. + */ + public TopicInfo createdTopicInfo() { + return createdTopicInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java new file mode 100644 index 00000000000..3ec9425a941 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TopicInfo; + +/** + * Represents an event that is generated when an attempt to create a topic fails due to an + * exception. + */ +@DeveloperApi +public final class CreateTopicFailureEvent extends TopicFailureEvent { + private final TopicInfo createTopicRequest; + + /** + * Constructs a {@code CreateTopicFailureEvent} instance, capturing detailed information about the + * failed topic creation attempt. + * + * @param user The user who initiated the topic creation operation. + * @param identifier The identifier of the topic that was attempted to be created. + * @param exception The exception that was thrown during the topic creation operation, providing + * insight into what went wrong. + * @param createTopicRequest The original request information used to attempt to create the topic. + * This includes details such as the intended topic schema, properties, and other + * configuration options that were specified. + */ + public CreateTopicFailureEvent( + String user, NameIdentifier identifier, Exception exception, TopicInfo createTopicRequest) { + super(user, identifier, exception); + this.createTopicRequest = createTopicRequest; + } + + /** + * Retrieves the original request information for the attempted topic creation. + * + * @return The {@link TopicInfo} instance representing the request information for the failed + * topic creation attempt. + */ + public TopicInfo createTopicRequest() { + return createTopicRequest; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java new file mode 100644 index 00000000000..70051e6ae28 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated after a topic is successfully dropped from the database. + */ +@DeveloperApi +public final class DropTopicEvent extends TopicEvent { + private final boolean isExists; + + /** + * Constructs a new {@code DropTopicEvent} instance, encapsulating information about the outcome + * of a topic drop operation. + * + * @param user The user who initiated the drop topic operation. + * @param identifier The identifier of the topic that was attempted to be dropped. + * @param isExists A boolean flag indicating whether the topic existed at the time of the drop + * operation. + */ + public DropTopicEvent(String user, NameIdentifier identifier, boolean isExists) { + super(user, identifier); + this.isExists = isExists; + } + + /** + * Retrieves the existence status of the topic at the time of the drop operation. + * + * @return A boolean value indicating whether the topic existed. {@code true} if the topic + * existed, otherwise {@code false}. + */ + public boolean isExists() { + return isExists; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java new file mode 100644 index 00000000000..5668783a660 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is generated when an attempt to drop a topic from the database fails due + * to an exception. + */ +@DeveloperApi +public final class DropTopicFailureEvent extends TopicFailureEvent { + /** + * Constructs a new {@code DropTopicFailureEvent} instance, capturing detailed information about + * the failed attempt to drop a topic. + * + * @param user The user who initiated the drop topic operation. + * @param identifier The identifier of the topic that the operation attempted to drop. + * @param exception The exception that was thrown during the drop topic operation, offering + * insights into what went wrong and why the operation failed. + */ + public DropTopicFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java new file mode 100644 index 00000000000..4eacc1fbf1b --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that is triggered upon the successful list of topics within a namespace. */ +@DeveloperApi +public final class ListTopicEvent extends TopicEvent { + private final Namespace namespace; + + /** + * Constructs an instance of {@code ListTopicEvent}. + * + * @param user The username of the individual who initiated the topic listing. + * @param namespace The namespace from which topics were listed. + */ + public ListTopicEvent(String user, Namespace namespace) { + super(user, NameIdentifier.parse(namespace.toString())); + this.namespace = namespace; + } + + /** + * Provides the namespace associated with this event. + * + * @return A {@link Namespace} instance from which topics were listed. + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java new file mode 100644 index 00000000000..e98212fb747 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an event that is triggered when an attempt to list topics within a namespace fails due + * to an exception. + */ +@DeveloperApi +public final class ListTopicFailureEvent extends TopicFailureEvent { + private final Namespace namespace; + + /** + * Constructs a {@code ListTopicFailureEvent} instance. + * + * @param user The username of the individual who initiated the operation to list topics. + * @param namespace The namespace for which the topic listing was attempted. + * @param exception The exception encountered during the attempt to list topics. + */ + public ListTopicFailureEvent(String user, Namespace namespace, Exception exception) { + super(user, NameIdentifier.of(namespace.toString()), exception); + this.namespace = namespace; + } + + /** + * Retrieves the namespace associated with this failure event. + * + * @return A {@link Namespace} instance for which the topic listing was attempted + */ + public Namespace namespace() { + return namespace; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java new file mode 100644 index 00000000000..ce516ffa9c8 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.listener.api.info.TopicInfo; + +/** Represents an event triggered upon the successful loading of a topic. */ +@DeveloperApi +public final class LoadTopicEvent extends TopicEvent { + private final TopicInfo loadedTopicInfo; + + /** + * Constructs an instance of {@code LoadTopicEvent}. + * + * @param user The username of the individual who initiated the topic loading. + * @param identifier The unique identifier of the topic that was loaded. + * @param topicInfo The state of the topic post-loading. + */ + public LoadTopicEvent(String user, NameIdentifier identifier, TopicInfo topicInfo) { + super(user, identifier); + this.loadedTopicInfo = topicInfo; + } + + /** + * Retrieves the state of the topic as it was made available to the user after successful loading. + * + * @return A {@link TopicInfo} instance encapsulating the details of the topic as loaded. + */ + public TopicInfo loadedTopicInfo() { + return loadedTopicInfo; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java new file mode 100644 index 00000000000..ee3f5b6401e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** Represents an event that occurs when an attempt to load a topic fails due to an exception. */ +@DeveloperApi +public final class LoadTopicFailureEvent extends TopicFailureEvent { + /** + * Constructs a {@code LoadTopicFailureEvent} instance. + * + * @param user The user who initiated the topic loading operation. + * @param identifier The identifier of the topic that the loading attempt was made for. + * @param exception The exception that was thrown during the topic loading operation, offering + * insight into the issues encountered. + */ + public LoadTopicFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java new file mode 100644 index 00000000000..f6fadae14aa --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * Represents an abstract base class for events related to topic operations. This class extends + * {@link Event} to provide a more specific context involving operations on topics, such as + * creation, deletion, or modification. It captures essential information including the user + * performing the operation and the identifier of the topic being operated on. + * + *

Concrete implementations of this class should provide additional details pertinent to the + * specific type of topic operation being represented. + */ +@DeveloperApi +public abstract class TopicEvent extends Event { + /** + * Constructs a new {@code TopicEvent} with the specified user and topic identifier. + * + * @param user The user responsible for triggering the topic operation. + * @param identifier The identifier of the topic involved in the operation. This encapsulates + * details such as the metalake, catalog, schema, and topic name. + */ + protected TopicEvent(String user, NameIdentifier identifier) { + super(user, identifier); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java new file mode 100644 index 00000000000..a05b45862d8 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.annotation.DeveloperApi; + +/** + * An abstract class representing events that are triggered when a topic operation fails due to an + * exception. This class extends {@link FailureEvent} to provide a more specific context related to + * topic operations, encapsulating details about the user who initiated the operation, the + * identifier of the topic involved, and the exception that led to the failure. + * + *

Implementations of this class can be used to convey detailed information about failures in + * operations such as creating, updating, deleting, or querying topics, making it easier to diagnose + * and respond to issues. + */ +@DeveloperApi +public abstract class TopicFailureEvent extends FailureEvent { + /** + * Constructs a new {@code TopicFailureEvent} instance, capturing information about the failed + * topic operation. + * + * @param user The user associated with the failed topic operation. + * @param identifier The identifier of the topic that was involved in the failed operation. + * @param exception The exception that was thrown during the topic operation, indicating the cause + * of the failure. + */ + protected TopicFailureEvent(String user, NameIdentifier identifier, Exception exception) { + super(user, identifier, exception); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java new file mode 100644 index 00000000000..2c5f8bd05dc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.info; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.annotation.DeveloperApi; +import com.datastrato.gravitino.messaging.Topic; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** Provides read-only access to topic information for event listeners. */ +@DeveloperApi +public final class TopicInfo { + private final String name; + @Nullable private final String comment; + private final Map properties; + @Nullable private final Audit audit; + + /** + * Constructs topic information based on a given topic. + * + * @param topic The topic to extract information from. + */ + public TopicInfo(Topic topic) { + this(topic.name(), topic.comment(), topic.properties(), topic.auditInfo()); + } + + /** + * Constructs topic information with detailed parameters. + * + * @param name The name of the topic. + * @param comment An optional description of the topic. + * @param properties A map of topic properties. + * @param audit Optional audit information. + */ + public TopicInfo(String name, String comment, Map properties, Audit audit) { + this.name = name; + this.comment = comment; + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + this.audit = audit; + } + + /** + * Gets the topic name. + * + * @return The topic name. + */ + public String name() { + return name; + } + + /** + * Gets the optional topic comment. + * + * @return The topic comment, or null if not provided. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Gets the topic properties. + * + * @return An immutable map of topic properties. + */ + public Map properties() { + return properties; + } + + /** + * Gets the optional audit information. + * + * @return The audit information, or null if not provided. + */ + @Nullable + public Audit audit() { + return audit; + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 88ca4a815ab..d54f3c8fdf3 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -10,6 +10,7 @@ import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; +import com.datastrato.gravitino.catalog.TopicDispatcher; import com.datastrato.gravitino.metalake.MetalakeDispatcher; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.MetricsSource; @@ -82,9 +83,7 @@ protected void configure() { bind(gravitinoEnv.schemaDispatcher()).to(SchemaDispatcher.class).ranked(1); bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); - bind(gravitinoEnv.topicOperationDispatcher()) - .to(com.datastrato.gravitino.catalog.TopicOperationDispatcher.class) - .ranked(1); + bind(gravitinoEnv.topicDispatcher()).to(TopicDispatcher.class).ranked(1); } }); register(ObjectMapperProvider.class).register(JacksonFeature.class); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TopicOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TopicOperations.java index c0b52751bb5..bff22a91514 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TopicOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TopicOperations.java @@ -8,7 +8,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.catalog.TopicOperationDispatcher; +import com.datastrato.gravitino.catalog.TopicDispatcher; import com.datastrato.gravitino.dto.requests.TopicCreateRequest; import com.datastrato.gravitino.dto.requests.TopicUpdateRequest; import com.datastrato.gravitino.dto.requests.TopicUpdatesRequest; @@ -40,12 +40,12 @@ public class TopicOperations { private static final Logger LOG = LoggerFactory.getLogger(TopicOperations.class); - private final TopicOperationDispatcher dispatcher; + private final TopicDispatcher dispatcher; @Context private HttpServletRequest httpRequest; @Inject - public TopicOperations(TopicOperationDispatcher dispatcher) { + public TopicOperations(TopicDispatcher dispatcher) { this.dispatcher = dispatcher; } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTopicOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTopicOperations.java index 35acfedf277..a65b75fdf4a 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTopicOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTopicOperations.java @@ -16,6 +16,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.catalog.TopicDispatcher; import com.datastrato.gravitino.catalog.TopicOperationDispatcher; import com.datastrato.gravitino.dto.messaging.TopicDTO; import com.datastrato.gravitino.dto.requests.TopicCreateRequest; @@ -91,7 +92,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(dispatcher).to(TopicOperationDispatcher.class).ranked(2); + bind(dispatcher).to(TopicDispatcher.class).ranked(2); bindFactory(TestTopicOperations.MockServletRequestFactory.class) .to(HttpServletRequest.class); } From 0e7fc9a3c8b30db80b115cb7969da5f6dc0e168a Mon Sep 17 00:00:00 2001 From: cai can <94670132+caican00@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:09:12 +0800 Subject: [PATCH 044/106] [#2979] feat(spark-connector): Implement FunctionCatalog in GravitinoIcebergCatalog (#2983) ### What changes were proposed in this pull request? Implement `FunctionCatalog` in `GravitinoIcebergCatalog` ### Why are the changes needed? Implement `FunctionCatalog` in `GravitinoIcebergCatalog` to support `Iceberg` partition functions. Fix: https://github.com/datastrato/gravitino/issues/2979 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? New ITs. --- .../spark/iceberg/SparkIcebergCatalogIT.java | 71 +++++++++++++++++++ .../test/util/spark/SparkUtilIT.java | 6 +- .../iceberg/GravitinoIcebergCatalog.java | 16 ++++- 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java index de1adb46c0b..ed5df8f08ac 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java @@ -6,7 +6,15 @@ import com.datastrato.gravitino.integration.test.spark.SparkCommonIT; import com.datastrato.gravitino.integration.test.util.spark.SparkTableInfo; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; import java.util.List; +import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException; +import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; +import org.apache.spark.sql.connector.catalog.CatalogPlugin; +import org.apache.spark.sql.connector.catalog.FunctionCatalog; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.functions.UnboundFunction; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -55,4 +63,67 @@ void testIcebergFileLevelDeleteOperation() { List queryResult2 = getTableData(tableName); Assertions.assertEquals(0, queryResult2.size()); } + + @Test + void testIcebergListAndLoadFunctions() throws NoSuchNamespaceException, NoSuchFunctionException { + String[] empty_namespace = new String[] {}; + String[] system_namespace = new String[] {"system"}; + String[] default_namespace = new String[] {getDefaultDatabase()}; + String[] non_exists_namespace = new String[] {"non_existent"}; + List functions = + Arrays.asList("iceberg_version", "years", "months", "days", "hours", "bucket", "truncate"); + + CatalogPlugin catalogPlugin = + getSparkSession().sessionState().catalogManager().catalog(getCatalogName()); + Assertions.assertInstanceOf(FunctionCatalog.class, catalogPlugin); + FunctionCatalog functionCatalog = (FunctionCatalog) catalogPlugin; + + for (String[] namespace : ImmutableList.of(empty_namespace, system_namespace)) { + Arrays.stream(functionCatalog.listFunctions(namespace)) + .map(Identifier::name) + .forEach(function -> Assertions.assertTrue(functions.contains(function))); + } + Arrays.stream(functionCatalog.listFunctions(default_namespace)) + .map(Identifier::name) + .forEach(function -> Assertions.assertFalse(functions.contains(function))); + Assertions.assertThrows( + NoSuchNamespaceException.class, () -> functionCatalog.listFunctions(non_exists_namespace)); + + for (String[] namespace : ImmutableList.of(empty_namespace, system_namespace)) { + for (String function : functions) { + Identifier identifier = Identifier.of(namespace, function); + UnboundFunction func = functionCatalog.loadFunction(identifier); + Assertions.assertEquals(function, func.name()); + } + } + functions.forEach( + function -> { + Identifier identifier = Identifier.of(new String[] {getDefaultDatabase()}, function); + Assertions.assertThrows( + NoSuchFunctionException.class, () -> functionCatalog.loadFunction(identifier)); + }); + } + + @Test + void testIcebergFunction() { + String[] catalogAndNamespaces = new String[] {getCatalogName() + ".system", getCatalogName()}; + Arrays.stream(catalogAndNamespaces) + .forEach( + catalogAndNamespace -> { + List bucket = + getQueryData(String.format("SELECT %s.bucket(2, 100)", catalogAndNamespace)); + Assertions.assertEquals(1, bucket.size()); + Assertions.assertEquals("0", bucket.get(0)); + }); + + Arrays.stream(catalogAndNamespaces) + .forEach( + catalogAndNamespace -> { + List bucket = + getQueryData( + String.format("SELECT %s.truncate(2, 'abcdef')", catalogAndNamespace)); + Assertions.assertEquals(1, bucket.size()); + Assertions.assertEquals("ab", bucket.get(0)); + }); + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java index 6616df7e2c0..94eb17d465e 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java @@ -86,7 +86,11 @@ protected List sql(String query) { // columns data are joined by ',' protected List getTableData(String tableName) { - return sql(getSelectAllSql(tableName)).stream() + return getQueryData(getSelectAllSql(tableName)); + } + + protected List getQueryData(String querySql) { + return sql(querySql).stream() .map( line -> Arrays.stream(line) diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java index e3b9783d41e..d1a8a7a9fcf 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java @@ -15,8 +15,12 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.spark.SparkCatalog; +import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException; +import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; +import org.apache.spark.sql.connector.catalog.FunctionCatalog; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.connector.catalog.functions.UnboundFunction; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** @@ -26,7 +30,7 @@ * StagingTableCatalog and FunctionCatalog, allowing for advanced operations like table staging and * function management tailored to the needs of Iceberg tables. */ -public class GravitinoIcebergCatalog extends BaseCatalog { +public class GravitinoIcebergCatalog extends BaseCatalog implements FunctionCatalog { @Override protected TableCatalog createAndInitSparkCatalog( @@ -74,6 +78,16 @@ protected PropertiesConverter getPropertiesConverter() { return new IcebergPropertiesConverter(); } + @Override + public Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceException { + return ((SparkCatalog) sparkCatalog).listFunctions(namespace); + } + + @Override + public UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionException { + return ((SparkCatalog) sparkCatalog).loadFunction(ident); + } + private void initHiveProperties( String catalogBackend, Map gravitinoProperties, From 30b7604997ad2bc7c21899898bb888698331b2e0 Mon Sep 17 00:00:00 2001 From: Kang Date: Wed, 17 Apr 2024 12:34:00 +0800 Subject: [PATCH 045/106] [MINOR]test: Increase the maximum wait time in Doris table operation test (#2977) ### What changes were proposed in this pull request? Increase the maximum wait time from 5 seconds to 30 seconds, which is set in schema alteration test for the Doris table operation Avoid the instability of the GitHub CI flow. ### Why are the changes needed? Github CI flow failed because table schema has not been done. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Github CI --- .../catalog/doris/operation/DorisTableOperations.java | 8 ++++++++ .../doris/integration/test/DorisTableOperationsIT.java | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index f302a77dfef..058e1c2f9fd 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -292,6 +292,14 @@ protected String generatePurgeTableSql(String tableName) { @Override protected String generateAlterTableSql( String databaseName, String tableName, TableChange... changes) { + /* + * NOTICE: + * As described in the Doris documentation, the creation of Schema Change is an asynchronous process. + * If you load the table immediately after altering it, you might get the old schema. + * You can see in: https://doris.apache.org/docs/1.2/advanced/alter-table/schema-change/#create-job + * TODO: return state of the operation to user + * */ + // Not all operations require the original table information, so lazy loading is used here JdbcTable lazyLoadTable = null; TableChange.UpdateComment updateComment = null; diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java index f1114dc9552..1ee61a91a1b 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/DorisTableOperationsIT.java @@ -40,7 +40,10 @@ public class DorisTableOperationsIT extends TestDorisAbstractIT { private static final String databaseName = GravitinoITUtils.genRandomName("doris_test_db"); - private static final long MAX_WAIT = 5; + // Because the creation of Schema Change is an asynchronous process, we need wait for a while + // For more information, you can refer to the comment in + // DorisTableOperations.generateAlterTableSql(). + private static final long MAX_WAIT = 30; private static final long WAIT_INTERVAL = 1; From d0ad18a8ee3214a5b2faa93023e0aac9786d32ab Mon Sep 17 00:00:00 2001 From: charliecheng630 <74488612+charliecheng630@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:59:19 +0800 Subject: [PATCH 046/106] [#2913] Improvement(catalog-hadoop) Return false if the schema is not dropped. (#2984) ### What changes were proposed in this pull request? When dropping the schema, if the schema is null or does not exist, then return false. ### Why are the changes needed? a clear drop behavior of return value and exception thrown. Fix: #2913 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? UT --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 4 ++-- .../gravitino/catalog/hadoop/TestHadoopCatalogOperations.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 2403cc5c17b..cd87e2d2cc4 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -461,13 +461,13 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty Path schemaPath = getSchemaPath(ident.name(), properties); // Nothing to delete if the schema path is not set. if (schemaPath == null) { - return true; + return false; } FileSystem fs = schemaPath.getFileSystem(hadoopConf); // Nothing to delete if the schema path does not exist. if (!fs.exists(schemaPath)) { - return true; + return false; } if (fs.listStatus(schemaPath).length > 0 && !cascade) { diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 1eca47aba44..46a370c4c90 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -327,6 +327,10 @@ public void testDropSchema() throws IOException { // Test drop non-empty schema with cascade = true ops.dropSchema(id, true); Assertions.assertFalse(fs.exists(schemaPath)); + + // Test drop empty schema + Assertions.assertFalse(ops.dropSchema(id, true)); + Assertions.assertFalse(ops.dropSchema(id, false)); } } From b8f6efa8d37595ed7916344f67d820d95075a779 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Wed, 17 Apr 2024 15:27:19 +0800 Subject: [PATCH 047/106] [#792] improvement(core): Optimize the deletion process of entities that have large sub-entities (#1132) ### What changes were proposed in this pull request? When removing an entity, we will delete the name-id mapping instead of the entity itself. ### Why are the changes needed? Deleting an entity with many sub-entities is time-consuming. By removing the name-id mapping, we can delete an entity more efficiently. Fix: #792 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing UT `testEntityDelete` can cover this change. --- .../gravitino/storage/kv/KvEntityStore.java | 16 ++-- .../gravitino/storage/TestEntityStorage.java | 88 +++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 6222c9d60c9..0fafc0c1745 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -282,20 +282,18 @@ public boolean delete(NameIdentifier ident, EntityType entityType, boolean casca ident, subEntities); } - for (byte[] prefix : subEntityPrefix) { - transactionalKvBackend.deleteRange( - new KvRange.KvRangeBuilder() - .start(prefix) - .startInclusive(true) - .end(Bytes.increment(Bytes.wrap(prefix)).get()) - .build()); - } + // Remove id-name mapping; + unbindNameAndId(ident, entityType); - deleteAuthorizationEntitiesIfNecessary(ident, entityType); return transactionalKvBackend.delete(dataKey); }); } + private void unbindNameAndId(NameIdentifier ident, EntityType entityType) throws IOException { + String identNameToIdKey = generateKeyForMapping(ident, entityType, nameMappingService); + nameMappingService.unbindNameAndId(identNameToIdKey); + } + @Override public R executeInTransaction(Executable executable) throws E, IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 4df9a63f7df..e79ccf9cacf 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -23,6 +23,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.Entity.EntityType; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.NameIdentifier; @@ -1910,4 +1911,91 @@ private void validateDeletedTable(EntityStore store) throws IOException { Entity.EntityType.TABLE, (e) -> e)); } + + @ParameterizedTest + @MethodSource("storageProvider") + void testOptimizedDeleteForKv(String type) throws IOException { + if ("relational".equalsIgnoreCase(type)) { + return; + } + + Config config = Mockito.mock(Config.class); + init(type, config); + + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { + store.initialize(config); + if (store instanceof RelationalEntityStore) { + prepareJdbcTable(); + } + + BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); + CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); + CatalogEntity catalogCopy = + createCatalog(2L, Namespace.of("metalake"), "catalogCopy", auditInfo); + + SchemaEntity schemaEntity = + createSchemaEntity(1L, Namespace.of("metalake", "catalog"), "schema1", auditInfo); + SchemaEntity schemaEntity2 = + createSchemaEntity(2L, Namespace.of("metalake", "catalog"), "schema2", auditInfo); + + TableEntity table = + createTableEntity( + 1L, Namespace.of("metalake", "catalog", "schema1"), "the same", auditInfo); + FilesetEntity filesetEntity = + createFilesetEntity( + 1L, Namespace.of("metalake", "catalog", "schema2"), "the same", auditInfo); + + store.put(metalake); + store.put(catalog); + store.put(catalogCopy); + store.put(schemaEntity); + store.put(schemaEntity2); + store.put(table); + store.put(filesetEntity); + + Assertions.assertDoesNotThrow( + () -> store.get(schemaEntity2.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); + Assertions.assertDoesNotThrow( + () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); + + // Test delete with the cascade or not + Assertions.assertThrows( + Exception.class, () -> store.delete(schemaEntity.nameIdentifier(), EntityType.SCHEMA)); + Assertions.assertDoesNotThrow( + () -> store.delete(schemaEntity.nameIdentifier(), EntityType.SCHEMA, true)); + + Assertions.assertDoesNotThrow( + () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); + + // Put the same schema back and see whether the deleted table exists or not + store.put(schemaEntity); + Assertions.assertDoesNotThrow( + () -> store.get(schemaEntity.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); + Assertions.assertThrows( + Exception.class, + () -> store.get(table.nameIdentifier(), EntityType.TABLE, TableEntity.class)); + Assertions.assertDoesNotThrow( + () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); + + store.put(table); + Assertions.assertDoesNotThrow( + () -> store.get(schemaEntity.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); + Assertions.assertDoesNotThrow( + () -> store.get(table.nameIdentifier(), EntityType.TABLE, TableEntity.class)); + Assertions.assertDoesNotThrow( + () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); + + store.delete(table.nameIdentifier(), EntityType.TABLE); + FilesetEntity filesetEntity1 = + createFilesetEntity( + 1L, Namespace.of("metalake", "catalog", "schema1"), "the same", auditInfo); + store.put(filesetEntity1); + Assertions.assertDoesNotThrow( + () -> + store.get(filesetEntity1.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); + } + } } From fc070178a34a2302bfec0cd97f552781b7a41cf3 Mon Sep 17 00:00:00 2001 From: xleoken Date: Wed, 17 Apr 2024 16:00:04 +0800 Subject: [PATCH 048/106] [MINOR] Fix incorrect numbering in the document (#2987) ### What changes were proposed in this pull request? Fix incorrect numbering in the document. ### Why are the changes needed? Fix incorrect numbering in the document. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? local. --- rfc/rfc-1/rfc-1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/rfc-1/rfc-1.md b/rfc/rfc-1/rfc-1.md index c791196aef5..06a6b134f22 100644 --- a/rfc/rfc-1/rfc-1.md +++ b/rfc/rfc-1/rfc-1.md @@ -30,7 +30,7 @@ ### Prerequisites 1. The field name of schema should use lowercase with "_" connecting different words, the name convention is `^[a-z](?:_?[a-z0-9]+)*$`. -1. The schema system uses Google Protobuf to describe the structs. +2. The schema system uses Google Protobuf to describe the structs. ### Schema Entities From e357f38d3ae64b34659ad323b2c97d539cc5b7bf Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Wed, 17 Apr 2024 16:23:12 +0800 Subject: [PATCH 049/106] [#2861][#2963][#2974] improve(web): add table properties and fileset fields displaying (#2972) ### What changes were proposed in this pull request? 1. add table properties displaying - partitioning - sortOrders - distribution - indexes image image image image image image 3. modified fileset icon image 4. fix tree list refresh table 5. add fileset `type` and `storageLocation` displaying image 6. fix login page logo icon error 9cc806574b49d211dd6b734bab9a01ce2ce0e944 ### Why are the changes needed? Fix: #2861 Fix: #2963 Fix: #2974 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --------- Co-authored-by: Qian Xia --- .../test/web/ui/CatalogsPageTest.java | 18 +-- .../test/web/ui/pages/CatalogsPage.java | 12 +- web/src/app/login/page.js | 7 +- .../app/metalakes/metalake/MetalakeTree.js | 4 +- .../rightContent/tabsContent/TabsContent.js | 137 +++++++++++++++++- .../tabsContent/detailsView/DetailsView.js | 16 ++ .../tabsContent/tableView/TableView.js | 78 +++++++++- web/src/lib/store/metalakes/index.js | 96 +++++++++++- 8 files changed, 341 insertions(+), 27 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java index 936bd17a15f..466500447c9 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java @@ -296,7 +296,7 @@ public void testEditCatalog() throws InterruptedException { public void testClickCatalogLink() { catalogsPage.clickCatalogLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); } @@ -306,7 +306,7 @@ public void testRefreshCatalogPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); List treeNodes = Arrays.asList( MODIFIED_CATALOG_NAME, @@ -329,7 +329,7 @@ public void testClickSchemaLink() { METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, COLUMN_NAME); catalogsPage.clickSchemaLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); } @@ -339,7 +339,7 @@ public void testRefreshSchemaPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); List treeNodes = Arrays.asList( MODIFIED_CATALOG_NAME, @@ -362,7 +362,7 @@ public void testClickTableLink() { METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); Assertions.assertTrue(catalogsPage.verifySelectedNode(TABLE_NAME)); } @@ -374,7 +374,7 @@ public void testRefreshTablePage() { Assertions.assertTrue(catalogsPage.verifyRefreshPage()); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); List treeNodes = Arrays.asList( MODIFIED_CATALOG_NAME, @@ -428,14 +428,14 @@ public void testClickTreeList() throws InterruptedException { METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); catalogsPage.clickTreeNode(schemaNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); String tableNode = String.format( "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); catalogsPage.clickTreeNode(tableNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); } @@ -458,7 +458,7 @@ public void testTreeNodeRefresh() throws InterruptedException { METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME_2); catalogsPage.clickTreeNode(tableNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME_2)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME_2, true)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java index 17b9c2d6aa3..8f8c1a73a20 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java @@ -450,13 +450,15 @@ public boolean verifyShowTableTitle(String title) { } } - public boolean verifyShowDataItemInList(String itemName) { + public boolean verifyShowDataItemInList(String itemName, Boolean isColumnLevel) { try { Thread.sleep(ACTION_SLEEP_MILLIS); - List list = - driver.findElements( - By.xpath( - "//div[@data-refer='table-grid']//div[contains(@class, 'MuiDataGrid-main')]/div[contains(@class, 'MuiDataGrid-virtualScroller')]/div/div[@role='rowgroup']//div[@data-field='name']")); + String xpath = + "//div[@data-refer='table-grid']//div[contains(@class, 'MuiDataGrid-main')]/div[contains(@class, 'MuiDataGrid-virtualScroller')]/div/div[@role='rowgroup']//div[@data-field='name']"; + if (isColumnLevel) { + xpath = xpath + "//p"; + } + List list = driver.findElements(By.xpath(xpath)); List texts = new ArrayList<>(); for (WebElement element : list) { texts.add(element.getText()); diff --git a/web/src/app/login/page.js b/web/src/app/login/page.js index 7ba6510ff43..d6efd0eff7f 100644 --- a/web/src/app/login/page.js +++ b/web/src/app/login/page.js @@ -62,7 +62,12 @@ const LoginPage = () => { - logo + logo Gravitino diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index 0b880448d9d..58141684290 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -58,6 +58,7 @@ const MetalakeTree = props => { case 'messaging': return 'skill-icons:kafka' case 'fileset': + return 'twemoji:file-folder' default: return 'bx:book' } @@ -70,7 +71,7 @@ const MetalakeTree = props => { case 'table': { if (store.selectedNodes.includes(nodeProps.data.key)) { const pathArr = extractPlaceholder(nodeProps.data.key) - const [metalake, catalog, schema, table] = pathArr + const [metalake, catalog, type, schema, table] = pathArr dispatch(getTableDetails({ init: true, metalake, catalog, schema, table })) } break @@ -263,6 +264,7 @@ const MetalakeTree = props => { useEffect(() => { dispatch(setExpandedNodes(store.expandedNodes)) + // eslint-disable-next-line react-hooks/exhaustive-deps }, [store.metalakeTree, dispatch]) return ( diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js index f7dfb7b0435..699d3bf069b 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js @@ -5,20 +5,24 @@ 'use client' +import { Inconsolata } from 'next/font/google' + import { useState, useEffect } from 'react' -import Box from '@mui/material/Box' -import Tab from '@mui/material/Tab' -import TabContext from '@mui/lab/TabContext' -import TabList from '@mui/lab/TabList' -import TabPanel from '@mui/lab/TabPanel' -import Typography from '@mui/material/Typography' +import { styled, Box, Divider, List, ListItem, ListItemText, Stack, Tab, Typography } from '@mui/material' +import Tooltip, { tooltipClasses } from '@mui/material/Tooltip' +import { TabContext, TabList, TabPanel } from '@mui/lab' + +import { useAppSelector } from '@/lib/hooks/useStore' + import { useSearchParams } from 'next/navigation' import TableView from './tableView/TableView' import DetailsView from './detailsView/DetailsView' import Icon from '@/components/Icon' +const fonts = Inconsolata({ subsets: ['latin'] }) + const CustomTab = props => { const { icon, label, value, ...others } = props @@ -38,6 +42,16 @@ const CustomTab = props => { ) } +const CustomTooltip = styled(({ className, ...props }) => )( + ({ theme }) => ({ + [`& .${tooltipClasses.tooltip}`]: { + backgroundColor: '#23282a', + padding: 0, + border: '1px solid #dadde9' + } + }) +) + const CustomTabPanel = props => { const { value, children, ...others } = props @@ -50,11 +64,13 @@ const CustomTabPanel = props => { const TabsContent = () => { let tableTitle = '' + const store = useAppSelector(state => state.metalakes) const searchParams = useSearchParams() const paramsSize = [...searchParams.keys()].length const type = searchParams.get('type') const [tab, setTab] = useState('table') const isNotNeedTableTab = type && ['fileset', 'messaging'].includes(type) && paramsSize === 5 + const isShowTableProps = paramsSize === 5 && !['fileset', 'messaging'].includes(type) const handleChangeTab = (event, newValue) => { setTab(newValue) @@ -98,13 +114,118 @@ const TabsContent = () => { return ( - - + + {!isNotNeedTableTab ? ( ) : null} + {isShowTableProps && ( + + + }> + {store.tableProps + .filter(i => i.items.length !== 0) + .map((item, index) => { + return ( + + + + {item.type}:{' '} + + + + + {item.items.map(i => { + return ( + + {item.type === 'sortOrders' ? i.text : i.fields} + + ) + })} + + + } + > + + theme.palette.text.primary + }} + > + + {item.type} + {' '} + + + + + } + secondary={ + + + {item.type === 'sortOrders' + ? item.items.map(i => i.text) + : item.items.map(i => i.fields)} + + + } + /> + + + ) + })} + + + + )} {!isNotNeedTableTab ? ( diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js index fb04dc81868..a15b287a259 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js @@ -73,6 +73,22 @@ const DetailsView = () => { ) : null} + {paramsSize === 5 && searchParams.get('fileset') ? ( + <> + + + Type + + {renderFieldText({ value: activatedItem?.type })} + + + + Storage location + + {renderFieldText({ value: activatedItem?.storageLocation })} + + + ) : null} Comment diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js index 4c0ee996f5b..1074e86aa43 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js @@ -5,11 +5,14 @@ 'use client' +import { Inconsolata } from 'next/font/google' + import { useState, useEffect } from 'react' import Link from 'next/link' -import { Box, Typography, IconButton } from '@mui/material' +import { styled, Box, Typography, IconButton, Stack } from '@mui/material' +import Tooltip, { tooltipClasses } from '@mui/material/Tooltip' import { DataGrid } from '@mui/x-data-grid' import { VisibilityOutlined as ViewIcon, @@ -17,6 +20,8 @@ import { DeleteOutlined as DeleteIcon } from '@mui/icons-material' +import Icon from '@/components/Icon' + import ColumnTypeChip from '@/components/ColumnTypeChip' import DetailsDrawer from '@/components/DetailsDrawer' import ConfirmDeleteDialog from '@/components/ConfirmDeleteDialog' @@ -29,6 +34,8 @@ import { to } from '@/lib/utils' import { getCatalogDetailsApi } from '@/lib/api/catalogs' import { useSearchParams } from 'next/navigation' +const fonts = Inconsolata({ subsets: ['latin'] }) + const EmptyText = () => { return ( theme.palette.text.disabled}> @@ -37,6 +44,16 @@ const EmptyText = () => { ) } +const CustomTooltip = styled(({ className, ...props }) => )( + ({ theme }) => ({ + [`& .${tooltipClasses.tooltip}`]: { + backgroundColor: '#23282a', + padding: 0, + border: '1px solid #dadde9' + } + }) +) + const TableView = () => { const searchParams = useSearchParams() const paramsSize = [...searchParams.keys()].length @@ -64,6 +81,56 @@ const TableView = () => { } } + const renderIconTooltip = (type, name) => { + const propsItem = store.tableProps.find(i => i.type === type) + const items = propsItem?.items || [] + + const isCond = propsItem?.items.find(i => i.fields.find(v => (Array.isArray(v) ? v.includes(name) : v === name))) + + const icon = propsItem?.icon + + return ( + <> + {isCond && ( + + + + {type}:{' '} + + + + + {items.map(i => { + return ( + + {i.text || i.fields} + + ) + })} + + + } + > + + + + + )} + + ) + } + const columns = [ { flex: 0.1, @@ -182,7 +249,7 @@ const TableView = () => { const tableColumns = [ { - flex: 0.1, + flex: 0.15, minWidth: 60, disableColumnMenu: true, type: 'string', @@ -197,6 +264,7 @@ const TableView = () => { title={name} noWrap sx={{ + pr: 4, fontWeight: 400, color: 'text.main', textDecoration: 'none' @@ -204,6 +272,12 @@ const TableView = () => { > {name} + + {renderIconTooltip('partitioning', name)} + {renderIconTooltip('sortOrders', name)} + {renderIconTooltip('distribution', name)} + {renderIconTooltip('indexes', name)} + ) } diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js index 912f7bac8a6..eeba3508a4c 100644 --- a/web/src/lib/store/metalakes/index.js +++ b/web/src/lib/store/metalakes/index.js @@ -600,6 +600,94 @@ export const getTableDetails = createAsyncThunk( const { table: resTable } = res + const { distribution, sortOrders = [], partitioning = [], indexes = [] } = resTable + + const tableProps = [ + { + type: 'partitioning', + icon: 'tabler:circle-letter-p-filled', + items: partitioning.map(i => { + let fields = i.fieldName || [] + let sub = '' + let last = i.fieldName.join('.') + + switch (i.strategy) { + case 'bucket': + sub = `[${i.numBuckets}]` + fields = i.fieldNames + last = i.fieldNames.map(v => v[0]).join(',') + break + case 'truncate': + sub = `[${i.width}]` + break + case 'list': + fields = i.fieldNames + last = i.fieldNames.map(v => v[0]).join(',') + break + case 'function': + sub = `[${i.funcName}]` + fields = i.funcArgs.map(v => v.fieldName) + last = fields.join(',') + break + default: + break + } + + return { + strategy: i.strategy, + numBuckets: i.numBuckets, + width: i.width, + funcName: i.funcName, + fields, + text: `${i.strategy}${sub}(${last})` + } + }) + }, + { + type: 'sortOrders', + icon: 'mdi:letter-s-circle', + items: sortOrders.map(i => { + return { + fields: i.sortTerm.fieldName, + dir: i.direction, + no: i.nullOrdering, + text: `${i.sortTerm.fieldName[0]} ${i.direction} ${i.nullOrdering}` + } + }) + }, + { + type: 'distribution', + icon: 'tabler:circle-letter-d-filled', + items: distribution.funcArgs.map(i => { + return { + fields: i.fieldName, + number: distribution.number, + strategy: distribution.strategy, + text: + distribution.strategy === '' + ? `` + : `${distribution.strategy}${ + distribution.number === 0 ? '' : `[${distribution.number}]` + }(${i.fieldName.join(',')})` + } + }) + }, + { + type: 'indexes', + icon: 'mdi:letter-i-circle', + items: indexes.map(i => { + return { + fields: i.fieldNames, + name: i.name, + indexType: i.indexType, + text: `${i.name}(${i.fieldNames.join(',')})` + } + }) + } + ] + + dispatch(setTableProps(tableProps)) + if (getState().metalakes.metalakeTree.length === 0) { dispatch(fetchCatalogs({ metalake })) } @@ -818,6 +906,7 @@ export const appMetalakesSlice = createSlice({ metalakes: [], filteredMetalakes: [], tableData: [], + tableProps: [], catalogs: [], schemas: [], tables: [], @@ -870,6 +959,7 @@ export const appMetalakesSlice = createSlice({ state.loadedNodes = [] state.tableData = [] + state.tableProps = [] state.catalogs = [] state.schemas = [] state.tables = [] @@ -898,6 +988,9 @@ export const appMetalakesSlice = createSlice({ removeCatalogFromTree(state, action) { state.metalakeTree = state.metalakeTree.filter(i => i.key !== action.payload) }, + setTableProps(state, action) { + state.tableProps = action.payload + }, resetMetalakeStore(state, action) {} }, extraReducers: builder => { @@ -1032,7 +1125,8 @@ export const { setExpanded, setExpandedNodes, addCatalogToTree, - removeCatalogFromTree + removeCatalogFromTree, + setTableProps } = appMetalakesSlice.actions export default appMetalakesSlice.reducer From e2d1fd0091cf89d6ccbcc386999832f9ad82c780 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 17 Apr 2024 16:52:06 +0800 Subject: [PATCH 050/106] [#2856] fix(core): Fix the namespace missing issue when get entities from kv storage (#2971) ### What changes were proposed in this pull request? This PR fixes the namespace missing issue when entities are gotten from kv storage. ### Why are the changes needed? Fix: #2856 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Modified UTs. --- build.gradle.kts | 3 +- .../com/datastrato/gravitino/EntitySerDe.java | 10 +- .../gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/meta/CatalogEntity.java | 14 +-- .../gravitino/meta/FilesetEntity.java | 1 + .../gravitino/meta/GroupEntity.java | 1 + .../datastrato/gravitino/meta/RoleEntity.java | 1 + .../gravitino/meta/SchemaEntity.java | 1 + .../gravitino/meta/TableEntity.java | 1 + .../gravitino/meta/TopicEntity.java | 1 + .../datastrato/gravitino/meta/UserEntity.java | 1 + .../gravitino/proto/AuditInfoSerDe.java | 3 +- .../gravitino/proto/BaseMetalakeSerDe.java | 5 +- .../gravitino/proto/CatalogEntitySerDe.java | 6 +- .../gravitino/proto/FilesetEntitySerDe.java | 6 +- .../gravitino/proto/GroupEntitySerDe.java | 6 +- .../gravitino/proto/ProtoEntitySerDe.java | 10 +- .../gravitino/proto/ProtoSerDe.java | 4 +- .../gravitino/proto/RoleEntitySerDe.java | 6 +- .../gravitino/proto/SchemaEntitySerDe.java | 6 +- .../gravitino/proto/TableEntitySerDe.java | 6 +- .../gravitino/proto/TopicEntitySerDe.java | 6 +- .../gravitino/proto/UserEntitySerDe.java | 6 +- .../gravitino/storage/kv/KvEntityStore.java | 6 +- .../gravitino/proto/TestEntityProtoSerDe.java | 98 ++++++++++++++----- 25 files changed, 142 insertions(+), 68 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7d383ced5ff..c70b4fc7cf3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -539,7 +539,8 @@ tasks.rat { "**/LICENSE.*", "**/NOTICE.*", "ROADMAP.md", - "clients/client-python/.pytest_cache/*" + "clients/client-python/.pytest_cache/*", + "clients/client-python/gravitino.egg-info/*" ) // Add .gitignore excludes to the Apache Rat exclusion list. diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java index 1ebcb9c0663..103ed34ed83 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java @@ -24,15 +24,17 @@ public interface EntitySerDe { * * @param bytes the byte array to deserialize * @param clazz the class of the entity + * @param namespace the namespace to use when deserializing the entity * @return the deserialized entity * @param The type of entity * @throws IOException if the deserialization fails */ - default T deserialize(byte[] bytes, Class clazz) throws IOException { + default T deserialize(byte[] bytes, Class clazz, Namespace namespace) + throws IOException { ClassLoader loader = Optional.ofNullable(Thread.currentThread().getContextClassLoader()) .orElse(getClass().getClassLoader()); - return deserialize(bytes, clazz, loader); + return deserialize(bytes, clazz, loader, namespace); } /** @@ -41,10 +43,12 @@ default T deserialize(byte[] bytes, Class clazz) throws IO * @param bytes the byte array to deserialize * @param clazz the class of the entity * @param classLoader the class loader to use + * @param namespace the namespace to use when deserializing the entity * @return the deserialized entity * @param The type of entity * @throws IOException if the deserialization fails */ - T deserialize(byte[] bytes, Class clazz, ClassLoader classLoader) + T deserialize( + byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) throws IOException; } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 884a5adfff0..266786da59d 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -528,7 +528,7 @@ private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeExce private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); - return createCatalogWrapper(entity.withNamespace(ident.namespace())); + return createCatalogWrapper(entity); } catch (NoSuchEntityException ne) { LOG.warn("Catalog {} does not exist", ident, ne); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java index 2a2d7757d2c..e1ce59ccace 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.connector.CatalogInfo; -import com.datastrato.gravitino.proto.CatalogEntitySerDe; import com.google.common.base.Objects; import java.util.Collections; import java.util.HashMap; @@ -124,18 +123,6 @@ public CatalogInfo toCatalogInfo() { return new CatalogInfo(id, name, type, provider, comment, properties, auditInfo, namespace); } - /** - * Sets the namespace of the catalog entity. because the {@link CatalogEntitySerDe} does not - * serialize the namespace field - * - * @param namespace the namespace of the catalog entity. - * @return the instance of the source catalog entity. - */ - public CatalogEntity withNamespace(Namespace namespace) { - this.namespace = namespace; - return this; - } - /** Builder class for creating instances of {@link CatalogEntity}. */ public static class Builder { @@ -263,6 +250,7 @@ public boolean equals(Object o) { CatalogEntity that = (CatalogEntity) o; return Objects.equal(id, that.id) && Objects.equal(name, that.name) + && Objects.equal(namespace, that.namespace) && type == that.type && Objects.equal(provider, that.provider) && Objects.equal(comment, that.comment) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java index ae6ccb97f48..0f8726a313b 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java @@ -166,6 +166,7 @@ public boolean equals(Object o) { FilesetEntity that = (FilesetEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(comment, that.comment) && Objects.equals(type, that.type) && Objects.equals(storageLocation, that.storageLocation) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java index 9ae0d8cee07..e00b59b908b 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java @@ -120,6 +120,7 @@ public boolean equals(Object o) { GroupEntity that = (GroupEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(roles, that.roles); } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java index b4642f3e5db..8e9ec407f8d 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -152,6 +152,7 @@ public boolean equals(Object o) { RoleEntity that = (RoleEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) && Objects.equals(securableObject, that.securableObject) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java index 9805af07a1c..46e747992ea 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java @@ -145,6 +145,7 @@ public boolean equals(Object o) { SchemaEntity schema = (SchemaEntity) o; return Objects.equal(id, schema.id) && Objects.equal(name, schema.name) + && Objects.equal(namespace, schema.namespace) && Objects.equal(comment, schema.comment) && Objects.equal(properties, schema.properties) && Objects.equal(auditInfo, schema.auditInfo); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java index 5275f9dac81..37d1ee5a984 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java @@ -109,6 +109,7 @@ public boolean equals(Object o) { TableEntity baseTable = (TableEntity) o; return Objects.equal(id, baseTable.id) && Objects.equal(name, baseTable.name) + && Objects.equal(namespace, baseTable.namespace) && Objects.equal(auditInfo, baseTable.auditInfo); } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java index 8b0a3180314..898e9b35bba 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java @@ -135,6 +135,7 @@ public boolean equals(Object o) { TopicEntity that = (TopicEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(comment, that.comment) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java index 11a6fe6978a..933ecb1eb5e 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java @@ -124,6 +124,7 @@ public boolean equals(Object o) { UserEntity that = (UserEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(roles, that.roles); } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java index 45058f00979..7891af5ccc5 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import java.util.Optional; /** A class for serializing and deserializing AuditInfo objects. */ @@ -41,7 +42,7 @@ public AuditInfo serialize(com.datastrato.gravitino.meta.AuditInfo auditInfo) { * @return The deserialized AuditInfo object. */ @Override - public com.datastrato.gravitino.meta.AuditInfo deserialize(AuditInfo p) { + public com.datastrato.gravitino.meta.AuditInfo deserialize(AuditInfo p, Namespace namespace) { com.datastrato.gravitino.meta.AuditInfo.Builder builder = com.datastrato.gravitino.meta.AuditInfo.builder(); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java index ad90e5fc152..0f9f09153dd 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.AuditInfo; /** A class for serializing and deserializing BaseMetalake objects. */ @@ -52,13 +53,13 @@ public Metalake serialize(com.datastrato.gravitino.meta.BaseMetalake baseMetalak * @return The deserialized BaseMetalake object. */ @Override - public com.datastrato.gravitino.meta.BaseMetalake deserialize(Metalake p) { + public com.datastrato.gravitino.meta.BaseMetalake deserialize(Metalake p, Namespace namespace) { com.datastrato.gravitino.meta.BaseMetalake.Builder builder = com.datastrato.gravitino.meta.BaseMetalake.builder(); builder .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java index 2aafa3d3d85..382761da4d9 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; @@ -49,13 +50,14 @@ public Catalog serialize(CatalogEntity catalogEntity) { * @return The deserialized CatalogEntity object. */ @Override - public CatalogEntity deserialize(Catalog p) { + public CatalogEntity deserialize(Catalog p, Namespace namespace) { CatalogEntity.Builder builder = CatalogEntity.builder(); builder .withId(p.getId()) .withName(p.getName()) + .withNamespace(namespace) .withProvider(p.getProvider()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java index ef724ecb41a..24680452739 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.FilesetEntity; public class FilesetEntitySerDe implements ProtoSerDe { @@ -31,13 +32,14 @@ public Fileset serialize(FilesetEntity filesetEntity) { } @Override - public FilesetEntity deserialize(Fileset p) { + public FilesetEntity deserialize(Fileset p, Namespace namespace) { FilesetEntity.Builder builder = FilesetEntity.builder() .withId(p.getId()) .withName(p.getName()) + .withNamespace(namespace) .withStorageLocation(p.getStorageLocation()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) .withFilesetType( com.datastrato.gravitino.file.Fileset.Type.valueOf(p.getType().name())); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java index 083b69029d8..e1fa6ab50c2 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.GroupEntity; import java.util.Collection; @@ -25,12 +26,13 @@ public Group serialize(GroupEntity groupEntity) { } @Override - public GroupEntity deserialize(Group group) { + public GroupEntity deserialize(Group group, Namespace namespace) { GroupEntity.Builder builder = GroupEntity.builder() .withId(group.getId()) .withName(group.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); if (group.getRolesCount() > 0) { builder.withRoles(group.getRolesList()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index b826b4cedb3..894607d7dbf 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntitySerDe; +import com.datastrato.gravitino.Namespace; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.protobuf.Any; @@ -91,7 +92,8 @@ public byte[] serialize(T t) throws IOException { } @Override - public T deserialize(byte[] bytes, Class clazz, ClassLoader classLoader) + public T deserialize( + byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) throws IOException { Any any = Any.parseFrom(bytes); Class protoClass = getProtoClass(clazz, classLoader); @@ -101,7 +103,7 @@ public T deserialize(byte[] bytes, Class clazz, ClassLoade } Message anyMessage = any.unpack(protoClass); - return fromProto(anyMessage, clazz, classLoader); + return fromProto(anyMessage, clazz, classLoader, namespace); } private ProtoSerDe getProtoSerde( @@ -151,9 +153,9 @@ private M toProto(T t, ClassLoader classLo } private T fromProto( - M m, Class entityClass, ClassLoader classLoader) throws IOException { + M m, Class entityClass, ClassLoader classLoader, Namespace namespace) throws IOException { ProtoSerDe protoSerDe = getProtoSerde(entityClass, classLoader); - return protoSerDe.deserialize(m); + return protoSerDe.deserialize(m, namespace); } private Class loadClass(String className, ClassLoader classLoader) throws IOException { diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java index 324f6a76321..33e44b60fa4 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.google.protobuf.Message; /** @@ -26,7 +27,8 @@ public interface ProtoSerDe { * Deserializes the provided Protocol Buffer message into its corresponding entity representation. * * @param p The Protocol Buffer message to be deserialized. + * @param namespace The namespace to be specified for entity deserialization. * @return The entity representing the deserialized Protocol Buffer message. */ - T deserialize(M p); + T deserialize(M p, Namespace namespace); } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index e4a96ae7e63..841a8f9c15e 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.RoleEntity; @@ -44,17 +45,18 @@ public Role serialize(RoleEntity roleEntity) { * @return The entity representing the deserialized Protocol Buffer message. */ @Override - public RoleEntity deserialize(Role role) { + public RoleEntity deserialize(Role role, Namespace namespace) { RoleEntity.Builder builder = RoleEntity.builder() .withId(role.getId()) .withName(role.getName()) + .withNamespace(namespace) .withPrivileges( role.getPrivilegesList().stream() .map(Privileges::fromString) .collect(Collectors.toList())) .withSecurableObject(SecurableObjects.parse(role.getSecurableObject())) - .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); if (!role.getPropertiesMap().isEmpty()) { builder.withProperties(role.getPropertiesMap()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java index 8bef65834d1..1df134bbd4e 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.SchemaEntity; public class SchemaEntitySerDe implements ProtoSerDe { @@ -27,12 +28,13 @@ public Schema serialize(SchemaEntity schemaEntity) { } @Override - public SchemaEntity deserialize(Schema p) { + public SchemaEntity deserialize(Schema p, Namespace namespace) { SchemaEntity.Builder builder = SchemaEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java index 06999f8fa8d..b2d79c4f7c9 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.TableEntity; public class TableEntitySerDe implements ProtoSerDe { @@ -17,11 +18,12 @@ public Table serialize(TableEntity tableEntity) { } @Override - public TableEntity deserialize(Table p) { + public TableEntity deserialize(Table p, Namespace namespace) { return TableEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())) + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) .build(); } } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java index c784030dc47..1fb6ce92275 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.TopicEntity; public class TopicEntitySerDe implements ProtoSerDe { @@ -28,12 +29,13 @@ public Topic serialize(TopicEntity topicEntity) { } @Override - public TopicEntity deserialize(Topic p) { + public TopicEntity deserialize(Topic p, Namespace namespace) { TopicEntity.Builder builder = TopicEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java index e87f3314361..47e7f2bb18b 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.UserEntity; import java.util.Collection; @@ -25,12 +26,13 @@ public User serialize(UserEntity userEntity) { } @Override - public UserEntity deserialize(User user) { + public UserEntity deserialize(User user, Namespace namespace) { UserEntity.Builder builder = UserEntity.builder() .withId(user.getId()) .withName(user.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); if (user.getRolesCount() > 0) { builder.withRoles(user.getRolesList()); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 0fafc0c1745..917558888ab 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -135,7 +135,7 @@ public List list( .limit(Integer.MAX_VALUE) .build())); for (Pair pairs : kvs) { - entities.add(serDe.deserialize(pairs.getRight(), e)); + entities.add(serDe.deserialize(pairs.getRight(), e, namespace)); } // TODO (yuqi), if the list is too large, we need to do pagination or streaming return entities; @@ -177,7 +177,7 @@ public E update( throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); } - E e = serDe.deserialize(value, type); + E e = serDe.deserialize(value, type, ident.namespace()); E updatedE = updater.apply(e); if (updatedE.nameIdentifier().equals(ident)) { transactionalKvBackend.put(key, serDe.serialize(updatedE), true); @@ -219,7 +219,7 @@ public E get( if (value == null) { throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); } - return serDe.deserialize(value, e); + return serDe.deserialize(value, e, ident.namespace()); } void deleteAuthorizationEntitiesIfNecessary(NameIdentifier ident, EntityType type) diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index 8223ac239c9..d0b91b79152 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -4,8 +4,10 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntitySerDe; import com.datastrato.gravitino.EntitySerDeFactory; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.GroupEntity; @@ -42,7 +44,8 @@ public void testAuditInfoSerDe() throws IOException { byte[] bytes = protoEntitySerDe.serialize(auditInfo); com.datastrato.gravitino.meta.AuditInfo auditInfoFromBytes = - protoEntitySerDe.deserialize(bytes, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo, auditInfoFromBytes); // Test with optional fields @@ -55,7 +58,8 @@ public void testAuditInfoSerDe() throws IOException { // Test from/to bytes bytes = protoEntitySerDe.serialize(auditInfo1); auditInfoFromBytes = - protoEntitySerDe.deserialize(bytes, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo1, auditInfoFromBytes); // Test with empty field @@ -64,7 +68,8 @@ public void testAuditInfoSerDe() throws IOException { byte[] bytes1 = protoEntitySerDe.serialize(auditInfo2); com.datastrato.gravitino.meta.AuditInfo auditInfoFromBytes1 = - protoEntitySerDe.deserialize(bytes1, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes1, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo2, auditInfoFromBytes1); } @@ -98,7 +103,7 @@ public void testEntitiesSerDe() throws IOException { byte[] metalakeBytes = protoEntitySerDe.serialize(metalake); com.datastrato.gravitino.meta.BaseMetalake metalakeFromBytes = protoEntitySerDe.deserialize( - metalakeBytes, com.datastrato.gravitino.meta.BaseMetalake.class); + metalakeBytes, com.datastrato.gravitino.meta.BaseMetalake.class, Namespace.empty()); Assertions.assertEquals(metalake, metalakeFromBytes); // Test metalake without props map @@ -113,7 +118,7 @@ public void testEntitiesSerDe() throws IOException { byte[] metalakeBytes1 = protoEntitySerDe.serialize(metalake1); com.datastrato.gravitino.meta.BaseMetalake metalakeFromBytes1 = protoEntitySerDe.deserialize( - metalakeBytes1, com.datastrato.gravitino.meta.BaseMetalake.class); + metalakeBytes1, com.datastrato.gravitino.meta.BaseMetalake.class, Namespace.empty()); Assertions.assertEquals(metalake1, metalakeFromBytes1); // Test CatalogEntity @@ -121,11 +126,13 @@ public void testEntitiesSerDe() throws IOException { String catalogName = "catalog"; String comment = "comment"; String provider = "test"; + Namespace catalogNamespace = Namespace.of("metalake"); com.datastrato.gravitino.meta.CatalogEntity catalogEntity = com.datastrato.gravitino.meta.CatalogEntity.builder() .withId(catalogId) .withName(catalogName) + .withNamespace(catalogNamespace) .withComment(comment) .withType(com.datastrato.gravitino.Catalog.Type.RELATIONAL) .withProvider(provider) @@ -135,7 +142,7 @@ public void testEntitiesSerDe() throws IOException { byte[] catalogBytes = protoEntitySerDe.serialize(catalogEntity); com.datastrato.gravitino.meta.CatalogEntity catalogEntityFromBytes = protoEntitySerDe.deserialize( - catalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class); + catalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class, catalogNamespace); Assertions.assertEquals(catalogEntity, catalogEntityFromBytes); // Test Fileset catalog @@ -143,6 +150,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.CatalogEntity.builder() .withId(catalogId) .withName(catalogName) + .withNamespace(catalogNamespace) .withComment(comment) .withType(com.datastrato.gravitino.Catalog.Type.FILESET) .withProvider(provider) @@ -151,22 +159,27 @@ public void testEntitiesSerDe() throws IOException { byte[] filesetCatalogBytes = protoEntitySerDe.serialize(filesetCatalogEntity); com.datastrato.gravitino.meta.CatalogEntity filesetCatalogEntityFromBytes = protoEntitySerDe.deserialize( - filesetCatalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class); + filesetCatalogBytes, + com.datastrato.gravitino.meta.CatalogEntity.class, + catalogNamespace); Assertions.assertEquals(filesetCatalogEntity, filesetCatalogEntityFromBytes); // Test SchemaEntity + Namespace schemaNamespace = Namespace.of("metalake", "catalog"); Long schemaId = 1L; String schemaName = "schema"; com.datastrato.gravitino.meta.SchemaEntity schemaEntity = com.datastrato.gravitino.meta.SchemaEntity.builder() .withId(schemaId) .withName(schemaName) + .withNamespace(schemaNamespace) .withAuditInfo(auditInfo) .build(); byte[] schemaBytes = protoEntitySerDe.serialize(schemaEntity); com.datastrato.gravitino.meta.SchemaEntity schemaEntityFromBytes = - protoEntitySerDe.deserialize(schemaBytes, com.datastrato.gravitino.meta.SchemaEntity.class); + protoEntitySerDe.deserialize( + schemaBytes, com.datastrato.gravitino.meta.SchemaEntity.class, schemaNamespace); Assertions.assertEquals(schemaEntity, schemaEntityFromBytes); // Test SchemaEntity with additional fields @@ -174,6 +187,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.SchemaEntity.builder() .withId(schemaId) .withName(schemaName) + .withNamespace(schemaNamespace) .withAuditInfo(auditInfo) .withComment(comment) .withProperties(props) @@ -181,33 +195,38 @@ public void testEntitiesSerDe() throws IOException { byte[] schemaBytes1 = protoEntitySerDe.serialize(schemaEntity1); com.datastrato.gravitino.meta.SchemaEntity schemaEntityFromBytes1 = protoEntitySerDe.deserialize( - schemaBytes1, com.datastrato.gravitino.meta.SchemaEntity.class); + schemaBytes1, com.datastrato.gravitino.meta.SchemaEntity.class, schemaNamespace); Assertions.assertEquals(schemaEntity1, schemaEntityFromBytes1); Assertions.assertEquals(comment, schemaEntityFromBytes1.comment()); Assertions.assertEquals(props, schemaEntityFromBytes1.properties()); // Test TableEntity + Namespace tableNamespace = Namespace.of("metalake", "catalog", "schema"); Long tableId = 1L; String tableName = "table"; com.datastrato.gravitino.meta.TableEntity tableEntity = com.datastrato.gravitino.meta.TableEntity.builder() .withId(tableId) .withName(tableName) + .withNamespace(tableNamespace) .withAuditInfo(auditInfo) .build(); byte[] tableBytes = protoEntitySerDe.serialize(tableEntity); com.datastrato.gravitino.meta.TableEntity tableEntityFromBytes = - protoEntitySerDe.deserialize(tableBytes, com.datastrato.gravitino.meta.TableEntity.class); + protoEntitySerDe.deserialize( + tableBytes, com.datastrato.gravitino.meta.TableEntity.class, tableNamespace); Assertions.assertEquals(tableEntity, tableEntityFromBytes); // Test FileEntity + Namespace filesetNamespace = Namespace.of("metalake", "catalog", "schema"); Long fileId = 1L; String fileName = "file"; com.datastrato.gravitino.meta.FilesetEntity fileEntity = com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.MANAGED) .withStorageLocation("testLocation") @@ -216,20 +235,23 @@ public void testEntitiesSerDe() throws IOException { .build(); byte[] fileBytes = protoEntitySerDe.serialize(fileEntity); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes = - protoEntitySerDe.deserialize(fileBytes, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity, fileEntityFromBytes); com.datastrato.gravitino.meta.FilesetEntity fileEntity1 = com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.MANAGED) .withStorageLocation("testLocation") .build(); byte[] fileBytes1 = protoEntitySerDe.serialize(fileEntity1); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes1 = - protoEntitySerDe.deserialize(fileBytes1, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes1, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity1, fileEntityFromBytes1); Assertions.assertNull(fileEntityFromBytes1.comment()); Assertions.assertNull(fileEntityFromBytes1.properties()); @@ -238,6 +260,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.EXTERNAL) .withProperties(props) @@ -246,63 +269,80 @@ public void testEntitiesSerDe() throws IOException { .build(); byte[] fileBytes2 = protoEntitySerDe.serialize(fileEntity2); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes2 = - protoEntitySerDe.deserialize(fileBytes2, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes2, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity2, fileEntityFromBytes2); Assertions.assertEquals("testLocation", fileEntityFromBytes2.storageLocation()); Assertions.assertEquals( com.datastrato.gravitino.file.Fileset.Type.EXTERNAL, fileEntityFromBytes2.filesetType()); // Test TopicEntity + Namespace topicNamespace = Namespace.of("metalake", "catalog", "default"); Long topicId = 1L; String topicName = "topic"; com.datastrato.gravitino.meta.TopicEntity topicEntity = com.datastrato.gravitino.meta.TopicEntity.builder() .withId(topicId) .withName(topicName) + .withNamespace(topicNamespace) .withAuditInfo(auditInfo) .withComment(comment) .withProperties(props) .build(); byte[] topicBytes = protoEntitySerDe.serialize(topicEntity); com.datastrato.gravitino.meta.TopicEntity topicEntityFromBytes = - protoEntitySerDe.deserialize(topicBytes, com.datastrato.gravitino.meta.TopicEntity.class); + protoEntitySerDe.deserialize( + topicBytes, com.datastrato.gravitino.meta.TopicEntity.class, topicNamespace); Assertions.assertEquals(topicEntity, topicEntityFromBytes); com.datastrato.gravitino.meta.TopicEntity topicEntity1 = com.datastrato.gravitino.meta.TopicEntity.builder() .withId(topicId) .withName(topicName) + .withNamespace(topicNamespace) .withAuditInfo(auditInfo) .build(); byte[] topicBytes1 = protoEntitySerDe.serialize(topicEntity1); com.datastrato.gravitino.meta.TopicEntity topicEntityFromBytes1 = - protoEntitySerDe.deserialize(topicBytes1, com.datastrato.gravitino.meta.TopicEntity.class); + protoEntitySerDe.deserialize( + topicBytes1, com.datastrato.gravitino.meta.TopicEntity.class, topicNamespace); Assertions.assertEquals(topicEntity1, topicEntityFromBytes1); Assertions.assertNull(topicEntityFromBytes1.comment()); Assertions.assertNull(topicEntityFromBytes1.properties()); // Test UserEntity + Namespace userNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); Long userId = 1L; String userName = "user"; UserEntity userEntity = UserEntity.builder() .withId(userId) .withName(userName) + .withNamespace(userNamespace) .withAuditInfo(auditInfo) .withRoles(Lists.newArrayList("role")) .build(); byte[] userBytes = protoEntitySerDe.serialize(userEntity); - UserEntity userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class); + UserEntity userEntityFromBytes = + protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); Assertions.assertEquals(userEntity, userEntityFromBytes); UserEntity userEntityWithoutFields = - UserEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); + UserEntity.builder() + .withId(userId) + .withName(userName) + .withNamespace(userNamespace) + .withAuditInfo(auditInfo) + .build(); userBytes = protoEntitySerDe.serialize(userEntityWithoutFields); - userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class); + userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); Assertions.assertNull(userEntityWithoutFields.roles()); // Test GroupEntity + Namespace groupNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME); Long groupId = 1L; String groupName = "group"; @@ -310,46 +350,58 @@ public void testEntitiesSerDe() throws IOException { GroupEntity.builder() .withId(groupId) .withName(groupName) + .withNamespace(groupNamespace) .withAuditInfo(auditInfo) .withRoles(Lists.newArrayList("role")) .build(); byte[] groupBytes = protoEntitySerDe.serialize(group); - GroupEntity groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class); + GroupEntity groupFromBytes = + protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); Assertions.assertEquals(group, groupFromBytes); GroupEntity groupWithoutFields = - GroupEntity.builder().withId(groupId).withName(groupName).withAuditInfo(auditInfo).build(); + GroupEntity.builder() + .withId(groupId) + .withName(groupName) + .withNamespace(groupNamespace) + .withAuditInfo(auditInfo) + .build(); groupBytes = protoEntitySerDe.serialize(groupWithoutFields); - groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class); + groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); Assertions.assertEquals(groupWithoutFields, groupFromBytes); Assertions.assertNull(groupWithoutFields.roles()); // Test RoleEntity + Namespace roleNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); Long roleId = 1L; String roleName = "testRole"; RoleEntity roleEntity = RoleEntity.builder() .withId(roleId) .withName(roleName) + .withNamespace(roleNamespace) .withAuditInfo(auditInfo) .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withProperties(props) .build(); byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); - RoleEntity roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + RoleEntity roleFromBytes = + protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); Assertions.assertEquals(roleEntity, roleFromBytes); RoleEntity roleWithoutFields = RoleEntity.builder() .withId(1L) .withName(roleName) + .withNamespace(roleNamespace) .withAuditInfo(auditInfo) .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .build(); roleBytes = protoEntitySerDe.serialize(roleWithoutFields); - roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); Assertions.assertEquals(roleWithoutFields, roleFromBytes); } } From 6cdd9aa7e2bf22f1f49be2a65f0dca1779c3ead3 Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Wed, 17 Apr 2024 19:56:54 +0800 Subject: [PATCH 051/106] [#2970]fix(UI): disabled update the location property of fileset catalog (#2995) ### What changes were proposed in this pull request? Disabled update the "location" property item for fileset catalog image ### Why are the changes needed? The "location" property of fileset catalog is immutable or reserved Fix: #2970 ### Does this PR introduce _any_ user-facing change? Edit a fileset catalog and open the dialog ### How was this patch tested? local e2e test image --- .../metalakes/metalake/rightContent/CreateCatalogDialog.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index 25bb5108df6..f3005fcd4f5 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -386,7 +386,8 @@ const CreateCatalogDialog = props => { if (findPropIndex === -1) { let propItem = { key: item, - value: properties[item] + value: properties[item], + disabled: data.type === 'fileset' && item === 'location' && type === 'update' } propsItems.push(propItem) } @@ -552,7 +553,7 @@ const CreateCatalogDialog = props => { name='key' label='Key' value={item.key} - disabled={item.required} + disabled={item.required || item.disabled} onChange={event => handleFormChange({ index, event })} error={item.hasDuplicateKey} data-refer={`props-key-${index}`} @@ -591,7 +592,7 @@ const CreateCatalogDialog = props => { )} - {!item.required ? ( + {!(item.required || item.disabled) ? ( removeFields(index)}> From eb1f11eefc85fc3fb3ac135e39e08462931d6ed8 Mon Sep 17 00:00:00 2001 From: Peidian li <38486782+coolderli@users.noreply.github.com> Date: Wed, 17 Apr 2024 21:26:56 +0800 Subject: [PATCH 052/106] [#2976] improve(api): support header for gravitino client (#2991) ### What changes were proposed in this pull request? - Support pass header when using the Java Gravitino Client. ### Why are the changes needed? - For now, we can't pass the useful header on the Java Gravitino Client. We want to pass on more information such as some audit messages. The base header may not be used for now, but it is useful for users to integrate with their platform. Fix: #2976 ### Does this PR introduce _any_ user-facing change? - yes 1. Change in user-facing APIs. - add `withHeaders` in GravitinoClientBuilder. ``` GravitinoClient.builder("uri") .withMetalake("metalake") .withHeaders(Map) // add header .builder(); GravitinoAdminClient.builder("uri") .withHeaders(Map) // add header .builder(); ``` 2. Addition or removal of property keys. - add withHeaders() method ### How was this patch tested? - original Uts --- .../client/GravitinoAdminClient.java | 8 +- .../gravitino/client/GravitinoClient.java | 11 ++- .../gravitino/client/GravitinoClientBase.java | 22 +++++- .../client/TestGravitinoClientBuilder.java | 79 +++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index 3429414c521..59393e8f58c 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -40,9 +40,11 @@ public class GravitinoAdminClient extends GravitinoClientBase implements Support * * @param uri The base URI for the Gravitino API. * @param authDataProvider The provider of the data which is used for authentication. + * @param headers The base header for Gravitino API. */ - private GravitinoAdminClient(String uri, AuthDataProvider authDataProvider) { - super(uri, authDataProvider); + private GravitinoAdminClient( + String uri, AuthDataProvider authDataProvider, Map headers) { + super(uri, authDataProvider, headers); } /** @@ -188,7 +190,7 @@ public GravitinoAdminClient build() { Preconditions.checkArgument( uri != null && !uri.isEmpty(), "The argument 'uri' must be a valid URI"); - return new GravitinoAdminClient(uri, authDataProvider); + return new GravitinoAdminClient(uri, authDataProvider, headers); } } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java index dd391266b6d..f5584adb625 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java @@ -33,10 +33,15 @@ public class GravitinoClient extends GravitinoClientBase implements SupportsCata * @param uri The base URI for the Gravitino API. * @param metalakeName The specified metalake name. * @param authDataProvider The provider of the data which is used for authentication. + * @param headers The base header for Gravitino API. * @throws NoSuchMetalakeException if the metalake with specified name does not exist. */ - private GravitinoClient(String uri, String metalakeName, AuthDataProvider authDataProvider) { - super(uri, authDataProvider); + private GravitinoClient( + String uri, + String metalakeName, + AuthDataProvider authDataProvider, + Map headers) { + super(uri, authDataProvider, headers); this.metalake = loadMetalake(NameIdentifier.of(metalakeName)); } @@ -138,7 +143,7 @@ public GravitinoClient build() { metalakeName != null && !metalakeName.isEmpty(), "The argument 'metalakeName' must be a valid name"); - return new GravitinoClient(uri, metalakeName, authDataProvider); + return new GravitinoClient(uri, metalakeName, authDataProvider, headers); } } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java index 75350a7b954..87a4410a4a3 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java @@ -9,10 +9,12 @@ import com.datastrato.gravitino.dto.responses.MetalakeResponse; import com.datastrato.gravitino.dto.responses.VersionResponse; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.google.common.collect.ImmutableMap; import java.io.Closeable; import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,12 +39,15 @@ public abstract class GravitinoClientBase implements Closeable { * * @param uri The base URI for the Gravitino API. * @param authDataProvider The provider of the data which is used for authentication. + * @param headers The base header of the Gravitino API. */ - protected GravitinoClientBase(String uri, AuthDataProvider authDataProvider) { + protected GravitinoClientBase( + String uri, AuthDataProvider authDataProvider, Map headers) { this.restClient = HTTPClient.builder(Collections.emptyMap()) .uri(uri) .withAuthDataProvider(authDataProvider) + .withHeaders(headers) .build(); } @@ -103,6 +108,8 @@ public abstract static class Builder { protected String uri; /** The authentication provider. */ protected AuthDataProvider authDataProvider; + /** The request base header for the Gravitino API. */ + protected Map headers = ImmutableMap.of(); /** * The constructor for the Builder class. @@ -154,6 +161,19 @@ public Builder withKerberosAuth(KerberosTokenProvider dataProvider) { return this; } + /** + * Set base header for Gravitino Client. + * + * @param headers the base header. + * @return This Builder instance for method chaining. + */ + public Builder withHeaders(Map headers) { + if (headers != null) { + this.headers = ImmutableMap.copyOf(headers); + } + return this; + } + /** * Builds a new instance. Subclasses should overwrite this method. * diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java new file mode 100644 index 00000000000..965632e64fa --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestGravitinoClientBuilder { + @Test + public void testGravitinoClientHeaders() { + Map headers = ImmutableMap.of("k1", "v1"); + try (MockGravitinoClient client = + MockGravitinoClient.builder("http://127.0.0.1").withHeaders(headers).build()) { + Assertions.assertEquals(headers, client.getHeaders()); + } + + try (MockGravitinoClient client1 = MockGravitinoClient.builder("http://127.0.0.1").build()) { + Assertions.assertEquals(ImmutableMap.of(), client1.getHeaders()); + } + + try (MockGravitinoClient client1 = + MockGravitinoClient.builder("http://127.0.0.1").withHeaders(null).build()) { + Assertions.assertEquals(ImmutableMap.of(), client1.getHeaders()); + } + } +} + +class MockGravitinoClient extends GravitinoClientBase { + + private Map headers; + + /** + * Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. + * + * @param uri The base URI for the Gravitino API. + * @param authDataProvider The provider of the data which is used for authentication. + * @param headers The base header of the Gravitino API. + */ + private MockGravitinoClient( + String uri, AuthDataProvider authDataProvider, Map headers) { + super(uri, authDataProvider, headers); + this.headers = headers; + } + + Map getHeaders() { + return headers; + } + + /** + * Creates a new builder for constructing a GravitinoClient. + * + * @param uri The base URI for the Gravitino API. + * @return A new instance of the Builder class for constructing a GravitinoClient. + */ + static MockGravitinoClientBuilder builder(String uri) { + return new MockGravitinoClientBuilder(uri); + } + + static class MockGravitinoClientBuilder extends GravitinoClientBase.Builder { + + /** + * The constructor for the Builder class. + * + * @param uri The base URI for the Gravitino API. + */ + protected MockGravitinoClientBuilder(String uri) { + super(uri); + } + + @Override + public MockGravitinoClient build() { + return new MockGravitinoClient(uri, authDataProvider, headers); + } + } +} From 4189e80e1ab86e70e2ca63716569ffec6d350baa Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Thu, 18 Apr 2024 10:58:58 +0800 Subject: [PATCH 053/106] [HOTFIX] Fix(ci): Disable Python-related pipeline temporarily (#3003) ### What changes were proposed in this pull request? Disable the Python-related pipeline temporarily. ### Why are the changes needed? Python pipeline issues cannot be resolved quickly. To avoid hindering development, we will temporarily disable Python-related pipelines. ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- .github/workflows/python-integration-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 2a3ebc6ed25..95e18493fa6 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -64,11 +64,11 @@ jobs: - name: Python Client Integration Test id: integrationTest run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - for pythonVersion in "3.8" "3.9" "3.10" "3.11" - do - ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest - done + #./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + #for pythonVersion in "3.8" "3.9" "3.10" "3.11" + #do + # ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest + #done - name: Upload integrate tests reports uses: actions/upload-artifact@v3 From ec3b6c4f6af3a9b38ada37de2cb0efb053f8ef0d Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:49:00 +0800 Subject: [PATCH 054/106] [#2237] feat(core): Add the support of PermissionManager (#2958) ### What changes were proposed in this pull request? Add the support for permission manager. ### Why are the changes needed? Fix: #2237 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add a new UT. --------- Co-authored-by: Heng Qin --- .../com/datastrato/gravitino/Configs.java | 7 + .../datastrato/gravitino/GravitinoEnv.java | 13 +- .../authorization/AccessControlManager.java | 78 ++++- .../gravitino/authorization/AdminManager.java | 39 +-- .../authorization/AuthorizationUtils.java | 40 ++- .../authorization/PermissionManager.java | 277 ++++++++++++++++++ .../gravitino/authorization/RoleManager.java | 137 +++++---- .../authorization/UserGroupManager.java | 153 ++++------ .../gravitino/meta/GroupEntity.java | 57 +++- .../datastrato/gravitino/meta/UserEntity.java | 57 +++- .../gravitino/proto/GroupEntitySerDe.java | 14 +- .../gravitino/proto/UserEntitySerDe.java | 14 +- .../TestAccessControlManager.java | 14 +- ...estAccessControlManagerForPermissions.java | 268 +++++++++++++++++ .../datastrato/gravitino/meta/TestEntity.java | 8 +- .../gravitino/proto/TestEntityProtoSerDe.java | 8 +- .../gravitino/storage/TestEntityStorage.java | 4 +- meta/src/main/proto/gravitino_meta.proto | 10 +- .../server/web/rest/TestGroupOperations.java | 2 +- .../web/rest/TestMetalakeAdminOperations.java | 2 +- .../server/web/rest/TestUserOperations.java | 2 +- 21 files changed, 974 insertions(+), 230 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java create mode 100644 core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 90373760870..4d361f43436 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -256,4 +256,11 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .toSequence() .create(); + + ConfigEntry ROLE_CACHE_EVICTION_INTERVAL_MS = + new ConfigBuilder("gravitino.authorization.roleCacheEvictionIntervalMs") + .doc("The interval in milliseconds to evict the role cache") + .version(ConfigConstants.VERSION_0_5_0) + .longConf() + .createWithDefault(60 * 60 * 1000L); } diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 57c3ea18bc4..cbc11ca53e0 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -101,7 +101,7 @@ public void setLockManager(LockManager lockManager) { /** * This method is used for testing purposes only to set the access manager for test in package - * `com.datastrato.gravitino.server.web.rest`. + * `com.datastrato.gravitino.server.web.rest` and `com.datastrato.gravitino.authorization`. * * @param accessControlManager The access control manager to be set. */ @@ -110,6 +110,17 @@ public void setAccessControlManager(AccessControlManager accessControlManager) { this.accessControlManager = accessControlManager; } + /** + * This method is used for testing purposes only to set the entity store for test in package + * `com.datastrato.gravitino.authorization`. + * + * @param entityStore The entity store to be set. + */ + @VisibleForTesting + public void setEntityStore(EntityStore entityStore) { + this.entityStore = entityStore; + } + /** * Initialize the Gravitino environment. * diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index fcfc911d883..131153f4512 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -14,6 +14,7 @@ import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.Executable; +import com.google.common.annotations.VisibleForTesting; import java.util.List; import java.util.Map; @@ -30,13 +31,15 @@ public class AccessControlManager { private final UserGroupManager userGroupManager; private final AdminManager adminManager; private final RoleManager roleManager; + private final PermissionManager permissionManager; private final Object adminOperationLock = new Object(); private final Object nonAdminOperationLock = new Object(); public AccessControlManager(EntityStore store, IdGenerator idGenerator, Config config) { - this.userGroupManager = new UserGroupManager(store, idGenerator); this.adminManager = new AdminManager(store, idGenerator, config); - this.roleManager = new RoleManager(store, idGenerator); + this.roleManager = new RoleManager(store, idGenerator, config); + this.userGroupManager = new UserGroupManager(store, idGenerator, roleManager); + this.permissionManager = new PermissionManager(store, roleManager); } /** @@ -106,7 +109,7 @@ public boolean removeGroup(String metalake, String group) { * Gets a Group. * * @param metalake The Metalake of the Group. - * @param group THe name of the Group. + * @param group The name of the Group. * @return The getting Group instance. * @throws NoSuchGroupException If the Group with the given identifier does not exist. * @throws RuntimeException If getting the Group encounters storage issues. @@ -115,6 +118,70 @@ public Group getGroup(String metalake, String group) throws NoSuchGroupException return doWithNonAdminLock(() -> userGroupManager.getGroup(metalake, group)); } + /** + * Grant a role to a user. + * + * @param metalake The metalake of the User. + * @param user The name of the User. + * @return true` if the User was successfully granted, `false` otherwise. + * @throws NoSuchUserException If the User with the given identifier does not exist. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the + * User. + * @throws RuntimeException If granting a role to a user encounters storage issues. + */ + public boolean grantRoleToUser(String metalake, String role, String user) { + return doWithNonAdminLock(() -> permissionManager.grantRoleToUser(metalake, role, user)); + } + + /** + * Grant a role to a group. + * + * @param metalake The metalake of the Group. + * @param group THe name of the Group. + * @return true` if the Group was successfully granted, `false` otherwise. + * @throws NoSuchGroupException If the Group with the given identifier does not exist. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the + * Group. + * @throws RuntimeException If granting a role to a group encounters storage issues. + */ + public boolean grantRoleToGroup(String metalake, String role, String group) { + return doWithNonAdminLock(() -> permissionManager.grantRoleToGroup(metalake, role, group)); + } + + /** + * Revoke a role from a group. + * + * @param metalake The metalake of the Group. + * @param group The name of the Group. + * @return true` if the Group was successfully revoked, `false` otherwise. + * @throws NoSuchGroupException If the Group with the given identifier does not exist. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the + * Group. + * @throws RuntimeException If revoking a role from a group encounters storage issues. + */ + public boolean revokeRoleFromGroup(String metalake, String role, String group) { + return doWithNonAdminLock(() -> permissionManager.revokeRoleFromGroup(metalake, role, group)); + } + + /** + * Revoke a role from a user. + * + * @param metalake The metalake of the User. + * @param user The name of the User. + * @return true` if the User was successfully revoked, `false` otherwise. + * @throws NoSuchUserException If the User with the given identifier does not exist. + * @throws NoSuchRoleException If the Role with the given identifier does not exist. + * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the + * User. + * @throws RuntimeException If revoking a role from a user encounters storage issues. + */ + public boolean revokeRoleFromUser(String metalake, String role, String user) { + return doWithNonAdminLock(() -> permissionManager.revokeRoleFromUser(metalake, role, user)); + } + /** * Adds a new metalake admin. * @@ -206,6 +273,11 @@ public boolean dropRole(String metalake, String role) { return doWithNonAdminLock(() -> roleManager.dropRole(metalake, role)); } + @VisibleForTesting + RoleManager getRoleManager() { + return roleManager; + } + private R doWithNonAdminLock(Executable executable) throws E { synchronized (nonAdminOperationLock) { return executable.execute(); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java index edbe18cc2c0..74ee9054027 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java @@ -38,21 +38,13 @@ class AdminManager { private final IdGenerator idGenerator; private final List serviceAdmins; - public AdminManager(EntityStore store, IdGenerator idGenerator, Config config) { + AdminManager(EntityStore store, IdGenerator idGenerator, Config config) { this.store = store; this.idGenerator = idGenerator; this.serviceAdmins = config.get(Configs.SERVICE_ADMINS); } - /** - * Adds a new metalake admin. - * - * @param user The name of the User. - * @return The added User instance. - * @throws UserAlreadyExistsException If a User with the same identifier already exists. - * @throws RuntimeException If adding the User encounters storage issues. - */ - public User addMetalakeAdmin(String user) { + User addMetalakeAdmin(String user) { UserEntity userEntity = UserEntity.builder() @@ -63,7 +55,7 @@ public User addMetalakeAdmin(String user) { Entity.SYSTEM_METALAKE_RESERVED_NAME, Entity.AUTHORIZATION_CATALOG_NAME, Entity.ADMIN_SCHEMA_NAME)) - .withRoles(Lists.newArrayList()) + .withRoleNames(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -82,14 +74,7 @@ public User addMetalakeAdmin(String user) { } } - /** - * Removes a metalake admin. - * - * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` otherwise. - * @throws RuntimeException If removing the User encounters storage issues. - */ - public boolean removeMetalakeAdmin(String user) { + boolean removeMetalakeAdmin(String user) { try { return store.delete(ofMetalakeAdmin(user), Entity.EntityType.USER); } catch (IOException ioe) { @@ -99,23 +84,11 @@ public boolean removeMetalakeAdmin(String user) { } } - /** - * Judges whether the user is the service admin. - * - * @param user the name of the user - * @return true, if the user is service admin, otherwise false. - */ - public boolean isServiceAdmin(String user) { + boolean isServiceAdmin(String user) { return serviceAdmins.contains(user); } - /** - * Judges whether the user is the metalake admin. - * - * @param user the name of the user - * @return true, if the user is metalake admin, otherwise false. - */ - public boolean isMetalakeAdmin(String user) { + boolean isMetalakeAdmin(String user) { try { return store.exists(ofMetalakeAdmin(user), Entity.EntityType.USER); } catch (IOException ioe) { diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java index e5fa100b252..d6ded111e3a 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java @@ -6,8 +6,12 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.SchemaEntity; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,14 +19,18 @@ /* The utilization class of authorization module*/ class AuthorizationUtils { + static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; + static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in th metalake %s"; + static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; private static final Logger LOG = LoggerFactory.getLogger(AuthorizationUtils.class); private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private AuthorizationUtils() {} - static void checkMetalakeExists(EntityStore store, String metalake) - throws NoSuchMetalakeException { + static void checkMetalakeExists(String metalake) throws NoSuchMetalakeException { try { + EntityStore store = GravitinoEnv.getInstance().entityStore(); + NameIdentifier metalakeIdent = NameIdentifier.ofMetalake(metalake); if (!store.exists(metalakeIdent, Entity.EntityType.METALAKE)) { LOG.warn("Metalake {} does not exist", metalakeIdent); @@ -33,4 +41,32 @@ static void checkMetalakeExists(EntityStore store, String metalake) throw new RuntimeException(e); } } + + public static NameIdentifier ofRole(String metalake, String role) { + return NameIdentifier.of( + metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME, role); + } + + public static NameIdentifier ofGroup(String metalake, String group) { + return NameIdentifier.of( + metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME, group); + } + + public static NameIdentifier ofUser(String metalake, String user) { + return NameIdentifier.of( + metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME, user); + } + + public static Namespace ofRoleNamespace(String metalake) { + return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); + } + + public static Namespace ofGroupNamespace(String metalake) { + return Namespace.of( + metalake, CatalogEntity.SYSTEM_CATALOG_RESERVED_NAME, SchemaEntity.GROUP_SCHEMA_NAME); + } + + public static Namespace ofUserNamespace(String metalake) { + return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java new file mode 100644 index 00000000000..4453e68d255 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java @@ -0,0 +1,277 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import static com.datastrato.gravitino.authorization.AuthorizationUtils.GROUP_DOES_NOT_EXIST_MSG; +import static com.datastrato.gravitino.authorization.AuthorizationUtils.USER_DOES_NOT_EXIST_MSG; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NoSuchGroupException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.meta.UserEntity; +import com.datastrato.gravitino.utils.PrincipalUtils; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * PermissionManager is used for managing the logic the granting and revoking roles. Role is used + * for manging permissions. GrantManager will filter the invalid roles, too. + */ +class PermissionManager { + private static final Logger LOG = LoggerFactory.getLogger(PermissionManager.class); + + private final EntityStore store; + private final RoleManager roleManager; + + PermissionManager(EntityStore store, RoleManager roleManager) { + this.store = store; + this.roleManager = roleManager; + } + + boolean grantRoleToUser(String metalake, String role, String user) { + try { + RoleEntity roleEntity = roleManager.loadRole(metalake, role); + + store.update( + AuthorizationUtils.ofUser(metalake, user), + UserEntity.class, + Entity.EntityType.USER, + userEntity -> { + List roleEntities = + roleManager.getValidRoles(metalake, userEntity.roleNames(), userEntity.roleIds()); + + List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + + if (roleNames.contains(roleEntity.name())) { + throw new RoleAlreadyExistsException( + "Role %s already exists in the user %s of the metalake %s", role, user, metalake); + } + + roleNames.add(roleEntity.name()); + roleIds.add(roleEntity.id()); + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(userEntity.auditInfo().creator()) + .withCreateTime(userEntity.auditInfo().createTime()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build(); + + return UserEntity.builder() + .withNamespace(userEntity.namespace()) + .withId(userEntity.id()) + .withName(userEntity.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(auditInfo) + .build(); + }); + return true; + } catch (NoSuchEntityException nse) { + LOG.warn("Failed to grant, user {} does not exist in the metalake {}", user, metalake, nse); + throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); + } catch (IOException ioe) { + LOG.error( + "Failed to grant role {} to user {} in the metalake {} due to storage issues", + role, + user, + metalake, + ioe); + throw new RuntimeException(ioe); + } + } + + boolean grantRoleToGroup(String metalake, String role, String group) { + try { + RoleEntity roleEntity = roleManager.loadRole(metalake, role); + + store.update( + AuthorizationUtils.ofGroup(metalake, group), + GroupEntity.class, + Entity.EntityType.GROUP, + groupEntity -> { + List roleEntities = + roleManager.getValidRoles(metalake, groupEntity.roleNames(), groupEntity.roleIds()); + List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + + if (roleNames.contains(roleEntity.name())) { + throw new RoleAlreadyExistsException( + "Role %s already exists in the group %s of the metalake %s", + role, group, metalake); + } + + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(groupEntity.auditInfo().creator()) + .withCreateTime(groupEntity.auditInfo().createTime()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build(); + + roleNames.add(roleEntity.name()); + roleIds.add(roleEntity.id()); + return GroupEntity.builder() + .withId(groupEntity.id()) + .withNamespace(groupEntity.namespace()) + .withName(groupEntity.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(auditInfo) + .build(); + }); + return true; + } catch (NoSuchEntityException nse) { + LOG.warn("Failed to grant, group {} does not exist in the metalake {}", group, metalake, nse); + throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); + } catch (IOException ioe) { + LOG.error( + "Failed to grant role {} to group {} in the metalake {} due to storage issues", + role, + group, + metalake, + ioe); + throw new RuntimeException(ioe); + } + } + + boolean revokeRoleFromGroup(String metalake, String role, String group) { + try { + RoleEntity roleEntity = roleManager.loadRole(metalake, role); + + AtomicBoolean removed = new AtomicBoolean(true); + + store.update( + AuthorizationUtils.ofGroup(metalake, group), + GroupEntity.class, + Entity.EntityType.GROUP, + groupEntity -> { + List roleEntities = + roleManager.getValidRoles(metalake, groupEntity.roleNames(), groupEntity.roleIds()); + List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + roleNames.remove(roleEntity.name()); + removed.set(roleIds.remove(roleEntity.id())); + + if (!removed.get()) { + LOG.warn( + "Failed to revoke, role {} does not exist in the group {} of metalake {}", + role, + group, + metalake); + } + + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(groupEntity.auditInfo().creator()) + .withCreateTime(groupEntity.auditInfo().createTime()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build(); + + return GroupEntity.builder() + .withNamespace(groupEntity.namespace()) + .withId(groupEntity.id()) + .withName(groupEntity.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(auditInfo) + .build(); + }); + + return removed.get(); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to revoke, group {} does not exist in the metalake {}", group, metalake, nse); + throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); + } catch (IOException ioe) { + LOG.error( + "Failed to revoke role {} from group {} in the metalake {} due to storage issues", + role, + group, + metalake, + ioe); + throw new RuntimeException(ioe); + } + } + + boolean revokeRoleFromUser(String metalake, String role, String user) { + try { + RoleEntity roleEntity = roleManager.loadRole(metalake, role); + AtomicBoolean removed = new AtomicBoolean(true); + + store.update( + AuthorizationUtils.ofUser(metalake, user), + UserEntity.class, + Entity.EntityType.USER, + userEntity -> { + List roleEntities = + roleManager.getValidRoles(metalake, userEntity.roleNames(), userEntity.roleIds()); + + List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + + roleNames.remove(roleEntity.name()); + removed.set(roleIds.remove(roleEntity.id())); + if (!removed.get()) { + LOG.warn( + "Failed to revoke, role {} doesn't exist in the user {} of metalake {}", + role, + user, + metalake); + } + + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(userEntity.auditInfo().creator()) + .withCreateTime(userEntity.auditInfo().createTime()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build(); + return UserEntity.builder() + .withId(userEntity.id()) + .withNamespace(userEntity.namespace()) + .withName(userEntity.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(auditInfo) + .build(); + }); + return removed.get(); + } catch (NoSuchEntityException nse) { + LOG.warn("Failed to revoke, user {} does not exist in the metalake {}", user, metalake, nse); + throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); + } catch (IOException ioe) { + LOG.error( + "Failed to revoke role {} from user {} in the metalake {} due to storage issues", + role, + user, + metalake, + ioe); + throw new RuntimeException(ioe); + } + } + + private List toRoleNames(List roleEntities) { + return roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList()); + } + + private List toRoleIds(List roleEntities) { + return roleEntities.stream().map(RoleEntity::id).collect(Collectors.toList()); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index 35639e47266..ce396599967 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -5,11 +5,12 @@ package com.datastrato.gravitino.authorization; +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; @@ -17,10 +18,18 @@ import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.PrincipalUtils; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,35 +41,44 @@ class RoleManager { private static final Logger LOG = LoggerFactory.getLogger(RoleManager.class); - private static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; private final EntityStore store; private final IdGenerator idGenerator; + private final Cache cache; - public RoleManager(EntityStore store, IdGenerator idGenerator) { + RoleManager(EntityStore store, IdGenerator idGenerator, Config config) { this.store = store; this.idGenerator = idGenerator; + + long cacheEvictionIntervalInMs = config.get(Configs.ROLE_CACHE_EVICTION_INTERVAL_MS); + // One role entity is about 40 bytes using jol estimate, there are usually about 100w+ + // roles in the production environment, this won't bring too much memory cost, but it + // can improve the performance significantly. + this.cache = + Caffeine.newBuilder() + .expireAfterAccess(cacheEvictionIntervalInMs, TimeUnit.MILLISECONDS) + .removalListener( + (k, v, c) -> { + LOG.info("Remove role {} from the cache.", k); + }) + .scheduler( + Scheduler.forScheduledExecutorService( + new ScheduledThreadPoolExecutor( + 1, + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("role-cleaner-%d") + .build()))) + .build(); } - /** - * Creates a new Role. - * - * @param metalake The Metalake of the Role. - * @param role The name of the Role. - * @param properties The properties of the Role. - * @param securableObject The securable object of the Role. - * @param privileges The privileges of the Role. - * @return The created Role instance. - * @throws RoleAlreadyExistsException If a Role with the same identifier already exists. - * @throws RuntimeException If creating the Role encounters storage issues. - */ - public Role createRole( + RoleEntity createRole( String metalake, String role, Map properties, SecurableObject securableObject, List privileges) throws RoleAlreadyExistsException { - AuthorizationUtils.checkMetalakeExists(store, metalake); + AuthorizationUtils.checkMetalakeExists(metalake); RoleEntity roleEntity = RoleEntity.builder() .withId(idGenerator.nextId()) @@ -68,9 +86,7 @@ public Role createRole( .withProperties(properties) .withSecurableObject(securableObject) .withPrivileges(privileges) - .withNamespace( - Namespace.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -79,6 +95,7 @@ public Role createRole( .build(); try { store.put(roleEntity, false /* overwritten */); + cache.put(roleEntity.nameIdentifier(), roleEntity); return roleEntity; } catch (EntityAlreadyExistsException e) { LOG.warn("Role {} in the metalake {} already exists", role, metalake, e); @@ -91,40 +108,23 @@ public Role createRole( } } - /** - * Loads a Role. - * - * @param metalake The Metalake of the Role. - * @param role The name of the Role. - * @return The loading Role instance. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RuntimeException If loading the Role encounters storage issues. - */ - public Role loadRole(String metalake, String role) throws NoSuchRoleException { + RoleEntity loadRole(String metalake, String role) throws NoSuchRoleException { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.get(ofRole(metalake, role), Entity.EntityType.ROLE, RoleEntity.class); + AuthorizationUtils.checkMetalakeExists(metalake); + return getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); } catch (NoSuchEntityException e) { LOG.warn("Role {} does not exist in the metalake {}", role, metalake, e); - throw new NoSuchRoleException(ROLE_DOES_NOT_EXIST_MSG, role, metalake); - } catch (IOException ioe) { - LOG.error("Loading role {} failed due to storage issues", role, ioe); - throw new RuntimeException(ioe); + throw new NoSuchRoleException(AuthorizationUtils.ROLE_DOES_NOT_EXIST_MSG, role, metalake); } } - /** - * Drops a Role. - * - * @param metalake The Metalake of the Role. - * @param role The name of the Role. - * @return `true` if the Role was successfully dropped, `false` otherwise. - * @throws RuntimeException If dropping the User encounters storage issues. - */ - public boolean dropRole(String metalake, String role) { + boolean dropRole(String metalake, String role) { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.delete(ofRole(metalake, role), Entity.EntityType.ROLE); + AuthorizationUtils.checkMetalakeExists(metalake); + NameIdentifier ident = AuthorizationUtils.ofRole(metalake, role); + cache.invalidate(ident); + + return store.delete(ident, Entity.EntityType.ROLE); } catch (IOException ioe) { LOG.error( "Deleting role {} in the metalake {} failed due to storage issues", role, metalake, ioe); @@ -132,8 +132,45 @@ public boolean dropRole(String metalake, String role) { } } - private NameIdentifier ofRole(String metalake, String role) { - return NameIdentifier.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME, role); + private RoleEntity getRoleEntity(NameIdentifier identifier) { + return cache.get( + identifier, + id -> { + try { + return store.get(identifier, Entity.EntityType.ROLE, RoleEntity.class); + } catch (IOException ioe) { + LOG.error("Failed to get roles {} due to storage issues", identifier, ioe); + throw new RuntimeException(ioe); + } + }); + } + + @VisibleForTesting + Cache getCache() { + return cache; + } + + List getValidRoles(String metalake, List roleNames, List roleIds) { + List roleEntities = Lists.newArrayList(); + if (roleNames == null || roleNames.isEmpty()) { + return roleEntities; + } + + int index = 0; + for (String role : roleNames) { + try { + + RoleEntity roleEntity = getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); + + if (roleEntity.id().equals(roleIds.get(index))) { + roleEntities.add(roleEntity); + } + index++; + + } catch (NoSuchEntityException nse) { + // ignore this entity + } + } + return roleEntities; } } diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java index 411c640a84e..0baa6fe460b 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java @@ -7,8 +7,6 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.EntityStore; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; @@ -16,6 +14,7 @@ import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.PrincipalUtils; @@ -23,6 +22,8 @@ import java.io.IOException; import java.time.Instant; import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,39 +36,26 @@ class UserGroupManager { private static final Logger LOG = LoggerFactory.getLogger(UserGroupManager.class); - private static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; - - private static final String GROUP_DOES_NOT_EXIST_MSG = - "Group %s does not exist in th metalake %s"; private final EntityStore store; private final IdGenerator idGenerator; + private final RoleManager roleManager; - public UserGroupManager(EntityStore store, IdGenerator idGenerator) { + UserGroupManager(EntityStore store, IdGenerator idGenerator, RoleManager roleManager) { this.store = store; this.idGenerator = idGenerator; + this.roleManager = roleManager; } - /** - * Adds a new User. - * - * @param metalake The Metalake of the User. - * @param name The name of the User. - * @return The added User instance. - * @throws UserAlreadyExistsException If a User with the same identifier already exists. - * @throws RuntimeException If adding the User encounters storage issues. - */ - public User addUser(String metalake, String name) throws UserAlreadyExistsException { + User addUser(String metalake, String name) throws UserAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); + AuthorizationUtils.checkMetalakeExists(metalake); UserEntity userEntity = UserEntity.builder() .withId(idGenerator.nextId()) .withName(name) - .withNamespace( - Namespace.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME)) - .withRoles(Lists.newArrayList()) + .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) + .withRoleNames(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -87,19 +75,10 @@ public User addUser(String metalake, String name) throws UserAlreadyExistsExcept } } - /** - * Removes a User. - * - * @param metalake The Metalake of the User. - * @param user THe name of the User. - * @return `true` if the User was successfully removed, `false` otherwise. - * @throws RuntimeException If removing the User encounters storage issues. - */ - public boolean removeUser(String metalake, String user) { - + boolean removeUser(String metalake, String user) { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.delete(ofUser(metalake, user), Entity.EntityType.USER); + AuthorizationUtils.checkMetalakeExists(metalake); + return store.delete(AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER); } catch (IOException ioe) { LOG.error( "Removing user {} in the metalake {} failed due to storage issues", user, metalake, ioe); @@ -107,48 +86,41 @@ public boolean removeUser(String metalake, String user) { } } - /** - * Gets a User. - * - * @param metalake The Metalake of the User. - * @param user The name of the User. - * @return The getting User instance. - * @throws NoSuchUserException If the User with the given identifier does not exist. - * @throws RuntimeException If getting the User encounters storage issues. - */ - public User getUser(String metalake, String user) throws NoSuchUserException { + User getUser(String metalake, String user) throws NoSuchUserException { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.get(ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); + AuthorizationUtils.checkMetalakeExists(metalake); + UserEntity entity = + store.get( + AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); + + List roleEntities = + roleManager.getValidRoles(metalake, entity.roles(), entity.roleIds()); + + return UserEntity.builder() + .withId(entity.id()) + .withName(entity.name()) + .withAuditInfo(entity.auditInfo()) + .withNamespace(entity.namespace()) + .withRoleNames(roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList())) + .build(); } catch (NoSuchEntityException e) { LOG.warn("User {} does not exist in the metalake {}", user, metalake, e); - throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); + throw new NoSuchUserException(AuthorizationUtils.USER_DOES_NOT_EXIST_MSG, user, metalake); } catch (IOException ioe) { LOG.error("Getting user {} failed due to storage issues", user, ioe); throw new RuntimeException(ioe); } } - /** - * Adds a new Group. - * - * @param metalake The Metalake of the Group. - * @param group The name of the Group. - * @return The Added Group instance. - * @throws GroupAlreadyExistsException If a Group with the same identifier already exists. - * @throws RuntimeException If adding the Group encounters storage issues. - */ - public Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { + Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); + AuthorizationUtils.checkMetalakeExists(metalake); GroupEntity groupEntity = GroupEntity.builder() .withId(idGenerator.nextId()) .withName(group) - .withNamespace( - Namespace.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME)) - .withRoles(Collections.emptyList()) + .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) + .withRoleNames(Collections.emptyList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -168,18 +140,10 @@ public Group addGroup(String metalake, String group) throws GroupAlreadyExistsEx } } - /** - * Removes a Group. - * - * @param metalake The Metalake of the Group. - * @param group THe name of the Group. - * @return `true` if the Group was successfully removed, `false` otherwise. - * @throws RuntimeException If removing the Group encounters storage issues. - */ - public boolean removeGroup(String metalake, String group) { + boolean removeGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.delete(ofGroup(metalake, group), Entity.EntityType.GROUP); + AuthorizationUtils.checkMetalakeExists(metalake); + return store.delete(AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP); } catch (IOException ioe) { LOG.error( "Removing group {} in the metalake {} failed due to storage issues", @@ -190,35 +154,32 @@ public boolean removeGroup(String metalake, String group) { } } - /** - * Gets a Group. - * - * @param metalake The Metalake of the Group. - * @param group THe name of the Group. - * @return The getting Group instance. - * @throws NoSuchGroupException If the Group with the given identifier does not exist. - * @throws RuntimeException If getting the Group encounters storage issues. - */ - public Group getGroup(String metalake, String group) { + Group getGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(store, metalake); - return store.get(ofGroup(metalake, group), Entity.EntityType.GROUP, GroupEntity.class); + AuthorizationUtils.checkMetalakeExists(metalake); + + GroupEntity entity = + store.get( + AuthorizationUtils.ofGroup(metalake, group), + Entity.EntityType.GROUP, + GroupEntity.class); + + List roleEntities = + roleManager.getValidRoles(metalake, entity.roles(), entity.roleIds()); + + return GroupEntity.builder() + .withId(entity.id()) + .withName(entity.name()) + .withAuditInfo(entity.auditInfo()) + .withNamespace(entity.namespace()) + .withRoleNames(roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList())) + .build(); } catch (NoSuchEntityException e) { LOG.warn("Group {} does not exist in the metalake {}", group, metalake, e); - throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); + throw new NoSuchGroupException(AuthorizationUtils.GROUP_DOES_NOT_EXIST_MSG, group, metalake); } catch (IOException ioe) { LOG.error("Getting group {} failed due to storage issues", group, ioe); throw new RuntimeException(ioe); } } - - private NameIdentifier ofUser(String metalake, String user) { - return NameIdentifier.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME, user); - } - - private NameIdentifier ofGroup(String metalake, String group) { - return NameIdentifier.of( - metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME, group); - } } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java index e00b59b908b..f9c3b9cd881 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java @@ -24,8 +24,11 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { public static final Field NAME = Field.required("name", String.class, "The name of the group entity."); - public static final Field ROLES = - Field.optional("roles", List.class, "The roles of the group entity."); + public static final Field ROLE_NAMES = + Field.optional("role_names", List.class, "The role names of the group entity."); + + public static final Field ROLE_IDS = + Field.optional("role_ids", List.class, "The role names of the group entity."); public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the group entity."); @@ -33,7 +36,8 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { private Long id; private String name; private AuditInfo auditInfo; - private List roles; + private List roleNames; + private List roleIds; private Namespace namespace; private GroupEntity() {} @@ -49,7 +53,8 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLES, roles); + fields.put(ROLE_NAMES, roleNames); + fields.put(ROLE_IDS, roleIds); return Collections.unmodifiableMap(fields); } @@ -109,7 +114,25 @@ public AuditInfo auditInfo() { * @return The roles of the group entity. */ public List roles() { - return roles; + return roleNames; + } + + /** + * Returns the role names of the group entity. + * + * @return The role names of the group entity. + */ + public List roleNames() { + return roleNames; + } + + /** + * Returns the role ids of the group entity. + * + * @return The role ids of the group entity. + */ + public List roleIds() { + return roleIds; } @Override @@ -122,12 +145,13 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roles, that.roles); + && Objects.equals(roleNames, that.roleNames) + && Objects.equals(roleIds, that.roleIds); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roles); + return Objects.hash(id, name, auditInfo, roleNames, roleIds); } public static Builder builder() { @@ -175,13 +199,24 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the roles of the group entity. + * Sets the role names of the group entity. + * + * @param roles The role names of the group entity. + * @return The builder instance. + */ + public Builder withRoleNames(List roles) { + groupEntity.roleNames = roles; + return this; + } + + /** + * Sets the role ids of the group entity. * - * @param roles The roles of the group entity. + * @param roleIds The role ids of the group entity. * @return The builder instance. */ - public Builder withRoles(List roles) { - groupEntity.roles = roles; + public Builder withRoleIds(List roleIds) { + groupEntity.roleIds = roleIds; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java index 933ecb1eb5e..954ec9924e3 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java @@ -30,13 +30,17 @@ public class UserEntity implements User, Entity, Auditable, HasIdentifier { public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the user entity."); - public static final Field ROLES = - Field.optional("roles", List.class, "The roles of the user entity"); + public static final Field ROLE_NAMES = + Field.optional("role_names", List.class, "The role names of the user entity"); + + public static final Field ROLE_IDS = + Field.optional("role_ids", List.class, "The role ids of the user entity"); private Long id; private String name; private AuditInfo auditInfo; - private List roles; + private List roleNames; + private List roleIds; private Namespace namespace; private UserEntity() {} @@ -52,7 +56,8 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLES, roles); + fields.put(ROLE_NAMES, roleNames); + fields.put(ROLE_IDS, roleIds); return Collections.unmodifiableMap(fields); } @@ -113,7 +118,25 @@ public AuditInfo auditInfo() { */ @Override public List roles() { - return roles; + return roleNames; + } + + /** + * Returns the role names of the user entity. + * + * @return The role names of the user entity. + */ + public List roleNames() { + return roleNames; + } + + /** + * Returns the role ids of the user entity. + * + * @return The role ids of the user entity. + */ + public List roleIds() { + return roleIds; } @Override @@ -126,12 +149,13 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roles, that.roles); + && Objects.equals(roleNames, that.roleNames) + && Objects.equals(roleIds, that.roleIds); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roles); + return Objects.hash(id, name, auditInfo, roleNames, roleIds); } public static Builder builder() { @@ -179,13 +203,24 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the roles of the user entity. + * Sets the role names of the user entity. + * + * @param roles The role names of the user entity. + * @return The builder instance. + */ + public Builder withRoleNames(List roles) { + userEntity.roleNames = roles; + return this; + } + + /** + * Sets the role ids of the user entity. * - * @param roles The roles of the user entity. + * @param roleIds The role ids of the user entity. * @return The builder instance. */ - public Builder withRoles(List roles) { - userEntity.roles = roles; + public Builder withRoleIds(List roleIds) { + userEntity.roleIds = roleIds; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java index e1fa6ab50c2..fb5465802d4 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java @@ -19,7 +19,11 @@ public Group serialize(GroupEntity groupEntity) { .setAuditInfo(new AuditInfoSerDe().serialize(groupEntity.auditInfo())); if (isCollectionNotEmpty(groupEntity.roles())) { - builder.addAllRoles(groupEntity.roles()); + builder.addAllRoleNames(groupEntity.roles()); + } + + if (isCollectionNotEmpty(groupEntity.roleIds())) { + builder.addAllRoleIds(groupEntity.roleIds()); } return builder.build(); @@ -34,8 +38,12 @@ public GroupEntity deserialize(Group group, Namespace namespace) { .withNamespace(namespace) .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); - if (group.getRolesCount() > 0) { - builder.withRoles(group.getRolesList()); + if (group.getRoleNamesCount() > 0) { + builder.withRoleNames(group.getRoleNamesList()); + } + + if (group.getRoleIdsCount() > 0) { + builder.withRoleIds(group.getRoleIdsList()); } return builder.build(); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java index 47e7f2bb18b..c839b574e3a 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java @@ -19,7 +19,11 @@ public User serialize(UserEntity userEntity) { .setAuditInfo(new AuditInfoSerDe().serialize(userEntity.auditInfo())); if (isCollectionNotEmpty(userEntity.roles())) { - builder.addAllRoles(userEntity.roles()); + builder.addAllRoleNames(userEntity.roles()); + } + + if (isCollectionNotEmpty(userEntity.roleIds())) { + builder.addAllRoleIds(userEntity.roleIds()); } return builder.build(); @@ -34,8 +38,12 @@ public UserEntity deserialize(User user, Namespace namespace) { .withNamespace(namespace) .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); - if (user.getRolesCount() > 0) { - builder.withRoles(user.getRolesList()); + if (user.getRoleNamesCount() > 0) { + builder.withRoleNames(user.getRoleNamesList()); + } + + if (user.getRoleIdsCount() > 0) { + builder.withRoleIds(user.getRoleIdsList()); } return builder.build(); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java index c53ada3a35f..e65115c3c51 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; @@ -39,12 +40,12 @@ public class TestAccessControlManager { private static Config config; - private static String metalake = "metalake"; + private static String METALAKE = "metalake"; private static BaseMetalake metalakeEntity = BaseMetalake.builder() .withId(1L) - .withName(metalake) + .withName(METALAKE) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .withVersion(SchemaVersion.V_0_1) @@ -62,6 +63,8 @@ public static void setUp() throws Exception { entityStore.put(metalakeEntity, true); accessControlManager = new AccessControlManager(entityStore, new RandomIdGenerator(), config); + GravitinoEnv.getInstance().setEntityStore(entityStore); + GravitinoEnv.getInstance().setAccessControlManager(accessControlManager); } @AfterAll @@ -241,7 +244,14 @@ public void testLoadRole() { accessControlManager.createRole( "metalake", "loadRole", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + + Role cachedRole = accessControlManager.loadRole("metalake", "loadRole"); + accessControlManager.getRoleManager().getCache().invalidateAll(); Role role = accessControlManager.loadRole("metalake", "loadRole"); + + // Verify the cached roleEntity is correct + Assertions.assertEquals(role, cachedRole); + Assertions.assertEquals("loadRole", role.name()); testProperties(props, role.properties()); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java new file mode 100644 index 00000000000..00dd6127bc4 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -0,0 +1,268 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchGroupException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.meta.UserEntity; +import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.memory.TestMemoryEntityStore; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.time.Instant; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestAccessControlManagerForPermissions { + + private static AccessControlManager accessControlManager; + + private static EntityStore entityStore; + + private static Config config; + + private static String METALAKE = "metalake"; + private static String CATALOG = "catalog"; + + private static String USER = "user"; + + private static String GROUP = "group"; + + private static String ROLE = "role"; + + private static AuditInfo auditInfo = + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + + private static BaseMetalake metalakeEntity = + BaseMetalake.builder() + .withId(1L) + .withName(METALAKE) + .withAuditInfo(auditInfo) + .withVersion(SchemaVersion.V_0_1) + .build(); + + private static UserEntity userEntity = + UserEntity.builder() + .withNamespace( + Namespace.of(METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME)) + .withId(1L) + .withName(USER) + .withAuditInfo(auditInfo) + .build(); + + private static GroupEntity groupEntity = + GroupEntity.builder() + .withNamespace( + Namespace.of(METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME)) + .withId(1L) + .withName(GROUP) + .withAuditInfo(auditInfo) + .build(); + + private static RoleEntity roleEntity = + RoleEntity.builder() + .withNamespace( + Namespace.of(METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withId(1L) + .withName(ROLE) + .withProperties(Maps.newHashMap()) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.of(CATALOG)) + .withAuditInfo(auditInfo) + .build(); + + @BeforeAll + public static void setUp() throws Exception { + config = new Config(false) {}; + + entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); + entityStore.initialize(config); + entityStore.setSerDe(null); + + entityStore.put(metalakeEntity, true); + entityStore.put(userEntity, true); + entityStore.put(groupEntity, true); + entityStore.put(roleEntity, true); + + accessControlManager = new AccessControlManager(entityStore, new RandomIdGenerator(), config); + + GravitinoEnv.getInstance().setEntityStore(entityStore); + GravitinoEnv.getInstance().setAccessControlManager(accessControlManager); + } + + @AfterAll + public static void tearDown() throws IOException { + if (entityStore != null) { + entityStore.close(); + entityStore = null; + } + } + + @Test + public void testAddRoleToUser() { + String notExist = "not-exist"; + + User user = accessControlManager.getUser(METALAKE, USER); + Assertions.assertTrue(user.roles().isEmpty()); + + Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); + user = accessControlManager.getUser(METALAKE, USER); + Assertions.assertEquals(1, user.roles().size()); + Assertions.assertEquals(ROLE, user.roles().get(0)); + + // Throw RoleAlreadyExistsException + Assertions.assertThrows( + RoleAlreadyExistsException.class, + () -> accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); + + // Throw NoSuchMetalakeException + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> accessControlManager.grantRoleToUser(notExist, ROLE, USER)); + + // Throw NoSuchRoleException + Assertions.assertThrows( + NoSuchRoleException.class, + () -> accessControlManager.grantRoleToUser(METALAKE, notExist, USER)); + + // Throw NoSuchUserException + Assertions.assertThrows( + NoSuchUserException.class, + () -> accessControlManager.grantRoleToUser(METALAKE, ROLE, notExist)); + + // Clear Resource + Assertions.assertTrue(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + } + + @Test + public void testRemoveRoleFromUser() { + String notExist = "not-exist"; + + Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); + Assertions.assertTrue(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + + // Throw NoSuchMetalakeException + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> accessControlManager.revokeRoleFromUser(notExist, ROLE, USER)); + + // Throw NoSuchRoleException + Assertions.assertThrows( + NoSuchRoleException.class, + () -> accessControlManager.revokeRoleFromUser(METALAKE, notExist, USER)); + + // Remove role which doesn't exist. + Assertions.assertFalse(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + + // Throw NoSuchUserException + Assertions.assertThrows( + NoSuchUserException.class, + () -> accessControlManager.revokeRoleFromUser(METALAKE, ROLE, notExist)); + } + + @Test + public void testAddRoleToGroup() { + String notExist = "not-exist"; + + Group group = accessControlManager.getGroup(METALAKE, GROUP); + Assertions.assertTrue(group.roles().isEmpty()); + + Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); + + group = accessControlManager.getGroup(METALAKE, GROUP); + Assertions.assertEquals(1, group.roles().size()); + Assertions.assertEquals(ROLE, group.roles().get(0)); + + // Throw RoleAlreadyExistsException + Assertions.assertThrows( + RoleAlreadyExistsException.class, + () -> accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); + + // Throw NoSuchMetalakeException + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> accessControlManager.grantRoleToGroup(notExist, ROLE, GROUP)); + + // Throw NoSuchRoleException + Assertions.assertThrows( + NoSuchRoleException.class, + () -> accessControlManager.grantRoleToGroup(METALAKE, notExist, GROUP)); + + // Throw NoSuchGroupException + Assertions.assertThrows( + NoSuchGroupException.class, + () -> accessControlManager.grantRoleToGroup(METALAKE, ROLE, notExist)); + + // Clear Resource + Assertions.assertTrue(accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, GROUP)); + } + + @Test + public void testRemoveRoleFormGroup() { + String notExist = "not-exist"; + + Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); + Assertions.assertTrue(accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, GROUP)); + + // Throw NoSuchMetalakeException + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> accessControlManager.revokeRoleFromGroup(notExist, ROLE, GROUP)); + + // Throw NoSuchRoleException + Assertions.assertThrows( + NoSuchRoleException.class, + () -> accessControlManager.revokeRoleFromGroup(METALAKE, notExist, USER)); + + // Remove not exist role + Assertions.assertFalse(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + + // Throw NoSuchGroupException + Assertions.assertThrows( + NoSuchGroupException.class, + () -> accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, notExist)); + } + + @Test + public void testDropRole() throws IOException { + String anotherRole = "anotherRole"; + + RoleEntity roleEntity = + RoleEntity.builder() + .withNamespace( + Namespace.of( + METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withId(1L) + .withName(anotherRole) + .withProperties(Maps.newHashMap()) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(CATALOG)) + .withAuditInfo(auditInfo) + .build(); + + entityStore.put(roleEntity, true); + Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, anotherRole, USER)); + Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, anotherRole, GROUP)); + accessControlManager.dropRole(METALAKE, anotherRole); + Group group = accessControlManager.getGroup(METALAKE, GROUP); + Assertions.assertTrue(group.roles().isEmpty()); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java index b852d340101..7108f33ef8d 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java @@ -223,14 +223,14 @@ public void testUser() { .withId(userId) .withName(userName) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList("role")) + .withRoleNames(Lists.newArrayList("role")) .build(); Map fields = testUserEntity.fields(); Assertions.assertEquals(userId, fields.get(UserEntity.ID)); Assertions.assertEquals(userName, fields.get(UserEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(UserEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(UserEntity.ROLES)); + Assertions.assertEquals(Lists.newArrayList("role"), fields.get(UserEntity.ROLE_NAMES)); UserEntity testUserEntityWithoutFields = UserEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); @@ -245,13 +245,13 @@ public void testGroup() { .withId(groupId) .withName(groupName) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList("role")) + .withRoleNames(Lists.newArrayList("role")) .build(); Map fields = group.fields(); Assertions.assertEquals(groupId, fields.get(GroupEntity.ID)); Assertions.assertEquals(groupName, fields.get(GroupEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(GroupEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(GroupEntity.ROLES)); + Assertions.assertEquals(Lists.newArrayList("role"), fields.get(GroupEntity.ROLE_NAMES)); GroupEntity groupWithoutFields = GroupEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index d0b91b79152..a314a3f1c73 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -321,7 +321,8 @@ public void testEntitiesSerDe() throws IOException { .withName(userName) .withNamespace(userNamespace) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList("role")) + .withRoleNames(Lists.newArrayList("role")) + .withRoleIds(Lists.newArrayList(1L)) .build(); byte[] userBytes = protoEntitySerDe.serialize(userEntity); UserEntity userEntityFromBytes = @@ -339,6 +340,7 @@ public void testEntitiesSerDe() throws IOException { userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); Assertions.assertNull(userEntityWithoutFields.roles()); + Assertions.assertNull(userEntityWithoutFields.roleIds()); // Test GroupEntity Namespace groupNamespace = @@ -352,7 +354,8 @@ public void testEntitiesSerDe() throws IOException { .withName(groupName) .withNamespace(groupNamespace) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList("role")) + .withRoleNames(Lists.newArrayList("role")) + .withRoleIds(Lists.newArrayList(1L)) .build(); byte[] groupBytes = protoEntitySerDe.serialize(group); GroupEntity groupFromBytes = @@ -370,6 +373,7 @@ public void testEntitiesSerDe() throws IOException { groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); Assertions.assertEquals(groupWithoutFields, groupFromBytes); Assertions.assertNull(groupWithoutFields.roles()); + Assertions.assertNull(groupWithoutFields.roleIds()); // Test RoleEntity Namespace roleNamespace = diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index e79ccf9cacf..3bd7cf8a70d 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -1160,7 +1160,7 @@ private static UserEntity createUser(String metalake, String name, AuditInfo aud Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME)) .withName(name) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList()) + .withRoleNames(Lists.newArrayList()) .build(); } @@ -1171,7 +1171,7 @@ private static GroupEntity createGroup(String metalake, String name, AuditInfo a Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME)) .withName(name) .withAuditInfo(auditInfo) - .withRoles(Lists.newArrayList()) + .withRoleNames(Lists.newArrayList()) .build(); } diff --git a/meta/src/main/proto/gravitino_meta.proto b/meta/src/main/proto/gravitino_meta.proto index 94cccc9185f..e160c8eaf02 100644 --- a/meta/src/main/proto/gravitino_meta.proto +++ b/meta/src/main/proto/gravitino_meta.proto @@ -105,15 +105,17 @@ message Topic { message User { uint64 id = 1; string name = 2; - repeated string roles = 3; - AuditInfo audit_info = 4; + repeated string role_names = 3; + repeated uint64 role_ids = 4; + AuditInfo audit_info = 5; } message Group { uint64 id = 1; string name = 2; - repeated string roles = 3; - AuditInfo audit_info = 4; + repeated string role_names = 3; + repeated uint64 role_ids = 4; + AuditInfo audit_info = 5; } message Role { diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestGroupOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestGroupOperations.java index b345427f2ec..ecba282167d 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestGroupOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestGroupOperations.java @@ -230,7 +230,7 @@ private Group buildGroup(String group) { return GroupEntity.builder() .withId(1L) .withName(group) - .withRoles(Collections.emptyList()) + .withRoleNames(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeAdminOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeAdminOperations.java index 0c83c56991c..8437626dc19 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeAdminOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestMetalakeAdminOperations.java @@ -147,7 +147,7 @@ private User buildUser(String user) { return UserEntity.builder() .withId(1L) .withName(user) - .withRoles(Collections.emptyList()) + .withRoleNames(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestUserOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestUserOperations.java index 75c2f142d80..71e00d6f33a 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestUserOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestUserOperations.java @@ -231,7 +231,7 @@ private User buildUser(String user) { return UserEntity.builder() .withId(1L) .withName(user) - .withRoles(Collections.emptyList()) + .withRoleNames(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); From aeb682afe21cc007fc3c77571013add3b5420751 Mon Sep 17 00:00:00 2001 From: Shaofeng Shi Date: Thu, 18 Apr 2024 14:13:07 +0800 Subject: [PATCH 055/106] [#2799] fix(license): add MIT license for Python module (#2960) ### What changes were proposed in this pull request? Correct the wrong header and add the license dependency to project's, as the issue mentioned; ### Why are the changes needed? No big change, just correct the file header and add the MIT license. Fix: #2799 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? No code change --------- Co-authored-by: Jerry Shao --- LICENSE | 7 ++++++ LICENSE.bin | 1 + build.gradle.kts | 4 +++- .../gravitino/utils/exceptions.py | 23 ++++++++++++++++-- .../gravitino/utils/http_client.py | 24 +++++++++++++++++-- licenses/kylinpy.txt | 21 ++++++++++++++++ 6 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 licenses/kylinpy.txt diff --git a/LICENSE b/LICENSE index eab8b763316..f36fefefb08 100644 --- a/LICENSE +++ b/LICENSE @@ -277,3 +277,10 @@ Apache Arrow ./dev/ci/util_free_space.sh + +This product bundles a third-party component under the + MIT License. + + Kyligence/kylinpy + ./clients/client-python/gravitino/utils/exceptions.py + ./clients/client-python/gravitino/utils/http_client.py \ No newline at end of file diff --git a/LICENSE.bin b/LICENSE.bin index fc972c8ed07..60db5658127 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -392,6 +392,7 @@ Janino Common Compiler Protocol Buffers Treelayout + Kyligence/kylinpy This product bundles various third-party components also under the Common Development and Distribution License 1.0 diff --git a/build.gradle.kts b/build.gradle.kts index c70b4fc7cf3..b8a79d08acc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -540,7 +540,9 @@ tasks.rat { "**/NOTICE.*", "ROADMAP.md", "clients/client-python/.pytest_cache/*", - "clients/client-python/gravitino.egg-info/*" + "clients/client-python/gravitino.egg-info/*", + "clients/client-python/gravitino/utils/exceptions.py", + "clients/client-python/gravitino/utils/http_client.py" ) // Add .gitignore excludes to the Apache Rat exclusion list. diff --git a/clients/client-python/gravitino/utils/exceptions.py b/clients/client-python/gravitino/utils/exceptions.py index 28afc373452..09314a5e0bb 100644 --- a/clients/client-python/gravitino/utils/exceptions.py +++ b/clients/client-python/gravitino/utils/exceptions.py @@ -1,6 +1,25 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +MIT License + +Copyright (c) 2016 Dhamu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. """ import json diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index 998f82b8e35..b5219d52e92 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -1,7 +1,27 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +MIT License + +Copyright (c) 2016 Dhamu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. """ + import logging from urllib.request import Request, build_opener from urllib.parse import urlencode diff --git a/licenses/kylinpy.txt b/licenses/kylinpy.txt new file mode 100644 index 00000000000..580127c7327 --- /dev/null +++ b/licenses/kylinpy.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Dhamu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From 4765fc42528d0e336ca77c25e7611b5f9c2d4cd8 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Thu, 18 Apr 2024 14:13:53 +0800 Subject: [PATCH 056/106] [#2866] improve(java-clients) Graviton java client support check version and backward compatibility (#2867) ### What changes were proposed in this pull request? Graviton java client support check version and backward compatibility ### Why are the changes needed? Fix: #2866 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? UT --- .gitignore | 2 +- .../client/GravitinoAdminClient.java | 12 ++- .../gravitino/client/GravitinoClient.java | 7 +- .../gravitino/client/GravitinoClientBase.java | 86 ++++++++++++++++-- .../gravitino/client/GravitinoVersion.java | 47 +++++++++- .../gravitino/client/HTTPClient.java | 58 ++++++++++++- .../datastrato/gravitino/client/TestBase.java | 5 +- .../gravitino/client/TestGravitinoClient.java | 87 +++++++++++++++++++ .../client/TestGravitinoClientBuilder.java | 2 +- .../client/TestGravitinoMetalake.java | 1 + .../client/TestGravitinoVersion.java | 73 ++++++++++++++++ .../hadoop/GravitinoMockServerBase.java | 17 ++++ common/build.gradle.kts | 65 ++++++++++++++ .../com/datastrato/gravitino/Version.java | 60 +++++++++++++ .../test/web/rest/KerberosOperationsIT.java | 4 +- .../test/web/rest/OAuth2OperationsIT.java | 2 +- .../test/web/rest/VersionOperationsIT.java | 2 +- server/build.gradle.kts | 43 --------- .../server/web/rest/VersionOperations.java | 19 +--- 19 files changed, 507 insertions(+), 85 deletions(-) create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java create mode 100644 common/src/main/java/com/datastrato/gravitino/Version.java diff --git a/.gitignore b/.gitignore index 15b5f41582d..e1620724b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,7 @@ out/** *.ipr distribution -server/src/main/resources/project.properties +common/src/main/resources/project.properties dev/docker/*/packages docs/build diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index 59393e8f58c..797991c7dde 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -40,11 +40,16 @@ public class GravitinoAdminClient extends GravitinoClientBase implements Support * * @param uri The base URI for the Gravitino API. * @param authDataProvider The provider of the data which is used for authentication. + * @param checkVersion Whether to check the version of the Gravitino server. Gravitino does not + * support the case that the client-side version is higher than the server-side version. * @param headers The base header for Gravitino API. */ private GravitinoAdminClient( - String uri, AuthDataProvider authDataProvider, Map headers) { - super(uri, authDataProvider, headers); + String uri, + AuthDataProvider authDataProvider, + boolean checkVersion, + Map headers) { + super(uri, authDataProvider, checkVersion, headers); } /** @@ -189,8 +194,7 @@ protected AdminClientBuilder(String uri) { public GravitinoAdminClient build() { Preconditions.checkArgument( uri != null && !uri.isEmpty(), "The argument 'uri' must be a valid URI"); - - return new GravitinoAdminClient(uri, authDataProvider, headers); + return new GravitinoAdminClient(uri, authDataProvider, checkVersion, headers); } } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java index f5584adb625..7b763a889ef 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java @@ -33,6 +33,8 @@ public class GravitinoClient extends GravitinoClientBase implements SupportsCata * @param uri The base URI for the Gravitino API. * @param metalakeName The specified metalake name. * @param authDataProvider The provider of the data which is used for authentication. + * @param checkVersion Whether to check the version of the Gravitino server. Gravitino does not + * support the case that the client-side version is higher than the server-side version. * @param headers The base header for Gravitino API. * @throws NoSuchMetalakeException if the metalake with specified name does not exist. */ @@ -40,8 +42,9 @@ private GravitinoClient( String uri, String metalakeName, AuthDataProvider authDataProvider, + boolean checkVersion, Map headers) { - super(uri, authDataProvider, headers); + super(uri, authDataProvider, checkVersion, headers); this.metalake = loadMetalake(NameIdentifier.of(metalakeName)); } @@ -143,7 +146,7 @@ public GravitinoClient build() { metalakeName != null && !metalakeName.isEmpty(), "The argument 'metalakeName' must be a valid name"); - return new GravitinoClient(uri, metalakeName, authDataProvider, headers); + return new GravitinoClient(uri, metalakeName, authDataProvider, checkVersion, headers); } } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java index 87a4410a4a3..6b83af03623 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java @@ -6,9 +6,14 @@ package com.datastrato.gravitino.client; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Version; import com.datastrato.gravitino.dto.responses.MetalakeResponse; import com.datastrato.gravitino.dto.responses.VersionResponse; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.json.JsonUtils; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.io.Closeable; import java.net.URI; @@ -39,16 +44,61 @@ public abstract class GravitinoClientBase implements Closeable { * * @param uri The base URI for the Gravitino API. * @param authDataProvider The provider of the data which is used for authentication. + * @param checkVersion Whether to check the version of the Gravitino server. * @param headers The base header of the Gravitino API. */ protected GravitinoClientBase( - String uri, AuthDataProvider authDataProvider, Map headers) { - this.restClient = - HTTPClient.builder(Collections.emptyMap()) - .uri(uri) - .withAuthDataProvider(authDataProvider) - .withHeaders(headers) - .build(); + String uri, + AuthDataProvider authDataProvider, + boolean checkVersion, + Map headers) { + ObjectMapper mapper = JsonUtils.objectMapper(); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + if (checkVersion) { + this.restClient = + HTTPClient.builder(Collections.emptyMap()) + .uri(uri) + .withAuthDataProvider(authDataProvider) + .withObjectMapper(mapper) + .withPreConnectHandler(this::checkVersion) + .withHeaders(headers) + .build(); + + } else { + this.restClient = + HTTPClient.builder(Collections.emptyMap()) + .uri(uri) + .withAuthDataProvider(authDataProvider) + .withObjectMapper(mapper) + .withHeaders(headers) + .build(); + } + } + + /** + * Check the compatibility of the client with the target server. + * + * @throws GravitinoRuntimeException If the client version is greater than the server version. + */ + public void checkVersion() { + GravitinoVersion serverVersion = serverVersion(); + GravitinoVersion clientVersion = clientVersion(); + if (clientVersion.compareTo(serverVersion) > 0) { + throw new GravitinoRuntimeException( + "Gravitino does not support the case that the client-side version is higher than the server-side version." + + "The client version is %s, and the server version %s", + clientVersion.version(), serverVersion.version()); + } + } + + /** + * Retrieves the version of the Gravitino client. + * + * @return A GravitinoVersion instance representing the version of the Gravitino client. + */ + public GravitinoVersion clientVersion() { + return new GravitinoVersion(Version.getCurrentVersionDTO()); } /** @@ -77,7 +127,17 @@ public GravitinoMetalake loadMetalake(NameIdentifier ident) throws NoSuchMetalak * * @return A GravitinoVersion instance representing the version of the Gravitino API. */ + @Deprecated public GravitinoVersion getVersion() { + return serverVersion(); + } + + /** + * Retrieves the server version of the Gravitino server. + * + * @return A GravitinoVersion instance representing the server version of the Gravitino API. + */ + public GravitinoVersion serverVersion() { VersionResponse resp = restClient.get( "api/version", @@ -108,6 +168,8 @@ public abstract static class Builder { protected String uri; /** The authentication provider. */ protected AuthDataProvider authDataProvider; + /** The check version flag. */ + protected boolean checkVersion = true; /** The request base header for the Gravitino API. */ protected Map headers = ImmutableMap.of(); @@ -130,6 +192,16 @@ public Builder withSimpleAuth() { return this; } + /** + * Optional, set a flag to verify the client is supported to connector the server + * + * @return This Builder instance for method chaining. + */ + public Builder withVersionCheckDisabled() { + this.checkVersion = false; + return this; + } + /** * Sets OAuth2TokenProvider for Gravitino. * diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java index 837ef347a03..26bf5007f38 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java @@ -5,10 +5,55 @@ package com.datastrato.gravitino.client; import com.datastrato.gravitino.dto.VersionDTO; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.google.common.annotations.VisibleForTesting; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** Gravitino version information. */ -public class GravitinoVersion extends VersionDTO { +public class GravitinoVersion extends VersionDTO implements Comparable { + + private static final int VERSION_PART_NUMBER = 3; + + @VisibleForTesting + GravitinoVersion(String version, String compoileDate, String gitCommit) { + super(version, compoileDate, gitCommit); + } + GravitinoVersion(VersionDTO versionDTO) { super(versionDTO.version(), versionDTO.compileDate(), versionDTO.gitCommit()); } + + @VisibleForTesting + /** @return parse the version number for a version string */ + int[] getVersionNumber() { + Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(-\\w+){0,1}"); + Matcher matcher = pattern.matcher(version()); + if (matcher.matches()) { + int[] versionNumbers = new int[VERSION_PART_NUMBER]; + for (int i = 0; i < VERSION_PART_NUMBER; i++) { + versionNumbers[i] = Integer.parseInt(matcher.group(i + 1)); + } + return versionNumbers; + } + throw new GravitinoRuntimeException("Invalid version string " + version()); + } + + @Override + public int compareTo(Object o) { + if (!(o instanceof GravitinoVersion)) { + return 1; + } + GravitinoVersion other = (GravitinoVersion) o; + + int[] left = getVersionNumber(); + int[] right = other.getVersionNumber(); + for (int i = 0; i < VERSION_PART_NUMBER; i++) { + int v = left[i] - right[i]; + if (v != 0) { + return v; + } + } + return 0; + } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java index 1e1c5f26de1..71d10f4aea9 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java @@ -79,6 +79,21 @@ public class HTTPClient implements RESTClient { private final ObjectMapper mapper; private final AuthDataProvider authDataProvider; + // Handler to be executed before connecting to the server. + private final Runnable beforeConnectHandler; + // Handler status + enum HandlerStatus { + // The handler has not been executed yet. + Start, + // The handler has been executed successfully. + Finished, + // The handler is currently running. + Running, + } + + // The status of the handler. + private volatile HandlerStatus handlerStatus = HandlerStatus.Start; + /** * Constructs an instance of HTTPClient with the provided information. * @@ -86,12 +101,14 @@ public class HTTPClient implements RESTClient { * @param baseHeaders A map of base headers to be included in all HTTP requests. * @param objectMapper The ObjectMapper used for JSON serialization and deserialization. * @param authDataProvider The provider of authentication data. + * @param beforeConnectHandler The function to be executed before connecting to the server. */ private HTTPClient( String uri, Map baseHeaders, ObjectMapper objectMapper, - AuthDataProvider authDataProvider) { + AuthDataProvider authDataProvider, + Runnable beforeConnectHandler) { this.uri = uri; this.mapper = objectMapper; @@ -106,6 +123,11 @@ private HTTPClient( this.httpClient = clientBuilder.build(); this.authDataProvider = authDataProvider; + + if (beforeConnectHandler == null) { + handlerStatus = HandlerStatus.Finished; + } + this.beforeConnectHandler = beforeConnectHandler; } /** @@ -314,6 +336,11 @@ private T execute( Map headers, Consumer errorHandler, Consumer> responseHeaders) { + + if (handlerStatus != HandlerStatus.Finished) { + performPreConnectHandler(); + } + if (path.startsWith("/")) { throw new RESTException( "Received a malformed path for a REST request: %s. Paths should not start with /", path); @@ -383,6 +410,21 @@ private T execute( } } + private synchronized void performPreConnectHandler() { + // beforeConnectHandler is a pre-connection handler that needs to be executed before the first + // HTTP request. if the handler execute fails, we set the status to Start to retry the handler. + if (handlerStatus == HandlerStatus.Start) { + handlerStatus = HandlerStatus.Running; + try { + beforeConnectHandler.run(); + handlerStatus = HandlerStatus.Finished; + } catch (Exception e) { + handlerStatus = HandlerStatus.Start; + throw e; + } + } + } + /** * Sends an HTTP HEAD request to the specified path and processes the response. * @@ -655,6 +697,7 @@ public static class Builder { private String uri; private ObjectMapper mapper = JsonUtils.objectMapper(); private AuthDataProvider authDataProvider; + private Runnable beforeConnectHandler; private Builder(Map properties) { this.properties = properties; @@ -707,6 +750,17 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { return this; } + /** + * Sets the preConnect handle for the HTTP client. + * + * @param beforeConnectHandler The handle run before connect to the server . + * @return This Builder instance for method chaining. + */ + public Builder withPreConnectHandler(Runnable beforeConnectHandler) { + this.beforeConnectHandler = beforeConnectHandler; + return this; + } + /** * Sets the AuthDataProvider for the HTTP client. * @@ -725,7 +779,7 @@ public Builder withAuthDataProvider(AuthDataProvider authDataProvider) { */ public HTTPClient build() { - return new HTTPClient(uri, baseHeaders, mapper, authDataProvider); + return new HTTPClient(uri, baseHeaders, mapper, authDataProvider, beforeConnectHandler); } } diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java index f55dceb9ab7..35ebe76661e 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java @@ -22,7 +22,7 @@ public abstract class TestBase { - private static final ObjectMapper MAPPER = JsonUtils.objectMapper(); + protected static final ObjectMapper MAPPER = JsonUtils.objectMapper(); protected static ClientAndServer mockServer; @@ -32,7 +32,8 @@ public abstract class TestBase { public static void setUp() throws Exception { mockServer = ClientAndServer.startClientAndServer(0); int port = mockServer.getLocalPort(); - client = GravitinoAdminClient.builder("http://127.0.0.1:" + port).build(); + client = + GravitinoAdminClient.builder("http://127.0.0.1:" + port).withVersionCheckDisabled().build(); } @AfterAll diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java index 7a5705da21b..9a630ce34df 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java @@ -6,14 +6,18 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Version; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.MetalakeDTO; +import com.datastrato.gravitino.dto.VersionDTO; import com.datastrato.gravitino.dto.requests.MetalakeCreateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdatesRequest; import com.datastrato.gravitino.dto.responses.DropResponse; import com.datastrato.gravitino.dto.responses.ErrorResponse; import com.datastrato.gravitino.dto.responses.MetalakeListResponse; import com.datastrato.gravitino.dto.responses.MetalakeResponse; +import com.datastrato.gravitino.dto.responses.VersionResponse; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.RESTException; @@ -27,6 +31,9 @@ import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockserver.matchers.Times; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; public class TestGravitinoClient extends TestBase { @@ -230,4 +237,84 @@ public void testDropMetalake() throws JsonProcessingException { Assertions.assertTrue( excep1.getMessage().contains("Metalake namespace must be non-null and empty")); } + + @Test + public void testGetServerVersion() throws JsonProcessingException { + String version = "0.1.3"; + String date = "2024-01-03 12:28:33"; + String commitId = "6ef1f9d"; + + VersionResponse resp = new VersionResponse(new VersionDTO(version, date, commitId)); + buildMockResource(Method.GET, "/api/version", null, resp, HttpStatus.SC_OK); + GravitinoVersion gravitinoVersion = client.serverVersion(); + + Assertions.assertEquals(version, gravitinoVersion.version()); + Assertions.assertEquals(date, gravitinoVersion.compileDate()); + Assertions.assertEquals(commitId, gravitinoVersion.gitCommit()); + } + + @Test + public void testGetClientVersion() { + GravitinoVersion version = client.clientVersion(); + Version.VersionInfo currentVersion = Version.getCurrentVersion(); + + Assertions.assertEquals(currentVersion.version, version.version()); + Assertions.assertEquals(currentVersion.compileDate, version.compileDate()); + Assertions.assertEquals(currentVersion.gitCommit, version.gitCommit()); + } + + @Test + public void testCheckVersionFailed() throws JsonProcessingException { + String version = "0.1.1"; + String date = "2024-01-03 12:28:33"; + String commitId = "6ef1f9d"; + + VersionResponse resp = new VersionResponse(new VersionDTO(version, date, commitId)); + buildMockResource(Method.GET, "/api/version", null, resp, HttpStatus.SC_OK); + + // check the client version is greater than server version + Assertions.assertThrows(GravitinoRuntimeException.class, () -> client.checkVersion()); + } + + @Test + public void testCheckVersionSuccess() throws JsonProcessingException { + VersionResponse resp = new VersionResponse(Version.getCurrentVersionDTO()); + buildMockResource(Method.GET, "/api/version", null, resp, HttpStatus.SC_OK); + + // check the client version is equal to server version + Assertions.assertDoesNotThrow(() -> client.checkVersion()); + + String version = "100.1.1-SNAPSHOT"; + String date = "2024-01-03 12:28:33"; + String commitId = "6ef1f9d"; + + resp = new VersionResponse(new VersionDTO(version, date, commitId)); + buildMockResource(Method.GET, "/api/version", null, resp, HttpStatus.SC_OK); + + // check the client version is less than server version + Assertions.assertDoesNotThrow(() -> client.checkVersion()); + } + + @Test + public void testUnusedDTOAttribute() throws JsonProcessingException { + VersionResponse resp = new VersionResponse(Version.getCurrentVersionDTO()); + + HttpRequest mockRequest = HttpRequest.request("/api/version").withMethod(Method.GET.name()); + HttpResponse mockResponse = HttpResponse.response().withStatusCode(HttpStatus.SC_OK); + String respJson = MAPPER.writeValueAsString(resp); + + // add unused attribute for version DTO + respJson = respJson.replace("\"gitCommit\"", "\"unused_key\":\"unused_value\", \"gitCommit\""); + mockResponse = mockResponse.withBody(respJson); + mockServer.when(mockRequest, Times.exactly(1)).respond(mockResponse); + + Assertions.assertDoesNotThrow( + () -> { + GravitinoVersion version = client.serverVersion(); + Version.VersionInfo currentVersion = Version.getCurrentVersion(); + Assertions.assertEquals(currentVersion.version, version.version()); + Assertions.assertEquals(currentVersion.compileDate, version.compileDate()); + Assertions.assertEquals(currentVersion.gitCommit, version.gitCommit()); + }); + } } diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java index 965632e64fa..b8ba3c64a0a 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java @@ -42,7 +42,7 @@ class MockGravitinoClient extends GravitinoClientBase { */ private MockGravitinoClient( String uri, AuthDataProvider authDataProvider, Map headers) { - super(uri, authDataProvider, headers); + super(uri, authDataProvider, false, headers); this.headers = headers; } diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java index b8f109736a8..6000bd1e613 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java @@ -64,6 +64,7 @@ public static void setUp() throws Exception { gravitinoClient = GravitinoClient.builder("http://127.0.0.1:" + mockServer.getLocalPort()) .withMetalake(metalakeName) + .withVersionCheckDisabled() .build(); } diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java new file mode 100644 index 00000000000..d9ca564359f --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java @@ -0,0 +1,73 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import org.junit.jupiter.api.Test; + +public class TestGravitinoVersion { + @Test + void testParseVersionString() { + // Test a valid the version string + GravitinoVersion version = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + int[] versionNumber = version.getVersionNumber(); + assertEquals(2, versionNumber[0]); + assertEquals(5, versionNumber[1]); + assertEquals(3, versionNumber[2]); + + // Test a valid the version string with SNAPSHOT + version = new GravitinoVersion("2.5.3-SNAPSHOT", "2023-01-01", "1234567"); + versionNumber = version.getVersionNumber(); + assertEquals(2, versionNumber[0]); + assertEquals(5, versionNumber[1]); + assertEquals(3, versionNumber[2]); + + // Test a valid the version string with alpha + version = new GravitinoVersion("2.5.3-alpha", "2023-01-01", "1234567"); + versionNumber = version.getVersionNumber(); + assertEquals(2, versionNumber[0]); + assertEquals(5, versionNumber[1]); + assertEquals(3, versionNumber[2]); + + // Test an invalid the version string with 2 part + version = new GravitinoVersion("2.5", "2023-01-01", "1234567"); + assertThrows(GravitinoRuntimeException.class, version::getVersionNumber); + + // Test an invalid the version string with 4 part + version = new GravitinoVersion("2.5.7.6", "2023-01-01", "1234567"); + assertThrows(GravitinoRuntimeException.class, version::getVersionNumber); + + // Test an invalid the version string with not number + version = new GravitinoVersion("a.b.c", "2023-01-01", "1234567"); + assertThrows(GravitinoRuntimeException.class, version::getVersionNumber); + } + + @Test + void testVersionCompare() { + GravitinoVersion version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + // test equal + GravitinoVersion version2 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + assertEquals(0, version1.compareTo(version2)); + + // test less than + version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + version2 = new GravitinoVersion("2.5.4", "2023-01-01", "1234567"); + assertTrue(version1.compareTo(version2) < 1); + + // test greater than + version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + version2 = new GravitinoVersion("2.5.2", "2023-01-01", "1234567"); + assertTrue(version1.compareTo(version2) > 0); + + // test equal with suffix + version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); + version2 = new GravitinoVersion("2.5.3-SNAPSHOT", "2023-01-01", "1234567"); + assertEquals(0, version1.compareTo(version2)); + } +} diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java index 40182891ff7..d2dcbec69fe 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java @@ -7,6 +7,7 @@ import static org.apache.hc.core5.http.HttpStatus.SC_OK; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Version; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.MetalakeDTO; @@ -14,6 +15,7 @@ import com.datastrato.gravitino.dto.responses.CatalogResponse; import com.datastrato.gravitino.dto.responses.FilesetResponse; import com.datastrato.gravitino.dto.responses.MetalakeResponse; +import com.datastrato.gravitino.dto.responses.VersionResponse; import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.shaded.com.fasterxml.jackson.core.JsonProcessingException; @@ -51,11 +53,13 @@ public abstract class GravitinoMockServerBase { public static void setup() { mockServer = ClientAndServer.startClientAndServer(0); port = mockServer.getLocalPort(); + mockAPIVersion(); } @AfterEach public void reset() { mockServer.reset(); + mockAPIVersion(); } @AfterAll @@ -102,6 +106,19 @@ protected static void buildMockResource( buildMockResource(method, path, Collections.emptyMap(), reqBody, respBody, statusCode); } + protected static void mockAPIVersion() { + try { + buildMockResource( + Method.GET, + "/api/version", + null, + new VersionResponse(Version.getCurrentVersionDTO()), + HttpStatus.SC_OK); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + protected static void mockMetalakeDTO(String name, String comment) { MetalakeDTO mockMetalake = MetalakeDTO.builder() diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 0348e4fbb68..5a019da140a 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -2,6 +2,10 @@ * Copyright 2023 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ + +import java.text.SimpleDateFormat +import java.util.Date + plugins { `maven-publish` id("java") @@ -31,3 +35,64 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) } + +fun getGitCommitId(): String { + var gitCommitId: String + try { + val gitFolder = rootDir.path + "/.git/" + val head = File(gitFolder + "HEAD").readText().split(":") + val isCommit = head.size == 1 + gitCommitId = if (isCommit) { + head[0].trim() + } else { + val refHead = File(gitFolder + head[1].trim()) + refHead.readText().trim() + } + } catch (e: Exception) { + println("WARN: Unable to get Git commit id : ${e.message}") + gitCommitId = "" + } + return gitCommitId +} + +val propertiesFile = "src/main/resources/project.properties" +val writeProjectPropertiesFile = tasks.register("writeProjectPropertiesFile") { + val propertiesFile = file(propertiesFile) + if (propertiesFile.exists()) { + propertiesFile.delete() + } + + val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm:ss") + + val compileDate = dateFormat.format(Date()) + val projectVersion = project.version.toString() + val commitId = getGitCommitId() + + propertiesFile.parentFile.mkdirs() + propertiesFile.createNewFile() + propertiesFile.writer().use { writer -> + writer.write( + "#\n" + + "# Copyright 2023 Datastrato Pvt Ltd.\n" + + "# This software is licensed under the Apache License version 2.\n" + + "#\n" + ) + writer.write("project.version=$projectVersion\n") + writer.write("compile.date=$compileDate\n") + writer.write("git.commit.id=$commitId\n") + } +} + +tasks { + jar { + dependsOn(writeProjectPropertiesFile) + doFirst() { + if (!file(propertiesFile).exists()) { + throw GradleException("$propertiesFile file not generated!") + } + } + } + clean { + delete("$propertiesFile") + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/Version.java b/common/src/main/java/com/datastrato/gravitino/Version.java new file mode 100644 index 00000000000..9f6664e38e5 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/Version.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino; + +import com.datastrato.gravitino.dto.VersionDTO; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.io.IOException; +import java.util.Properties; + +/** Retrieve the version and build information from the building process */ +public class Version { + + private static final Version INSTANCE = new Version(); + + private VersionInfo versionInfo; + private VersionDTO versionDTO; + + private Version() { + Properties projectProperties = new Properties(); + try { + VersionInfo currentVersionInfo = new VersionInfo(); + projectProperties.load( + Version.class.getClassLoader().getResourceAsStream("project.properties")); + currentVersionInfo.version = projectProperties.getProperty("project.version"); + currentVersionInfo.compileDate = projectProperties.getProperty("compile.date"); + currentVersionInfo.gitCommit = projectProperties.getProperty("git.commit.id"); + + versionInfo = currentVersionInfo; + versionDTO = + new VersionDTO( + currentVersionInfo.version, + currentVersionInfo.compileDate, + currentVersionInfo.gitCommit); + } catch (IOException e) { + throw new GravitinoRuntimeException(e, "Failed to get Gravitino version"); + } + } + + /** @return the current versionInfo */ + public static VersionInfo getCurrentVersion() { + return INSTANCE.versionInfo; + } + + /** @return the current version DTO */ + public static VersionDTO getCurrentVersionDTO() { + return INSTANCE.versionDTO; + } + + /** Store version information */ + public static class VersionInfo { + /** build version */ + public String version; + /** build time */ + public String compileDate; + /** build commit id */ + public String gitCommit; + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java index 92b4a03bb16..0c28d1b5d2b 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java @@ -76,8 +76,7 @@ public static void stopIntegrationTest() throws IOException, InterruptedExceptio @Test public void testAuthenticationApi() throws Exception { - GravitinoVersion gravitinoVersion = client.getVersion(); - client.getVersion(); + GravitinoVersion gravitinoVersion = client.serverVersion(); Assertions.assertEquals(System.getenv("PROJECT_VERSION"), gravitinoVersion.version()); Assertions.assertFalse(gravitinoVersion.compileDate().isEmpty()); @@ -88,7 +87,6 @@ public void testAuthenticationApi() throws Exception { // Test to re-login with the keytab Uninterruptibles.sleepUninterruptibly(6, TimeUnit.SECONDS); - client.getVersion(); Assertions.assertEquals(System.getenv("PROJECT_VERSION"), gravitinoVersion.version()); Assertions.assertFalse(gravitinoVersion.compileDate().isEmpty()); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/OAuth2OperationsIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/OAuth2OperationsIT.java index 5a19c2ce1ca..e17afc1daf6 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/OAuth2OperationsIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/OAuth2OperationsIT.java @@ -59,7 +59,7 @@ public static void startIntegrationTest() throws Exception { @Test public void testAuthenticationApi() throws Exception { - GravitinoVersion gravitinoVersion = client.getVersion(); + GravitinoVersion gravitinoVersion = client.serverVersion(); Assertions.assertEquals(System.getenv("PROJECT_VERSION"), gravitinoVersion.version()); Assertions.assertFalse(gravitinoVersion.compileDate().isEmpty()); if (testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) { diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/VersionOperationsIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/VersionOperationsIT.java index 0de4a0f407c..a86e3edf89d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/VersionOperationsIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/VersionOperationsIT.java @@ -13,7 +13,7 @@ public class VersionOperationsIT extends AbstractIT { @Test public void testGetVersion() { - GravitinoVersion gravitinoVersion = client.getVersion(); + GravitinoVersion gravitinoVersion = client.serverVersion(); Assertions.assertEquals(System.getenv("PROJECT_VERSION"), gravitinoVersion.version()); Assertions.assertFalse(gravitinoVersion.compileDate().isEmpty()); if (testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) { diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 9400188fdfd..bdd557ec5a7 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -3,9 +3,6 @@ * This software is licensed under the Apache License version 2. */ -import java.text.SimpleDateFormat -import java.util.Date - plugins { `maven-publish` id("java") @@ -71,49 +68,9 @@ fun getGitCommitId(): String { return gitCommitId } -val propertiesFile = "src/main/resources/project.properties" -fun writeProjectPropertiesFile() { - val propertiesFile = file(propertiesFile) - if (propertiesFile.exists()) { - propertiesFile.delete() - } - - val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm:ss") - - val compileDate = dateFormat.format(Date()) - val projectVersion = project.version.toString() - val commitId = getGitCommitId() - - propertiesFile.parentFile.mkdirs() - propertiesFile.createNewFile() - propertiesFile.writer().use { writer -> - writer.write( - "#\n" + - "# Copyright 2023 Datastrato Pvt Ltd.\n" + - "# This software is licensed under the Apache License version 2.\n" + - "#\n" - ) - writer.write("project.version=$projectVersion\n") - writer.write("compile.date=$compileDate\n") - writer.write("git.commit.id=$commitId\n") - } -} - tasks { - jar { - doFirst() { - writeProjectPropertiesFile() - val file = file(propertiesFile) - if (!file.exists()) { - throw GradleException("$propertiesFile file not generated!") - } - } - } test { environment("GRAVITINO_HOME", rootDir.path) environment("GRAVITINO_TEST", "true") } - clean { - delete(file(propertiesFile).getParentFile()) - } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/VersionOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/VersionOperations.java index 9f42dedcf7b..14b5bbe4e7c 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/VersionOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/VersionOperations.java @@ -6,12 +6,10 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; -import com.datastrato.gravitino.dto.VersionDTO; +import com.datastrato.gravitino.Version; import com.datastrato.gravitino.dto.responses.VersionResponse; import com.datastrato.gravitino.metrics.MetricNames; import com.datastrato.gravitino.server.web.Utils; -import java.io.IOException; -import java.util.Properties; import javax.servlet.http.HttpServlet; import javax.ws.rs.Consumes; import javax.ws.rs.GET; @@ -29,19 +27,6 @@ public class VersionOperations extends HttpServlet { @Timed(name = "version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "version", absolute = true) public Response getVersion() { - Properties projectProperties = new Properties(); - try { - projectProperties.load( - VersionOperations.class.getClassLoader().getResourceAsStream("project.properties")); - String version = projectProperties.getProperty("project.version"); - String compileDate = projectProperties.getProperty("compile.date"); - String gitCommit = projectProperties.getProperty("git.commit.id"); - - VersionDTO versionDTO = new VersionDTO(version, compileDate, gitCommit); - - return Utils.ok(new VersionResponse(versionDTO)); - } catch (IOException e) { - return Utils.internalError("Failed to get Gravitino version", e); - } + return Utils.ok(new VersionResponse(Version.getCurrentVersionDTO())); } } From 4a09d11bd0d4f9c43660b063d71d482338699132 Mon Sep 17 00:00:00 2001 From: FANNG Date: Thu, 18 Apr 2024 16:17:37 +0800 Subject: [PATCH 057/106] [#2980] test(core): add test for event listener (#2981) ### What changes were proposed in this pull request? add test for event ### Why are the changes needed? Fix: #2980 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? test patch --- core/build.gradle.kts | 1 + .../listener/api/event/ListCatalogEvent.java | 2 +- .../api/event/ListCatalogFailureEvent.java | 2 +- .../listener/api/event/ListFilesetEvent.java | 2 +- .../api/event/ListFilesetFailureEvent.java | 2 +- .../listener/api/event/ListSchemaEvent.java | 2 +- .../api/event/ListSchemaFailureEvent.java | 2 +- .../listener/api/event/ListTableEvent.java | 2 +- .../api/event/ListTableFailureEvent.java | 2 +- .../listener/api/event/ListTopicEvent.java | 2 +- .../api/event/ListTopicFailureEvent.java | 2 +- .../listener/api/info/FilesetInfo.java | 56 +++- .../listener/api/info/TableInfo.java | 94 ++++-- .../listener/DummyEventListener.java | 69 +++++ .../listener/TestEventListenerManager.java | 57 +--- .../listener/api/event/TestCatalogEvent.java | 248 ++++++++++++++++ .../listener/api/event/TestFilesetEvent.java | 228 ++++++++++++++ .../listener/api/event/TestMetalakeEvent.java | 210 +++++++++++++ .../listener/api/event/TestSchemaEvent.java | 215 ++++++++++++++ .../listener/api/event/TestTableEvent.java | 281 ++++++++++++++++++ .../listener/api/event/TestTopicEvent.java | 210 +++++++++++++ 21 files changed, 1591 insertions(+), 98 deletions(-) create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/DummyEventListener.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java create mode 100644 core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 6f6fb3b7266..983bbb59c02 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) + testImplementation(libs.awaitility) testImplementation(libs.h2db) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java index 37b4d666f97..2e1421ac09e 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java @@ -21,7 +21,7 @@ public final class ListCatalogEvent extends CatalogEvent { * @param namespace The namespace from which catalogs were listed. */ public ListCatalogEvent(String user, Namespace namespace) { - super(user, NameIdentifier.of(namespace.toString())); + super(user, NameIdentifier.of(namespace.levels())); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java index 30e4b24e3e6..203310fde74 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java @@ -25,7 +25,7 @@ public final class ListCatalogFailureEvent extends CatalogFailureEvent { * @param exception The exception encountered during the attempt to list catalogs. */ public ListCatalogFailureEvent(String user, Exception exception, Namespace namespace) { - super(user, NameIdentifier.of(namespace.toString()), exception); + super(user, NameIdentifier.of(namespace.levels()), exception); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java index c5e99d6a70f..5d188631466 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java @@ -23,7 +23,7 @@ public final class ListFilesetEvent extends FilesetEvent { * contextual information, identifying the scope and boundaries of the listing operation. */ public ListFilesetEvent(String user, Namespace namespace) { - super(user, NameIdentifier.of(namespace.toString())); + super(user, NameIdentifier.of(namespace.levels())); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java index bf063fd891e..745945b3847 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java @@ -26,7 +26,7 @@ public final class ListFilesetFailureEvent extends FilesetFailureEvent { * an indicator of the issues that caused the failure. */ public ListFilesetFailureEvent(String user, Namespace namespace, Exception exception) { - super(user, NameIdentifier.of(namespace.toString()), exception); + super(user, NameIdentifier.of(namespace.levels()), exception); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java index dd13768aabb..08aa0b38a33 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java @@ -15,7 +15,7 @@ public final class ListSchemaEvent extends SchemaEvent { private final Namespace namespace; public ListSchemaEvent(String user, Namespace namespace) { - super(user, NameIdentifier.of(namespace.toString())); + super(user, NameIdentifier.of(namespace.levels())); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java index cb233503db1..b1c0a037ae4 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java @@ -18,7 +18,7 @@ public final class ListSchemaFailureEvent extends SchemaFailureEvent { private final Namespace namespace; public ListSchemaFailureEvent(String user, Namespace namespace, Exception exception) { - super(user, NameIdentifier.of(namespace.toString()), exception); + super(user, NameIdentifier.of(namespace.levels()), exception); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java index 63eb0d410cc..7a28ebe08e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java @@ -28,7 +28,7 @@ public final class ListTableEvent extends TableEvent { * @param namespace The namespace from which tables were listed. */ public ListTableEvent(String user, Namespace namespace) { - super(user, NameIdentifier.parse(namespace.toString())); + super(user, NameIdentifier.of(namespace.levels())); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java index 706fc71d166..169b6ea80d1 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java @@ -25,7 +25,7 @@ public final class ListTableFailureEvent extends TableFailureEvent { * @param exception The exception encountered during the attempt to list tables. */ public ListTableFailureEvent(String user, Namespace namespace, Exception exception) { - super(user, NameIdentifier.of(namespace.toString()), exception); + super(user, NameIdentifier.of(namespace.levels()), exception); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java index 4eacc1fbf1b..16297fe385b 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java @@ -21,7 +21,7 @@ public final class ListTopicEvent extends TopicEvent { * @param namespace The namespace from which topics were listed. */ public ListTopicEvent(String user, Namespace namespace) { - super(user, NameIdentifier.parse(namespace.toString())); + super(user, NameIdentifier.of(namespace.levels())); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java index e98212fb747..a7c686f6256 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java @@ -25,7 +25,7 @@ public final class ListTopicFailureEvent extends TopicFailureEvent { * @param exception The exception encountered during the attempt to list topics. */ public ListTopicFailureEvent(String user, Namespace namespace, Exception exception) { - super(user, NameIdentifier.of(namespace.toString()), exception); + super(user, NameIdentifier.of(namespace.levels()), exception); this.namespace = namespace; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java index eb81f86510f..93b0f1e5aba 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java @@ -12,6 +12,7 @@ import java.util.Map; import javax.annotation.Nullable; +/** Encapsulates read-only information about a fileset, intended for use in event listeners. */ @DeveloperApi public final class FilesetInfo { private final String name; @@ -21,6 +22,11 @@ public final class FilesetInfo { private final Map properties; @Nullable private final Audit audit; + /** + * Constructs a FilesetInfo object from a Fileset instance. + * + * @param fileset The source Fileset instance. + */ public FilesetInfo(Fileset fileset) { this( fileset.name(), @@ -31,6 +37,16 @@ public FilesetInfo(Fileset fileset) { fileset.auditInfo()); } + /** + * Constructs a FilesetInfo object with specified details. + * + * @param name The name of the fileset. + * @param comment An optional comment about the fileset. Can be {@code null}. + * @param type The type of the fileset. + * @param storageLocation The storage location of the fileset. + * @param properties A map of properties associated with the fileset. Can be {@code null}. + * @param audit Optional audit information. Can be {@code null}. + */ public FilesetInfo( String name, String comment, @@ -42,36 +58,62 @@ public FilesetInfo( this.comment = comment; this.type = type; this.storageLocation = storageLocation; - if (properties == null) { - this.properties = ImmutableMap.of(); - } else { - this.properties = ImmutableMap.builder().putAll(properties).build(); - } + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); this.audit = audit; } + /** + * Returns the audit information. + * + * @return The audit information, or {@code null} if not available. + */ @Nullable public Audit auditInfo() { return audit; } + /** + * Returns the name of the fileset. + * + * @return The fileset name. + */ public String name() { return name; } - public Fileset.Type getType() { + /** + * Returns the type of the fileset. + * + * @return The fileset type. + */ + public Fileset.Type type() { return type; } - public String getStorageLocation() { + /** + * Returns the storage location of the fileset. + * + * @return The storage location. + */ + public String storageLocation() { return storageLocation; } + /** + * Returns the optional comment about the fileset. + * + * @return The comment, or {@code null} if not provided. + */ @Nullable public String comment() { return comment; } + /** + * Returns the properties associated with the fileset. + * + * @return The properties map. + */ public Map properties() { return properties; } diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java index 2f7329e0502..a877c00af24 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java @@ -35,6 +35,11 @@ public final class TableInfo { private final Index[] indexes; @Nullable private final Audit auditInfo; + /** + * Constructs a TableInfo object from a Table instance. + * + * @param table The source Table instance. + */ public TableInfo(Table table) { this( table.name(), @@ -48,6 +53,19 @@ public TableInfo(Table table) { table.auditInfo()); } + /** + * Constructs a TableInfo object with specified details. + * + * @param name Name of the table. + * @param columns Array of columns in the table. + * @param comment Optional comment about the table. + * @param properties Map of table properties. + * @param partitions Array of partition transforms. + * @param distribution Table distribution configuration. + * @param sortOrders Array of sort order configurations. + * @param indexes Array of indexes on the table. + * @param auditInfo Optional audit information. + */ public TableInfo( String name, Column[] columns, @@ -61,69 +79,93 @@ public TableInfo( this.name = name; this.columns = columns.clone(); this.comment = comment; - if (properties == null) { - this.properties = ImmutableMap.of(); - } else { - this.properties = ImmutableMap.builder().putAll(properties).build(); - } - if (partitions == null) { - this.partitions = new Transform[0]; - } else { - this.partitions = partitions.clone(); - } - if (distribution == null) { - this.distribution = Distributions.NONE; - } else { - this.distribution = distribution; - } - if (sortOrders == null) { - this.sortOrders = new SortOrder[0]; - } else { - this.sortOrders = sortOrders.clone(); - } - if (indexes == null) { - this.indexes = Indexes.EMPTY_INDEXES; - } else { - this.indexes = indexes.clone(); - } + this.properties = properties == null ? ImmutableMap.of() : ImmutableMap.copyOf(properties); + this.partitions = partitions == null ? new Transform[0] : partitions.clone(); + this.distribution = distribution == null ? Distributions.NONE : distribution; + this.sortOrders = sortOrders == null ? new SortOrder[0] : sortOrders.clone(); + this.indexes = indexes == null ? Indexes.EMPTY_INDEXES : indexes.clone(); this.auditInfo = auditInfo; } - /** Audit information is null when tableInfo is generated from create table request. */ + /** + * Returns the audit information for the table. + * + * @return Audit information, or {@code null} if not available. + */ @Nullable public Audit auditInfo() { return this.auditInfo; } + /** + * Returns the name of the table. + * + * @return Table name. + */ public String name() { return name; } + /** + * Returns the columns of the table. + * + * @return Array of table columns. + */ public Column[] columns() { return columns; } + /** + * Returns the partitioning transforms applied to the table. + * + * @return Array of partition transforms. + */ public Transform[] partitioning() { return partitions; } + /** + * Returns the sort order configurations for the table. + * + * @return Array of sort orders. + */ public SortOrder[] sortOrder() { return sortOrders; } + /** + * Returns the distribution configuration for the table. + * + * @return Distribution configuration. + */ public Distribution distribution() { return distribution; } + /** + * Returns the indexes applied to the table. + * + * @return Array of indexes. + */ public Index[] index() { return indexes; } + /** + * Returns the optional comment about the table. + * + * @return Table comment, or {@code null} if not provided. + */ @Nullable public String comment() { return comment; } + /** + * Returns the properties associated with the table. + * + * @return Map of table properties. + */ public Map properties() { return properties; } diff --git a/core/src/test/java/com/datastrato/gravitino/listener/DummyEventListener.java b/core/src/test/java/com/datastrato/gravitino/listener/DummyEventListener.java new file mode 100644 index 00000000000..a78a07349a0 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/DummyEventListener.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener; + +import com.datastrato.gravitino.listener.api.EventListenerPlugin; +import com.datastrato.gravitino.listener.api.event.Event; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Assertions; + +public class DummyEventListener implements EventListenerPlugin { + Map properties; + @Getter LinkedList events = new LinkedList<>(); + + @Override + public void init(Map properties) { + this.properties = properties; + } + + @Override + public void start() {} + + @Override + public void stop() {} + + @Override + public void onPostEvent(Event event) { + this.events.add(event); + } + + @Override + public Mode mode() { + return Mode.SYNC; + } + + public Event popEvent() { + Assertions.assertTrue(events.size() > 0, "No events to pop"); + return events.removeLast(); + } + + public static class DummyAsyncEventListener extends DummyEventListener { + public List tryGetEvents() { + Awaitility.await() + .atMost(20, TimeUnit.SECONDS) + .pollInterval(10, TimeUnit.MILLISECONDS) + .until(() -> getEvents().size() > 0); + return getEvents(); + } + + @Override + public Mode mode() { + return Mode.ASYNC_SHARED; + } + } + + public static class DummyAsyncIsolatedEventListener extends DummyAsyncEventListener { + @Override + public Mode mode() { + return Mode.ASYNC_ISOLATED; + } + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java b/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java index f670a39404c..fa1c15044e0 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/TestEventListenerManager.java @@ -6,17 +6,16 @@ package com.datastrato.gravitino.listener; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.listener.DummyEventListener.DummyAsyncEventListener; +import com.datastrato.gravitino.listener.DummyEventListener.DummyAsyncIsolatedEventListener; import com.datastrato.gravitino.listener.api.EventListenerPlugin; import com.datastrato.gravitino.listener.api.event.Event; import com.google.common.collect.ImmutableSet; -import java.time.Instant; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.Getter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -27,58 +26,6 @@ protected DummyEvent(String user, NameIdentifier identifier) { } } - static class DummyEventListener implements EventListenerPlugin { - Map properties; - @Getter List events = new ArrayList<>(); - - @Override - public void init(Map properties) { - this.properties = properties; - } - - @Override - public void start() {} - - @Override - public void stop() {} - - @Override - public void onPostEvent(Event event) { - this.events.add(event); - } - - @Override - public Mode mode() { - return Mode.SYNC; - } - } - - static class DummyAsyncEventListener extends DummyEventListener { - List tryGetEvents() { - Instant waitTime = Instant.now().plusSeconds(20); - while (getEvents().size() == 0 && Instant.now().isBefore(waitTime)) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - return getEvents(); - } - - @Override - public Mode mode() { - return Mode.ASYNC_SHARED; - } - } - - static class DummyAsyncIsolatedEventListener extends DummyAsyncEventListener { - @Override - public Mode mode() { - return Mode.ASYNC_ISOLATED; - } - } - private static final DummyEvent DUMMY_EVENT_INSTANCE = new DummyEvent("user", NameIdentifier.of("a", "b")); diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java new file mode 100644 index 00000000000..bccd7e705f9 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java @@ -0,0 +1,248 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.CatalogDispatcher; +import com.datastrato.gravitino.catalog.CatalogEventDispatcher; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.CatalogInfo; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestCatalogEvent { + private CatalogEventDispatcher dispatcher; + private CatalogEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Catalog catalog; + + @BeforeAll + void init() { + this.catalog = mockCatalog(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + CatalogDispatcher catalogDispatcher = mockCatalogDispatcher(); + this.dispatcher = new CatalogEventDispatcher(eventBus, catalogDispatcher); + CatalogDispatcher catalogExceptionDispatcher = mockExceptionCatalogDispatcher(); + this.failureDispatcher = new CatalogEventDispatcher(eventBus, catalogExceptionDispatcher); + } + + @Test + void testCreateCatalogEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", catalog.name()); + dispatcher.createCatalog( + identifier, catalog.type(), catalog.provider(), catalog.comment(), catalog.properties()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateCatalogEvent.class, event.getClass()); + CatalogInfo catalogInfo = ((CreateCatalogEvent) event).createdCatalogInfo(); + checkCatalogInfo(catalogInfo, catalog); + } + + @Test + void testLoadCatalogEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", catalog.name()); + dispatcher.loadCatalog(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadCatalogEvent.class, event.getClass()); + CatalogInfo catalogInfo = ((LoadCatalogEvent) event).loadedCatalogInfo(); + checkCatalogInfo(catalogInfo, catalog); + } + + @Test + void testAlterCatalogEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", catalog.name()); + CatalogChange catalogChange = CatalogChange.setProperty("a", "b"); + dispatcher.alterCatalog(identifier, catalogChange); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterCatalogEvent.class, event.getClass()); + CatalogInfo catalogInfo = ((AlterCatalogEvent) event).updatedCatalogInfo(); + checkCatalogInfo(catalogInfo, catalog); + CatalogChange[] catalogChanges = ((AlterCatalogEvent) event).catalogChanges(); + Assertions.assertEquals(1, catalogChanges.length); + Assertions.assertEquals(catalogChange, catalogChanges[0]); + } + + @Test + void testDropCatalogEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", catalog.name()); + dispatcher.dropCatalog(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropCatalogEvent.class, event.getClass()); + Assertions.assertEquals(true, ((DropCatalogEvent) event).isExists()); + } + + @Test + void testListCatalogEvent() { + Namespace namespace = Namespace.of("metalake"); + dispatcher.listCatalogs(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListCatalogEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListCatalogEvent) event).namespace()); + } + + @Test + void testListCatalogInfoEvent() { + Namespace namespace = Namespace.of("metalake"); + dispatcher.listCatalogsInfo(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListCatalogEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListCatalogEvent) event).namespace()); + } + + @Test + void testCreateCatalogFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", catalog.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> + failureDispatcher.createCatalog( + identifier, + catalog.type(), + catalog.provider(), + catalog.comment(), + catalog.properties())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, + ((CreateCatalogFailureEvent) event).exception().getClass()); + checkCatalogInfo(((CreateCatalogFailureEvent) event).createCatalogRequest(), catalog); + } + + @Test + void testLoadCatalogFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadCatalog(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadCatalogFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterCatalogFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog"); + CatalogChange catalogChange = CatalogChange.setProperty("a", "b"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.alterCatalog(identifier, catalogChange)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((AlterCatalogFailureEvent) event).exception().getClass()); + CatalogChange[] catalogChanges = ((AlterCatalogFailureEvent) event).catalogChanges(); + Assertions.assertEquals(1, catalogChanges.length); + Assertions.assertEquals(catalogChange, catalogChanges[0]); + } + + @Test + void testDropCatalogFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropCatalog(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropCatalogFailureEvent) event).exception().getClass()); + } + + @Test + void testListCatalogFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listCatalogs(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(ListCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListCatalogFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListCatalogFailureEvent) event).namespace()); + } + + @Test + void testListCatalogInfoFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listCatalogsInfo(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(ListCatalogFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListCatalogFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListCatalogFailureEvent) event).namespace()); + } + + private void checkCatalogInfo(CatalogInfo catalogInfo, Catalog catalog) { + Assertions.assertEquals(catalog.name(), catalogInfo.name()); + Assertions.assertEquals(catalog.type(), catalogInfo.type()); + Assertions.assertEquals(catalog.provider(), catalogInfo.provider()); + Assertions.assertEquals(catalog.properties(), catalogInfo.properties()); + Assertions.assertEquals(catalog.comment(), catalogInfo.comment()); + } + + private Catalog mockCatalog() { + Catalog catalog = mock(Catalog.class); + when(catalog.comment()).thenReturn("comment"); + when(catalog.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(catalog.name()).thenReturn("catalog"); + when(catalog.provider()).thenReturn("hive"); + when(catalog.type()).thenReturn(Catalog.Type.RELATIONAL); + when(catalog.auditInfo()).thenReturn(null); + return catalog; + } + + private CatalogDispatcher mockCatalogDispatcher() { + CatalogDispatcher dispatcher = mock(CatalogDispatcher.class); + when(dispatcher.createCatalog( + any(NameIdentifier.class), + any(Catalog.Type.class), + any(String.class), + any(String.class), + any(Map.class))) + .thenReturn(catalog); + when(dispatcher.loadCatalog(any(NameIdentifier.class))).thenReturn(catalog); + when(dispatcher.dropCatalog(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.listCatalogs(any(Namespace.class))).thenReturn(null); + when(dispatcher.alterCatalog(any(NameIdentifier.class), any(CatalogChange.class))) + .thenReturn(catalog); + return dispatcher; + } + + private CatalogDispatcher mockExceptionCatalogDispatcher() { + CatalogDispatcher dispatcher = + mock( + CatalogDispatcher.class, + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java new file mode 100644 index 00000000000..7f5a679d5a7 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java @@ -0,0 +1,228 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.FilesetDispatcher; +import com.datastrato.gravitino.catalog.FilesetEventDispatcher; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetChange; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.FilesetInfo; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestFilesetEvent { + private FilesetEventDispatcher dispatcher; + private FilesetEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Fileset fileset; + + @BeforeAll + void init() { + this.fileset = mockFileset(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + FilesetDispatcher filesetDispatcher = mockFilesetDispatcher(); + this.dispatcher = new FilesetEventDispatcher(eventBus, filesetDispatcher); + FilesetDispatcher filesetExceptionDispatcher = mockExceptionFilesetDispatcher(); + this.failureDispatcher = new FilesetEventDispatcher(eventBus, filesetExceptionDispatcher); + } + + @Test + void testCreateFilesetEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); + dispatcher.createFileset( + identifier, + fileset.comment(), + fileset.type(), + fileset.storageLocation(), + fileset.properties()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateFilesetEvent.class, event.getClass()); + FilesetInfo filesetInfo = ((CreateFilesetEvent) event).createdFilesetInfo(); + checkFilesetInfo(filesetInfo, fileset); + } + + @Test + void testLoadFilesetEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); + dispatcher.loadFileset(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadFilesetEvent.class, event.getClass()); + FilesetInfo filesetInfo = ((LoadFilesetEvent) event).loadedFilesetInfo(); + checkFilesetInfo(filesetInfo, fileset); + } + + @Test + void testAlterFilesetEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); + FilesetChange change = FilesetChange.setProperty("a", "b"); + dispatcher.alterFileset(identifier, change); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterFilesetEvent.class, event.getClass()); + FilesetInfo filesetInfo = ((AlterFilesetEvent) event).updatedFilesetInfo(); + checkFilesetInfo(filesetInfo, fileset); + Assertions.assertEquals(1, ((AlterFilesetEvent) event).filesetChanges().length); + Assertions.assertEquals(change, ((AlterFilesetEvent) event).filesetChanges()[0]); + } + + @Test + void testDropFilesetEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); + dispatcher.dropFileset(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropFilesetEvent.class, event.getClass()); + Assertions.assertTrue(((DropFilesetEvent) event).isExists()); + } + + @Test + void testListFilesetEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + dispatcher.listFilesets(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListFilesetEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListFilesetEvent) event).namespace()); + } + + @Test + void testCreateSchemaFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "fileset"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> + failureDispatcher.createFileset( + identifier, + fileset.comment(), + fileset.type(), + fileset.storageLocation(), + fileset.properties())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, + ((CreateFilesetFailureEvent) event).exception().getClass()); + checkFilesetInfo(((CreateFilesetFailureEvent) event).createFilesetRequest(), fileset); + } + + @Test + void testLoadFilesetFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "fileset"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadFileset(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadFilesetFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterFilesetFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "fileset"); + FilesetChange change = FilesetChange.setProperty("a", "b"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.alterFileset(identifier, change)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((AlterFilesetFailureEvent) event).exception().getClass()); + Assertions.assertEquals(1, ((AlterFilesetFailureEvent) event).filesetChanges().length); + Assertions.assertEquals(change, ((AlterFilesetFailureEvent) event).filesetChanges()[0]); + } + + @Test + void testDropFilesetFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "fileset"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropFileset(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropFilesetFailureEvent) event).exception().getClass()); + } + + @Test + void testListFilesetFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listFilesets(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListFilesetFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListFilesetFailureEvent) event).namespace()); + } + + private void checkFilesetInfo(FilesetInfo filesetInfo, Fileset fileset) { + Assertions.assertEquals(fileset.name(), filesetInfo.name()); + Assertions.assertEquals(fileset.type(), filesetInfo.type()); + Assertions.assertEquals(fileset.storageLocation(), filesetInfo.storageLocation()); + Assertions.assertEquals(fileset.properties(), filesetInfo.properties()); + Assertions.assertEquals(fileset.comment(), filesetInfo.comment()); + } + + private Fileset mockFileset() { + Fileset fileset = mock(Fileset.class); + when(fileset.comment()).thenReturn("comment"); + when(fileset.type()).thenReturn(Fileset.Type.MANAGED); + when(fileset.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(fileset.name()).thenReturn("fileset"); + when(fileset.auditInfo()).thenReturn(null); + when(fileset.storageLocation()).thenReturn("location"); + return fileset; + } + + private FilesetDispatcher mockFilesetDispatcher() { + FilesetDispatcher dispatcher = mock(FilesetDispatcher.class); + when(dispatcher.createFileset( + any(NameIdentifier.class), + any(String.class), + any(Fileset.Type.class), + any(String.class), + any(Map.class))) + .thenReturn(fileset); + when(dispatcher.loadFileset(any(NameIdentifier.class))).thenReturn(fileset); + when(dispatcher.dropFileset(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.listFilesets(any(Namespace.class))).thenReturn(null); + when(dispatcher.alterFileset(any(NameIdentifier.class), any(FilesetChange.class))) + .thenReturn(fileset); + return dispatcher; + } + + private FilesetDispatcher mockExceptionFilesetDispatcher() { + FilesetDispatcher dispatcher = + mock( + FilesetDispatcher.class, + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java new file mode 100644 index 00000000000..09fb7aab534 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java @@ -0,0 +1,210 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.Metalake; +import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.MetalakeInfo; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; +import com.datastrato.gravitino.metalake.MetalakeEventDispatcher; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestMetalakeEvent { + private MetalakeEventDispatcher dispatcher; + private MetalakeEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Metalake metalake; + + @BeforeAll + void init() { + this.metalake = mockMetalake(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + MetalakeDispatcher metalakeDispatcher = mockMetalakeDispatcher(); + this.dispatcher = new MetalakeEventDispatcher(eventBus, metalakeDispatcher); + MetalakeDispatcher metalakeExceptionDispatcher = mockExceptionMetalakeDispatcher(); + this.failureDispatcher = new MetalakeEventDispatcher(eventBus, metalakeExceptionDispatcher); + } + + @Test + void testCreateMetalakeEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake"); + dispatcher.createMetalake(identifier, metalake.comment(), metalake.properties()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateMetalakeEvent.class, event.getClass()); + MetalakeInfo metalakeInfo = ((CreateMetalakeEvent) event).createdMetalakeInfo(); + checkMetalakeInfo(metalakeInfo, metalake); + } + + @Test + void testLoadMetalakeEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake"); + dispatcher.loadMetalake(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadMetalakeEvent.class, event.getClass()); + MetalakeInfo metalakeInfo = ((LoadMetalakeEvent) event).loadedMetalakeInfo(); + checkMetalakeInfo(metalakeInfo, metalake); + } + + @Test + void testAlterMetalakeEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake"); + MetalakeChange metalakeChange = MetalakeChange.setProperty("a", "b"); + dispatcher.alterMetalake(identifier, metalakeChange); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterMetalakeEvent.class, event.getClass()); + MetalakeInfo metalakeInfo = ((AlterMetalakeEvent) event).updatedMetalakeInfo(); + checkMetalakeInfo(metalakeInfo, metalake); + MetalakeChange[] metalakeChanges = ((AlterMetalakeEvent) event).metalakeChanges(); + Assertions.assertTrue(metalakeChanges.length == 1); + Assertions.assertEquals(metalakeChange, metalakeChanges[0]); + } + + @Test + void testDropMetalakeEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake"); + dispatcher.dropMetalake(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropMetalakeEvent.class, event.getClass()); + Assertions.assertTrue(((DropMetalakeEvent) event).isExists()); + } + + @Test + void testListMetalakeEvent() { + dispatcher.listMetalakes(); + Event event = dummyEventListener.popEvent(); + Assertions.assertNull(event.identifier()); + Assertions.assertEquals(ListMetalakeEvent.class, event.getClass()); + } + + @Test + void testCreateMetalakeFailureEvent() { + NameIdentifier identifier = NameIdentifier.of(metalake.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> + failureDispatcher.createMetalake( + identifier, metalake.comment(), metalake.properties())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateMetalakeFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, + ((CreateMetalakeFailureEvent) event).exception().getClass()); + checkMetalakeInfo(((CreateMetalakeFailureEvent) event).createMetalakeRequest(), metalake); + } + + @Test + void testLoadMetalakeFailureEvent() { + NameIdentifier identifier = NameIdentifier.of(metalake.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadMetalake(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadMetalakeFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadMetalakeFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterMetalakeFailureEvent() { + NameIdentifier identifier = NameIdentifier.of(metalake.name()); + MetalakeChange metalakeChange = MetalakeChange.setProperty("a", "b"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.alterMetalake(identifier, metalakeChange)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterMetalakeFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, + ((AlterMetalakeFailureEvent) event).exception().getClass()); + Assertions.assertEquals(1, ((AlterMetalakeFailureEvent) event).metalakeChanges().length); + Assertions.assertEquals( + metalakeChange, ((AlterMetalakeFailureEvent) event).metalakeChanges()[0]); + } + + @Test + void testDropMetalakeFailureEvent() { + NameIdentifier identifier = NameIdentifier.of(metalake.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropMetalake(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropMetalakeFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropMetalakeFailureEvent) event).exception().getClass()); + } + + @Test + void testListMetalakeFailureEvent() { + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listMetalakes()); + Event event = dummyEventListener.popEvent(); + Assertions.assertNull(event.identifier()); + Assertions.assertEquals(ListMetalakeFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListMetalakeFailureEvent) event).exception().getClass()); + } + + private void checkMetalakeInfo(MetalakeInfo metalakeInfo, Metalake metalake) { + Assertions.assertEquals(metalake.name(), metalakeInfo.name()); + Assertions.assertEquals(metalake.properties(), metalakeInfo.properties()); + Assertions.assertEquals(metalake.comment(), metalakeInfo.comment()); + Assertions.assertEquals(metalake.auditInfo(), metalakeInfo.auditInfo()); + } + + private Metalake mockMetalake() { + Metalake metalake = mock(Metalake.class); + when(metalake.comment()).thenReturn("comment"); + when(metalake.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(metalake.name()).thenReturn("metalake"); + when(metalake.auditInfo()).thenReturn(null); + return metalake; + } + + private MetalakeDispatcher mockMetalakeDispatcher() { + MetalakeDispatcher dispatcher = mock(MetalakeDispatcher.class); + when(dispatcher.createMetalake(any(NameIdentifier.class), any(String.class), any(Map.class))) + .thenReturn(metalake); + when(dispatcher.loadMetalake(any(NameIdentifier.class))).thenReturn(metalake); + when(dispatcher.dropMetalake(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.listMetalakes()).thenReturn(null); + when(dispatcher.alterMetalake(any(NameIdentifier.class), any(MetalakeChange.class))) + .thenReturn(metalake); + return dispatcher; + } + + private MetalakeDispatcher mockExceptionMetalakeDispatcher() { + MetalakeDispatcher dispatcher = + mock( + MetalakeDispatcher.class, + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java new file mode 100644 index 00000000000..9bd7ef49efa --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java @@ -0,0 +1,215 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.SchemaDispatcher; +import com.datastrato.gravitino.catalog.SchemaEventDispatcher; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.SchemaInfo; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.SchemaChange; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.mockito.stubbing.Answer; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestSchemaEvent { + private SchemaEventDispatcher dispatcher; + private SchemaEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Schema schema; + + @BeforeAll + void init() { + this.schema = mockSchema(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + SchemaDispatcher schemaDispatcher = mockSchemaDispatcher(); + this.dispatcher = new SchemaEventDispatcher(eventBus, schemaDispatcher); + SchemaDispatcher schemaExceptionDispatcher = mockExceptionSchemaDispatcher(); + this.failureDispatcher = new SchemaEventDispatcher(eventBus, schemaExceptionDispatcher); + } + + @Test + void testCreateSchemaEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + dispatcher.createSchema(identifier, "", ImmutableMap.of()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateSchemaEvent.class, event.getClass()); + SchemaInfo schemaInfo = ((CreateSchemaEvent) event).createdSchemaInfo(); + checkSchemaInfo(schemaInfo, schema); + } + + @Test + void testLoadSchemaEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + dispatcher.loadSchema(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadSchemaEvent.class, event.getClass()); + SchemaInfo schemaInfo = ((LoadSchemaEvent) event).loadedSchemaInfo(); + checkSchemaInfo(schemaInfo, schema); + } + + @Test + void testListSchemaEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + dispatcher.listSchemas(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(ListSchemaEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListSchemaEvent) event).namespace()); + } + + @Test + void testAlterSchemaEvent() { + SchemaChange schemaChange = SchemaChange.setProperty("a", "b"); + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + dispatcher.alterSchema(identifier, schemaChange); + Event event = dummyEventListener.popEvent(); + + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterSchemaEvent.class, event.getClass()); + SchemaInfo schemaInfo = ((AlterSchemaEvent) event).updatedSchemaInfo(); + checkSchemaInfo(schemaInfo, schema); + + Assertions.assertEquals(1, ((AlterSchemaEvent) event).schemaChanges().length); + Assertions.assertEquals(schemaChange, ((AlterSchemaEvent) event).schemaChanges()[0]); + } + + @Test + void testDropSchemaEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + dispatcher.dropSchema(identifier, true); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropSchemaEvent.class, event.getClass()); + Assertions.assertEquals(true, ((DropSchemaEvent) event).cascade()); + Assertions.assertEquals(false, ((DropSchemaEvent) event).isExists()); + } + + @Test + void testCreateSchemaFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.createSchema(identifier, schema.comment(), schema.properties())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateSchemaFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((CreateSchemaFailureEvent) event).exception().getClass()); + checkSchemaInfo(((CreateSchemaFailureEvent) event).createSchemaRequest(), schema); + } + + @Test + void testLoadSchemaFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadSchema(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadSchemaFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadSchemaFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterSchemaFailureEvent() { + // alter schema + SchemaChange schemaChange = SchemaChange.setProperty("a", "b"); + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.alterSchema(identifier, schemaChange)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterSchemaFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((AlterSchemaFailureEvent) event).exception().getClass()); + Assertions.assertEquals(1, ((AlterSchemaFailureEvent) event).schemaChanges().length); + Assertions.assertEquals(schemaChange, ((AlterSchemaFailureEvent) event).schemaChanges()[0]); + } + + @Test + void testDropSchemaFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "schema"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropSchema(identifier, true)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropSchemaFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropSchemaFailureEvent) event).exception().getClass()); + Assertions.assertEquals(true, ((DropSchemaFailureEvent) event).cascade()); + } + + @Test + void testListSchemaFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listSchemas(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListSchemaFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListSchemaFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListSchemaFailureEvent) event).namespace()); + } + + private void checkSchemaInfo(SchemaInfo schemaInfo, Schema schema) { + Assertions.assertEquals(schema.name(), schemaInfo.name()); + Assertions.assertEquals(schema.properties(), schemaInfo.properties()); + Assertions.assertEquals(schema.comment(), schemaInfo.comment()); + } + + private Schema mockSchema() { + Schema schema = mock(Schema.class); + when(schema.comment()).thenReturn("comment"); + when(schema.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(schema.name()).thenReturn("schema"); + when(schema.auditInfo()).thenReturn(null); + return schema; + } + + private SchemaDispatcher mockSchemaDispatcher() { + SchemaDispatcher dispatcher = mock(SchemaDispatcher.class); + when(dispatcher.createSchema(any(NameIdentifier.class), any(String.class), any(Map.class))) + .thenReturn(schema); + when(dispatcher.loadSchema(any(NameIdentifier.class))).thenReturn(schema); + when(dispatcher.dropSchema(any(NameIdentifier.class), eq(true))).thenReturn(false); + when(dispatcher.listSchemas(any(Namespace.class))).thenReturn(null); + when(dispatcher.alterSchema(any(NameIdentifier.class), any(SchemaChange.class))) + .thenReturn(schema); + return dispatcher; + } + + private SchemaDispatcher mockExceptionSchemaDispatcher() { + SchemaDispatcher dispatcher = + mock( + SchemaDispatcher.class, + (Answer) + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java new file mode 100644 index 00000000000..572cfa4746f --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java @@ -0,0 +1,281 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.TableDispatcher; +import com.datastrato.gravitino.catalog.TableEventDispatcher; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.TableInfo; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.distributions.Strategy; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestTableEvent { + private TableEventDispatcher dispatcher; + private TableEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Table table; + + @BeforeAll + void init() { + this.table = mockTable(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + TableDispatcher tableDispatcher = mockTableDispatcher(); + this.dispatcher = new TableEventDispatcher(eventBus, tableDispatcher); + TableDispatcher tableExceptionDispatcher = mockExceptionTableDispatcher(); + this.failureDispatcher = new TableEventDispatcher(eventBus, tableExceptionDispatcher); + } + + @Test + void testCreateTableEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", table.name()); + dispatcher.createTable( + identifier, + table.columns(), + table.comment(), + table.properties(), + table.partitioning(), + table.distribution(), + table.sortOrder(), + table.index()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateTableEvent.class, event.getClass()); + TableInfo tableInfo = ((CreateTableEvent) event).createdTableInfo(); + checkTableInfo(tableInfo, table); + } + + @Test + void testLoadTableEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", table.name()); + dispatcher.loadTable(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadTableEvent.class, event.getClass()); + TableInfo tableInfo = ((LoadTableEvent) event).loadedTableInfo(); + checkTableInfo(tableInfo, table); + } + + @Test + void testAlterTableEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", table.name()); + TableChange change = TableChange.setProperty("a", "b"); + dispatcher.alterTable(identifier, change); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterTableEvent.class, event.getClass()); + TableInfo tableInfo = ((AlterTableEvent) event).updatedTableInfo(); + checkTableInfo(tableInfo, table); + Assertions.assertEquals(1, ((AlterTableEvent) event).tableChanges().length); + Assertions.assertEquals(change, ((AlterTableEvent) event).tableChanges()[0]); + } + + @Test + void testDropTableEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", table.name()); + dispatcher.dropTable(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropTableEvent.class, event.getClass()); + Assertions.assertEquals(true, ((DropTableEvent) event).isExists()); + } + + @Test + void testPurgeTableEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", table.name()); + dispatcher.purgeTable(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(PurgeTableEvent.class, event.getClass()); + Assertions.assertEquals(true, ((PurgeTableEvent) event).isExists()); + } + + @Test + void testListTableEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + dispatcher.listTables(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListTableEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListTableEvent) event).namespace()); + } + + @Test + void testCreateTableFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "table", table.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> + failureDispatcher.createTable( + identifier, + table.columns(), + table.comment(), + table.properties(), + table.partitioning(), + table.distribution(), + table.sortOrder(), + table.index())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((CreateTableFailureEvent) event).exception().getClass()); + checkTableInfo(((CreateTableFailureEvent) event).createTableRequest(), table); + } + + @Test + void testLoadTableFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "table", table.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadTable(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadTableFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterTableFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "table", table.name()); + TableChange change = TableChange.setProperty("a", "b"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.alterTable(identifier, change)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((AlterTableFailureEvent) event).exception().getClass()); + Assertions.assertEquals(1, ((AlterTableFailureEvent) event).tableChanges().length); + Assertions.assertEquals(change, ((AlterTableFailureEvent) event).tableChanges()[0]); + } + + @Test + void testDropTableFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "table", table.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropTable(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropTableFailureEvent) event).exception().getClass()); + } + + @Test + void testPurgeTableFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "table", table.name()); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.purgeTable(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(PurgeTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((PurgeTableFailureEvent) event).exception().getClass()); + } + + @Test + void testListTableFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listTables(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListTableFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListTableFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListTableFailureEvent) event).namespace()); + } + + private void checkTableInfo(TableInfo tableInfo, Table table) { + Assertions.assertEquals(table.name(), tableInfo.name()); + Assertions.assertEquals(table.properties(), tableInfo.properties()); + Assertions.assertEquals(table.comment(), tableInfo.comment()); + Assertions.assertArrayEquals(table.columns(), tableInfo.columns()); + Assertions.assertArrayEquals(table.partitioning(), tableInfo.partitioning()); + Assertions.assertEquals(table.distribution(), tableInfo.distribution()); + Assertions.assertArrayEquals(table.sortOrder(), tableInfo.sortOrder()); + Assertions.assertArrayEquals(table.index(), tableInfo.index()); + Assertions.assertEquals(table.auditInfo(), tableInfo.auditInfo()); + } + + private Table mockTable() { + Table table = mock(Table.class); + when(table.name()).thenReturn("table"); + when(table.comment()).thenReturn("comment"); + when(table.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(table.columns()).thenReturn(new Column[] {Column.of("a", Types.IntegerType.get())}); + when(table.distribution()) + .thenReturn(Distributions.of(Strategy.HASH, 10, NamedReference.field("a"))); + when(table.index()) + .thenReturn(new Index[] {Indexes.primary("p", new String[][] {{"a"}, {"b"}})}); + when(table.sortOrder()) + .thenReturn(new SortOrder[] {SortOrders.ascending(NamedReference.field("a"))}); + when(table.partitioning()).thenReturn(new Transform[] {Transforms.identity("a")}); + when(table.auditInfo()).thenReturn(null); + return table; + } + + private TableDispatcher mockTableDispatcher() { + TableDispatcher dispatcher = mock(TableDispatcher.class); + when(dispatcher.createTable( + any(NameIdentifier.class), + any(Column[].class), + any(String.class), + any(Map.class), + any(Transform[].class), + any(Distribution.class), + any(SortOrder[].class), + any(Index[].class))) + .thenReturn(table); + when(dispatcher.loadTable(any(NameIdentifier.class))).thenReturn(table); + when(dispatcher.dropTable(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.purgeTable(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.listTables(any(Namespace.class))).thenReturn(null); + when(dispatcher.alterTable(any(NameIdentifier.class), any(TableChange.class))) + .thenReturn(table); + return dispatcher; + } + + private TableDispatcher mockExceptionTableDispatcher() { + TableDispatcher dispatcher = + mock( + TableDispatcher.class, + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java new file mode 100644 index 00000000000..557c52a45eb --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java @@ -0,0 +1,210 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.listener.api.event; + +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.TopicDispatcher; +import com.datastrato.gravitino.catalog.TopicEventDispatcher; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.DummyEventListener; +import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.api.info.TopicInfo; +import com.datastrato.gravitino.messaging.Topic; +import com.datastrato.gravitino.messaging.TopicChange; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class TestTopicEvent { + private TopicEventDispatcher dispatcher; + private TopicEventDispatcher failureDispatcher; + private DummyEventListener dummyEventListener; + private Topic topic; + + @BeforeAll + void init() { + this.topic = mockTopic(); + this.dummyEventListener = new DummyEventListener(); + EventBus eventBus = new EventBus(Arrays.asList(dummyEventListener)); + TopicDispatcher topicDispatcher = mockTopicDispatcher(); + this.dispatcher = new TopicEventDispatcher(eventBus, topicDispatcher); + TopicDispatcher topicExceptionDispatcher = mockExceptionTopicDispatcher(); + this.failureDispatcher = new TopicEventDispatcher(eventBus, topicExceptionDispatcher); + } + + @Test + void testCreateTopicEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + dispatcher.createTopic(identifier, topic.comment(), null, topic.properties()); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateTopicEvent.class, event.getClass()); + TopicInfo topicInfo = ((CreateTopicEvent) event).createdTopicInfo(); + checkTopicInfo(topicInfo, topic); + } + + @Test + void testLoadTopicEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + dispatcher.loadTopic(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadTopicEvent.class, event.getClass()); + TopicInfo topicInfo = ((LoadTopicEvent) event).loadedTopicInfo(); + checkTopicInfo(topicInfo, topic); + } + + @Test + void testAlterTopicEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + TopicChange topicChange = TopicChange.setProperty("a", "b"); + dispatcher.alterTopic(identifier, topicChange); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterTopicEvent.class, event.getClass()); + TopicInfo topicInfo = ((AlterTopicEvent) event).updatedTopicInfo(); + checkTopicInfo(topicInfo, topic); + Assertions.assertEquals(1, ((AlterTopicEvent) event).topicChanges().length); + Assertions.assertEquals(topicChange, ((AlterTopicEvent) event).topicChanges()[0]); + } + + @Test + void testDropTopicEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + dispatcher.dropTopic(identifier); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropTopicEvent.class, event.getClass()); + Assertions.assertEquals(true, ((DropTopicEvent) event).isExists()); + } + + @Test + void testListTopicEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + dispatcher.listTopics(namespace); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListTopicEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListTopicEvent) event).namespace()); + } + + @Test + void testCreateTopicFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.createTopic(identifier, topic.comment(), null, topic.properties())); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateTopicFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((CreateTopicFailureEvent) event).exception().getClass()); + checkTopicInfo(((CreateTopicFailureEvent) event).createTopicRequest(), topic); + } + + @Test + void testLoadTopicFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.loadTopic(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadTopicFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((LoadTopicFailureEvent) event).exception().getClass()); + } + + @Test + void testAlterTopicFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + TopicChange topicChange = TopicChange.setProperty("a", "b"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.alterTopic(identifier, topicChange)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterTopicFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((AlterTopicFailureEvent) event).exception().getClass()); + Assertions.assertEquals(1, ((AlterTopicFailureEvent) event).topicChanges().length); + Assertions.assertEquals(topicChange, ((AlterTopicFailureEvent) event).topicChanges()[0]); + } + + @Test + void testDropTopicFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "topic"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.dropTopic(identifier)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropTopicFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((DropTopicFailureEvent) event).exception().getClass()); + } + + @Test + void testListTopicFailureEvent() { + Namespace namespace = Namespace.of("metalake", "catalog"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, () -> failureDispatcher.listTopics(namespace)); + Event event = dummyEventListener.popEvent(); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListTopicFailureEvent.class, event.getClass()); + Assertions.assertEquals( + GravitinoRuntimeException.class, ((ListTopicFailureEvent) event).exception().getClass()); + Assertions.assertEquals(namespace, ((ListTopicFailureEvent) event).namespace()); + } + + private void checkTopicInfo(TopicInfo topicInfo, Topic topic) { + Assertions.assertEquals(topic.name(), topicInfo.name()); + Assertions.assertEquals(topic.properties(), topicInfo.properties()); + Assertions.assertEquals(topic.comment(), topicInfo.comment()); + } + + private Topic mockTopic() { + Topic topic = mock(Topic.class); + when(topic.comment()).thenReturn("comment"); + when(topic.properties()).thenReturn(ImmutableMap.of("a", "b")); + when(topic.name()).thenReturn("topic"); + when(topic.auditInfo()).thenReturn(null); + return topic; + } + + private TopicDispatcher mockTopicDispatcher() { + TopicDispatcher dispatcher = mock(TopicDispatcher.class); + when(dispatcher.createTopic( + any(NameIdentifier.class), any(String.class), isNull(), any(Map.class))) + .thenReturn(topic); + when(dispatcher.loadTopic(any(NameIdentifier.class))).thenReturn(topic); + when(dispatcher.dropTopic(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.listTopics(any(Namespace.class))).thenReturn(null); + when(dispatcher.alterTopic(any(NameIdentifier.class), any(TopicChange.class))) + .thenReturn(topic); + return dispatcher; + } + + private TopicDispatcher mockExceptionTopicDispatcher() { + TopicDispatcher dispatcher = + mock( + TopicDispatcher.class, + invocation -> { + throw new GravitinoRuntimeException("Exception for all methods"); + }); + return dispatcher; + } +} From 629c0d373ed651ce52d5482316f793f94250a196 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Thu, 18 Apr 2024 20:41:43 +0800 Subject: [PATCH 058/106] [#2246] feat(client): Add the client for user, group,admin and role (#3000) ### What changes were proposed in this pull request? The client supports to operate the user, group, admin and role ### Why are the changes needed? Fix: #2246 ### Does this PR introduce _any_ user-facing change? Yes, I will add the document later. ### How was this patch tested? Add new uts. --------- Co-authored-by: Heng Qin --- .../gravitino/client/ErrorHandlers.java | 138 ++++++++ .../client/GravitinoAdminClient.java | 300 ++++++++++++++++++ .../gravitino/client/TestMetalakeAdmin.java | 96 ++++++ .../datastrato/gravitino/client/TestRole.java | 187 +++++++++++ .../gravitino/client/TestUserGroup.java | 266 ++++++++++++++++ .../dto/responses/DeleteResponse.java | 43 +++ .../authorization/AccessControlManager.java | 79 +++-- .../authorization/PermissionManager.java | 8 +- .../gravitino/authorization/RoleManager.java | 4 +- .../TestAccessControlManager.java | 11 +- ...estAccessControlManagerForPermissions.java | 10 +- .../server/web/rest/OperationType.java | 3 +- .../server/web/rest/RoleOperations.java | 31 +- .../server/web/rest/TestRoleOperations.java | 34 +- 14 files changed, 1130 insertions(+), 80 deletions(-) create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java index a18d3febcdb..d140b0c2a70 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java @@ -10,22 +10,28 @@ import com.datastrato.gravitino.exceptions.BadRequestException; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; +import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchFilesetException; +import com.datastrato.gravitino.exceptions.NoSuchGroupException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchPartitionException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.NoSuchTopicException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.NotFoundException; import com.datastrato.gravitino.exceptions.PartitionAlreadyExistsException; import com.datastrato.gravitino.exceptions.RESTException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; import com.datastrato.gravitino.exceptions.UnauthorizedException; +import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import java.util.List; @@ -123,6 +129,33 @@ public static Consumer topicErrorHandler() { return TopicErrorHandler.INSTANCE; } + /** + * Creates an error handler specific to User operations. + * + * @return A Consumer representing the User error handler. + */ + public static Consumer userErrorHandler() { + return UserErrorHandler.INSTANCE; + } + + /** + * Creates an error handler specific to Group operations. + * + * @return A Consumer representing the Group error handler. + */ + public static Consumer groupErrorHandler() { + return GroupErrorHandler.INSTANCE; + } + + /** + * Creates an error handler specific to Role operations. + * + * @return A Consumer representing the Role error handler. + */ + public static Consumer roleErrorHandler() { + return RoleErrorHandler.INSTANCE; + } + private ErrorHandlers() {} /** @@ -459,6 +492,111 @@ public void accept(ErrorResponse errorResponse) { } } + /** Error handler specific to User operations. */ + @SuppressWarnings("FormatStringAnnotation") + private static class UserErrorHandler extends RestErrorHandler { + + private static final UserErrorHandler INSTANCE = new UserErrorHandler(); + + @Override + public void accept(ErrorResponse errorResponse) { + String errorMessage = formatErrorMessage(errorResponse); + + switch (errorResponse.getCode()) { + case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: + throw new IllegalArgumentException(errorMessage); + + case ErrorConstants.NOT_FOUND_CODE: + if (errorResponse.getType().equals(NoSuchMetalakeException.class.getSimpleName())) { + throw new NoSuchMetalakeException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchUserException.class.getSimpleName())) { + throw new NoSuchUserException(errorMessage); + } else { + throw new NotFoundException(errorMessage); + } + + case ErrorConstants.ALREADY_EXISTS_CODE: + throw new UserAlreadyExistsException(errorMessage); + + case ErrorConstants.INTERNAL_ERROR_CODE: + throw new RuntimeException(errorMessage); + + default: + super.accept(errorResponse); + } + } + } + + /** Error handler specific to Group operations. */ + @SuppressWarnings("FormatStringAnnotation") + private static class GroupErrorHandler extends RestErrorHandler { + + private static final GroupErrorHandler INSTANCE = new GroupErrorHandler(); + + @Override + public void accept(ErrorResponse errorResponse) { + String errorMessage = formatErrorMessage(errorResponse); + + switch (errorResponse.getCode()) { + case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: + throw new IllegalArgumentException(errorMessage); + + case ErrorConstants.NOT_FOUND_CODE: + if (errorResponse.getType().equals(NoSuchMetalakeException.class.getSimpleName())) { + throw new NoSuchMetalakeException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchGroupException.class.getSimpleName())) { + throw new NoSuchGroupException(errorMessage); + } else { + throw new NotFoundException(errorMessage); + } + + case ErrorConstants.ALREADY_EXISTS_CODE: + throw new GroupAlreadyExistsException(errorMessage); + + case ErrorConstants.INTERNAL_ERROR_CODE: + throw new RuntimeException(errorMessage); + + default: + super.accept(errorResponse); + } + } + } + + /** Error handler specific to Role operations. */ + @SuppressWarnings("FormatStringAnnotation") + private static class RoleErrorHandler extends RestErrorHandler { + + private static final RoleErrorHandler INSTANCE = new RoleErrorHandler(); + + @Override + public void accept(ErrorResponse errorResponse) { + String errorMessage = formatErrorMessage(errorResponse); + + switch (errorResponse.getCode()) { + case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: + throw new IllegalArgumentException(errorMessage); + + case ErrorConstants.NOT_FOUND_CODE: + if (errorResponse.getType().equals(NoSuchMetalakeException.class.getSimpleName())) { + throw new NoSuchMetalakeException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchRoleException.class.getSimpleName())) { + throw new NoSuchRoleException(errorMessage); + } else { + throw new NotFoundException(errorMessage); + } + + case ErrorConstants.ALREADY_EXISTS_CODE: + throw new RoleAlreadyExistsException(errorMessage); + + case ErrorConstants.INTERNAL_ERROR_CODE: + throw new RuntimeException(errorMessage); + + default: + super.accept(errorResponse); + } + } + } + /** Generic error handler for REST requests. */ private static class RestErrorHandler extends ErrorHandler { private static final ErrorHandler INSTANCE = new RestErrorHandler(); diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index 797991c7dde..07675037f2f 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -8,19 +8,39 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.SupportsMetalakes; +import com.datastrato.gravitino.authorization.Group; +import com.datastrato.gravitino.authorization.Privilege; +import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObject; +import com.datastrato.gravitino.authorization.User; +import com.datastrato.gravitino.dto.requests.GroupAddRequest; import com.datastrato.gravitino.dto.requests.MetalakeCreateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdatesRequest; +import com.datastrato.gravitino.dto.requests.RoleCreateRequest; +import com.datastrato.gravitino.dto.requests.UserAddRequest; +import com.datastrato.gravitino.dto.responses.DeleteResponse; import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.GroupResponse; import com.datastrato.gravitino.dto.responses.MetalakeListResponse; import com.datastrato.gravitino.dto.responses.MetalakeResponse; +import com.datastrato.gravitino.dto.responses.RemoveResponse; +import com.datastrato.gravitino.dto.responses.RoleResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchGroupException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +54,10 @@ public class GravitinoAdminClient extends GravitinoClientBase implements SupportsMetalakes { private static final Logger LOG = LoggerFactory.getLogger(GravitinoAdminClient.class); + private static final String API_METALAKES_USERS_PATH = "api/metalakes/%s/users/%s"; + private static final String API_METALAKES_GROUPS_PATH = "api/metalakes/%s/groups/%s"; + private static final String API_METALAKES_ROLES_PATH = "api/metalakes/%s/roles/%s"; + private static final String API_ADMIN_PATH = "api/admins/%s"; /** * Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. @@ -162,6 +186,282 @@ public boolean dropMetalake(NameIdentifier ident) { } } + /** + * Adds a new User. + * + * @param metalake The Metalake of the User. + * @param user The name of the User. + * @return The added User instance. + * @throws UserAlreadyExistsException If a User with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If adding the User encounters storage issues. + */ + public User addUser(String metalake, String user) + throws UserAlreadyExistsException, NoSuchMetalakeException { + UserAddRequest req = new UserAddRequest(user); + req.validate(); + + UserResponse resp = + restClient.post( + String.format(API_METALAKES_USERS_PATH, metalake, ""), + req, + UserResponse.class, + Collections.emptyMap(), + ErrorHandlers.userErrorHandler()); + resp.validate(); + + return resp.getUser(); + } + + /** + * Removes a User. + * + * @param metalake The Metalake of the User. + * @param user The name of the User. + * @return `true` if the User was successfully removed, `false` only when there's no such user, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If removing the User encounters storage issues. + */ + public boolean removeUser(String metalake, String user) throws NoSuchMetalakeException { + RemoveResponse resp = + restClient.delete( + String.format(API_METALAKES_USERS_PATH, metalake, user), + RemoveResponse.class, + Collections.emptyMap(), + ErrorHandlers.userErrorHandler()); + resp.validate(); + + return resp.removed(); + } + + /** + * Gets a User. + * + * @param metalake The Metalake of the User. + * @param user The name of the User. + * @return The getting User instance. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If getting the User encounters storage issues. + */ + public User getUser(String metalake, String user) + throws NoSuchUserException, NoSuchMetalakeException { + UserResponse resp = + restClient.get( + String.format(API_METALAKES_USERS_PATH, metalake, user), + UserResponse.class, + Collections.emptyMap(), + ErrorHandlers.userErrorHandler()); + resp.validate(); + + return resp.getUser(); + } + + /** + * Adds a new Group. + * + * @param metalake The Metalake of the Group. + * @param group The name of the Group. + * @return The Added Group instance. + * @throws GroupAlreadyExistsException If a Group with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If adding the Group encounters storage issues. + */ + public Group addGroup(String metalake, String group) + throws GroupAlreadyExistsException, NoSuchMetalakeException { + GroupAddRequest req = new GroupAddRequest(group); + req.validate(); + + GroupResponse resp = + restClient.post( + String.format(API_METALAKES_GROUPS_PATH, metalake, ""), + req, + GroupResponse.class, + Collections.emptyMap(), + ErrorHandlers.groupErrorHandler()); + resp.validate(); + + return resp.getGroup(); + } + + /** + * Removes a Group. + * + * @param metalake The Metalake of the Group. + * @param group THe name of the Group. + * @return `true` if the Group was successfully removed, `false` only when there's no such group, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If removing the Group encounters storage issues. + */ + public boolean removeGroup(String metalake, String group) throws NoSuchMetalakeException { + RemoveResponse resp = + restClient.delete( + String.format(API_METALAKES_GROUPS_PATH, metalake, group), + RemoveResponse.class, + Collections.emptyMap(), + ErrorHandlers.groupErrorHandler()); + resp.validate(); + + return resp.removed(); + } + + /** + * Gets a Group. + * + * @param metalake The Metalake of the Group. + * @param group The name of the Group. + * @return The getting Group instance. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If getting the Group encounters storage issues. + */ + public Group getGroup(String metalake, String group) + throws NoSuchGroupException, NoSuchMetalakeException { + GroupResponse resp = + restClient.get( + String.format(API_METALAKES_GROUPS_PATH, metalake, group), + GroupResponse.class, + Collections.emptyMap(), + ErrorHandlers.groupErrorHandler()); + resp.validate(); + + return resp.getGroup(); + } + + /** + * Adds a new metalake admin. + * + * @param user The name of the User. + * @return The added User instance. + * @throws UserAlreadyExistsException If a metalake admin with the same name already exists. + * @throws RuntimeException If adding the User encounters storage issues. + */ + public User addMetalakeAdmin(String user) throws UserAlreadyExistsException { + UserAddRequest req = new UserAddRequest(user); + req.validate(); + + UserResponse resp = + restClient.post( + String.format(API_ADMIN_PATH, ""), + req, + UserResponse.class, + Collections.emptyMap(), + ErrorHandlers.userErrorHandler()); + resp.validate(); + + return resp.getUser(); + } + + /** + * Removes a metalake admin. + * + * @param user The name of the User. + * @return `true` if the User was successfully removed, `false` only when there's no such metalake + * admin, otherwise it will throw an exception. + * @throws RuntimeException If removing the User encounters storage issues. + */ + public boolean removeMetalakeAdmin(String user) { + RemoveResponse resp = + restClient.delete( + String.format(API_ADMIN_PATH, user), + RemoveResponse.class, + Collections.emptyMap(), + ErrorHandlers.userErrorHandler()); + resp.validate(); + + return resp.removed(); + } + + /** + * Gets a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return The getting Role instance. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If getting the Role encounters storage issues. + */ + public Role getRole(String metalake, String role) + throws NoSuchRoleException, NoSuchMetalakeException { + RoleResponse resp = + restClient.get( + String.format(API_METALAKES_ROLES_PATH, metalake, role), + RoleResponse.class, + Collections.emptyMap(), + ErrorHandlers.roleErrorHandler()); + resp.validate(); + + return resp.getRole(); + } + + /** + * Deletes a Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @return `true` if the Role was successfully deleted, `false` only when there's no such role, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If deleting the Role encounters storage issues. + */ + public boolean deleteRole(String metalake, String role) throws NoSuchMetalakeException { + DeleteResponse resp = + restClient.delete( + String.format(API_METALAKES_ROLES_PATH, metalake, role), + DeleteResponse.class, + Collections.emptyMap(), + ErrorHandlers.roleErrorHandler()); + resp.validate(); + + return resp.deleted(); + } + + /** + * Creates a new Role. + * + * @param metalake The Metalake of the Role. + * @param role The name of the Role. + * @param properties The properties of the Role. + * @param securableObject The securable object of the Role. + * @param privileges The privileges of the Role. + * @return The created Role instance. + * @throws RoleAlreadyExistsException If a Role with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If creating the Role encounters storage issues. + */ + public Role createRole( + String metalake, + String role, + Map properties, + SecurableObject securableObject, + List privileges) + throws RoleAlreadyExistsException, NoSuchMetalakeException { + RoleCreateRequest req = + new RoleCreateRequest( + role, + properties, + privileges.stream() + .map(Privilege::name) + .map(Objects::toString) + .collect(Collectors.toList()), + securableObject.toString()); + req.validate(); + + RoleResponse resp = + restClient.post( + String.format(API_METALAKES_ROLES_PATH, metalake, ""), + req, + RoleResponse.class, + Collections.emptyMap(), + ErrorHandlers.roleErrorHandler()); + resp.validate(); + + return resp.getRole(); + } + /** * Creates a new builder for constructing a GravitinoClient. * diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java new file mode 100644 index 00000000000..66096b7f7b5 --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java @@ -0,0 +1,96 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; + +import com.datastrato.gravitino.authorization.User; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.authorization.UserDTO; +import com.datastrato.gravitino.dto.requests.UserAddRequest; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.RemoveResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; +import java.time.Instant; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestMetalakeAdmin extends TestBase { + + private static final String API_ADMINS_PATH = "api/admins/%s"; + + @BeforeAll + public static void setUp() throws Exception { + TestBase.setUp(); + } + + @Test + public void testAddMetalakeAdmin() throws Exception { + String username = "user"; + String userPath = withSlash(String.format(API_ADMINS_PATH, "")); + UserAddRequest request = new UserAddRequest(username); + + UserDTO mockUser = mockUserDTO(username); + UserResponse userResponse = new UserResponse(mockUser); + buildMockResource(Method.POST, userPath, request, userResponse, SC_OK); + + User addedUser = client.addMetalakeAdmin(username); + Assertions.assertNotNull(addedUser); + assertUser(addedUser, mockUser); + + // test UserAlreadyExistsException + ErrorResponse errResp1 = + ErrorResponse.alreadyExists( + UserAlreadyExistsException.class.getSimpleName(), "user already exists"); + buildMockResource(Method.POST, userPath, request, errResp1, SC_CONFLICT); + Exception ex = + Assertions.assertThrows( + UserAlreadyExistsException.class, () -> client.addMetalakeAdmin(username)); + Assertions.assertEquals("user already exists", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.POST, userPath, request, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.addMetalakeAdmin(username), "internal error"); + } + + @Test + public void testRemoveMetalakeAdmin() throws Exception { + String username = "user"; + String rolePath = withSlash(String.format(API_ADMINS_PATH, username)); + + RemoveResponse removeResponse = new RemoveResponse(true); + buildMockResource(Method.DELETE, rolePath, null, removeResponse, SC_OK); + + Assertions.assertTrue(client.removeMetalakeAdmin(username)); + + removeResponse = new RemoveResponse(false); + buildMockResource(Method.DELETE, rolePath, null, removeResponse, SC_OK); + Assertions.assertFalse(client.removeMetalakeAdmin(username)); + + // test RuntimeException + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.DELETE, rolePath, null, errResp, SC_SERVER_ERROR); + Assertions.assertThrows(RuntimeException.class, () -> client.removeMetalakeAdmin(username)); + } + + private UserDTO mockUserDTO(String name) { + return UserDTO.builder() + .withName(name) + .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + private void assertUser(User expected, User actual) { + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(expected.roles(), actual.roles()); + } +} diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java new file mode 100644 index 00000000000..560bb52e498 --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java @@ -0,0 +1,187 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; + +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.authorization.RoleDTO; +import com.datastrato.gravitino.dto.requests.RoleCreateRequest; +import com.datastrato.gravitino.dto.responses.DeleteResponse; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.RoleResponse; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.time.Instant; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestRole extends TestBase { + + private static final String API_METALAKES_ROLES_PATH = "api/metalakes/%s/roles/%s"; + protected static final String metalakeName = "testMetalake"; + + @BeforeAll + public static void setUp() throws Exception { + TestBase.setUp(); + } + + @Test + public void testCreateRoles() throws Exception { + String roleName = "role"; + String rolePath = withSlash(String.format(API_METALAKES_ROLES_PATH, metalakeName, "")); + RoleCreateRequest request = + new RoleCreateRequest( + roleName, ImmutableMap.of("k1", "v1"), Lists.newArrayList("LOAD_CATALOG"), "catalog"); + + RoleDTO mockRole = mockRoleDTO(roleName); + RoleResponse roleResponse = new RoleResponse(mockRole); + buildMockResource(Method.POST, rolePath, request, roleResponse, SC_OK); + + Role createdRole = + client.createRole( + metalakeName, + roleName, + ImmutableMap.of("k1", "v1"), + SecurableObjects.ofCatalog("catalog"), + Lists.newArrayList(Privileges.LoadCatalog.get())); + Assertions.assertNotNull(createdRole); + assertRole(createdRole, mockRole); + + // test RoleAlreadyExistsException + ErrorResponse errResp1 = + ErrorResponse.alreadyExists( + RoleAlreadyExistsException.class.getSimpleName(), "role already exists"); + buildMockResource(Method.POST, rolePath, request, errResp1, SC_CONFLICT); + Exception ex = + Assertions.assertThrows( + RoleAlreadyExistsException.class, + () -> + client.createRole( + metalakeName, + roleName, + ImmutableMap.of("k1", "v1"), + SecurableObjects.ofCatalog("catalog"), + Lists.newArrayList(Privileges.LoadCatalog.get()))); + Assertions.assertEquals("role already exists", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.POST, rolePath, request, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> + client.createRole( + metalakeName, + roleName, + ImmutableMap.of("k1", "v1"), + SecurableObjects.ofCatalog("catalog"), + Lists.newArrayList(Privileges.LoadCatalog.get()))); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.POST, rolePath, request, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, + () -> + client.createRole( + metalakeName, + roleName, + ImmutableMap.of("k1", "v1"), + SecurableObjects.ofCatalog("catalog"), + Lists.newArrayList(Privileges.LoadCatalog.get())), + "internal error"); + } + + @Test + public void testGetRoles() throws Exception { + String roleName = "role"; + String rolePath = withSlash(String.format(API_METALAKES_ROLES_PATH, metalakeName, roleName)); + + RoleDTO mockRole = mockRoleDTO(roleName); + RoleResponse roleResponse = new RoleResponse(mockRole); + buildMockResource(Method.GET, rolePath, null, roleResponse, SC_OK); + + Role loadedRole = client.getRole(metalakeName, roleName); + Assertions.assertNotNull(loadedRole); + assertRole(mockRole, loadedRole); + + // test NoSuchRoleException + ErrorResponse errResp1 = + ErrorResponse.notFound(NoSuchRoleException.class.getSimpleName(), "role not found"); + buildMockResource(Method.GET, rolePath, null, errResp1, SC_NOT_FOUND); + Exception ex = + Assertions.assertThrows( + NoSuchRoleException.class, () -> client.getRole(metalakeName, roleName)); + Assertions.assertEquals("role not found", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.GET, rolePath, null, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, () -> client.getRole(metalakeName, roleName)); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.GET, rolePath, null, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.getRole(metalakeName, roleName), "internal error"); + } + + @Test + public void testDeleteRoles() throws Exception { + String roleName = "role"; + String rolePath = withSlash(String.format(API_METALAKES_ROLES_PATH, metalakeName, roleName)); + + DeleteResponse deleteResponse = new DeleteResponse(true); + buildMockResource(Method.DELETE, rolePath, null, deleteResponse, SC_OK); + + Assertions.assertTrue(client.deleteRole(metalakeName, roleName)); + + deleteResponse = new DeleteResponse(false); + buildMockResource(Method.DELETE, rolePath, null, deleteResponse, SC_OK); + Assertions.assertFalse(client.deleteRole(metalakeName, roleName)); + + // test RuntimeException + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.DELETE, rolePath, null, errResp, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.deleteRole(metalakeName, roleName)); + } + + private RoleDTO mockRoleDTO(String name) { + return RoleDTO.builder() + .withName(name) + .withProperties(ImmutableMap.of("k1", "v1")) + .withSecurableObject(SecurableObjects.of("catalog")) + .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + private void assertRole(Role expected, Role actual) { + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(expected.privileges(), actual.privileges()); + Assertions.assertEquals( + expected.securableObject().toString(), actual.securableObject().toString()); + } +} diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java new file mode 100644 index 00000000000..beb37fdf81e --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java @@ -0,0 +1,266 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; + +import com.datastrato.gravitino.authorization.Group; +import com.datastrato.gravitino.authorization.User; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.authorization.GroupDTO; +import com.datastrato.gravitino.dto.authorization.UserDTO; +import com.datastrato.gravitino.dto.requests.GroupAddRequest; +import com.datastrato.gravitino.dto.requests.UserAddRequest; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.GroupResponse; +import com.datastrato.gravitino.dto.responses.RemoveResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchGroupException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; +import java.time.Instant; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestUserGroup extends TestBase { + + private static final String API_METALAKES_USERS_PATH = "api/metalakes/%s/users/%s"; + private static final String API_METALAKES_GROUPS_PATH = "api/metalakes/%s/groups/%s"; + protected static final String metalakeName = "testMetalake"; + + @BeforeAll + public static void setUp() throws Exception { + TestBase.setUp(); + } + + @Test + public void testAddUsers() throws Exception { + String username = "user"; + String userPath = withSlash(String.format(API_METALAKES_USERS_PATH, metalakeName, "")); + UserAddRequest request = new UserAddRequest(username); + + UserDTO mockUser = mockUserDTO(username); + UserResponse userResponse = new UserResponse(mockUser); + buildMockResource(Method.POST, userPath, request, userResponse, SC_OK); + + User addedUser = client.addUser(metalakeName, username); + Assertions.assertNotNull(addedUser); + assertUser(addedUser, mockUser); + + // test UserAlreadyExistsException + ErrorResponse errResp1 = + ErrorResponse.alreadyExists( + UserAlreadyExistsException.class.getSimpleName(), "user already exists"); + buildMockResource(Method.POST, userPath, request, errResp1, SC_CONFLICT); + Exception ex = + Assertions.assertThrows( + UserAlreadyExistsException.class, () -> client.addUser(metalakeName, username)); + Assertions.assertEquals("user already exists", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.POST, userPath, request, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, () -> client.addUser(metalakeName, username)); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.POST, userPath, request, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.addUser(metalakeName, username), "internal error"); + } + + @Test + public void testGetUsers() throws Exception { + String username = "user"; + String userPath = withSlash(String.format(API_METALAKES_USERS_PATH, metalakeName, username)); + + UserDTO mockUser = mockUserDTO(username); + UserResponse userResponse = new UserResponse(mockUser); + buildMockResource(Method.GET, userPath, null, userResponse, SC_OK); + + User loadedUser = client.getUser(metalakeName, username); + Assertions.assertNotNull(loadedUser); + assertUser(mockUser, loadedUser); + + // test NoSuchUserException + ErrorResponse errResp1 = + ErrorResponse.notFound(NoSuchUserException.class.getSimpleName(), "user not found"); + buildMockResource(Method.GET, userPath, null, errResp1, SC_NOT_FOUND); + Exception ex = + Assertions.assertThrows( + NoSuchUserException.class, () -> client.getUser(metalakeName, username)); + Assertions.assertEquals("user not found", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.GET, userPath, null, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, () -> client.getUser(metalakeName, username)); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.GET, userPath, null, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.getUser(metalakeName, username), "internal error"); + } + + @Test + public void testRemoveUsers() throws Exception { + String username = "user"; + String userPath = withSlash(String.format(API_METALAKES_USERS_PATH, metalakeName, username)); + + RemoveResponse removeResponse = new RemoveResponse(true); + buildMockResource(Method.DELETE, userPath, null, removeResponse, SC_OK); + + Assertions.assertTrue(client.removeUser(metalakeName, username)); + + removeResponse = new RemoveResponse(false); + buildMockResource(Method.DELETE, userPath, null, removeResponse, SC_OK); + Assertions.assertFalse(client.removeUser(metalakeName, username)); + + // test RuntimeException + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.DELETE, userPath, null, errResp, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.removeUser(metalakeName, username)); + } + + @Test + public void testAddGroups() throws Exception { + String groupName = "group"; + String groupPath = withSlash(String.format(API_METALAKES_GROUPS_PATH, metalakeName, "")); + GroupAddRequest request = new GroupAddRequest(groupName); + + GroupDTO mockGroup = mockGroupDTO(groupName); + GroupResponse groupResponse = new GroupResponse(mockGroup); + buildMockResource(Method.POST, groupPath, request, groupResponse, SC_OK); + + Group addedGroup = client.addGroup(metalakeName, groupName); + Assertions.assertNotNull(addedGroup); + assertGroup(addedGroup, mockGroup); + + // test GroupAlreadyExistsException + ErrorResponse errResp1 = + ErrorResponse.alreadyExists( + GroupAlreadyExistsException.class.getSimpleName(), "group already exists"); + buildMockResource(Method.POST, groupPath, request, errResp1, SC_CONFLICT); + Exception ex = + Assertions.assertThrows( + GroupAlreadyExistsException.class, () -> client.addGroup(metalakeName, groupName)); + Assertions.assertEquals("group already exists", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.POST, groupPath, request, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, () -> client.addGroup(metalakeName, groupName)); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.POST, groupPath, request, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.addGroup(metalakeName, groupName), "internal error"); + } + + @Test + public void testGetGroups() throws Exception { + String groupName = "group"; + String groupPath = withSlash(String.format(API_METALAKES_GROUPS_PATH, metalakeName, groupName)); + + GroupDTO mockGroup = mockGroupDTO(groupName); + GroupResponse groupResponse = new GroupResponse(mockGroup); + buildMockResource(Method.GET, groupPath, null, groupResponse, SC_OK); + + Group loadedGroup = client.getGroup(metalakeName, groupName); + Assertions.assertNotNull(loadedGroup); + assertGroup(mockGroup, loadedGroup); + + // test NoSuchGroupException + ErrorResponse errResp1 = + ErrorResponse.notFound(NoSuchGroupException.class.getSimpleName(), "group not found"); + buildMockResource(Method.GET, groupPath, null, errResp1, SC_NOT_FOUND); + Exception ex = + Assertions.assertThrows( + NoSuchGroupException.class, () -> client.getGroup(metalakeName, groupName)); + Assertions.assertEquals("group not found", ex.getMessage()); + + // test NoSuchMetalakeException + ErrorResponse errResp2 = + ErrorResponse.notFound(NoSuchMetalakeException.class.getSimpleName(), "metalake not found"); + buildMockResource(Method.GET, groupPath, null, errResp2, SC_NOT_FOUND); + ex = + Assertions.assertThrows( + NoSuchMetalakeException.class, () -> client.getGroup(metalakeName, groupName)); + Assertions.assertEquals("metalake not found", ex.getMessage()); + + // test RuntimeException + ErrorResponse errResp3 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.GET, groupPath, null, errResp3, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.getGroup(metalakeName, groupName), "internal error"); + } + + @Test + public void testRemoveGroups() throws Exception { + String groupName = "user"; + String groupPath = withSlash(String.format(API_METALAKES_GROUPS_PATH, metalakeName, groupName)); + + RemoveResponse removeResponse = new RemoveResponse(true); + buildMockResource(Method.DELETE, groupPath, null, removeResponse, SC_OK); + + Assertions.assertTrue(client.removeGroup(metalakeName, groupName)); + + removeResponse = new RemoveResponse(false); + buildMockResource(Method.DELETE, groupPath, null, removeResponse, SC_OK); + Assertions.assertFalse(client.removeGroup(metalakeName, groupName)); + + // test RuntimeException + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.DELETE, groupPath, null, errResp, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.removeGroup(metalakeName, groupName)); + } + + private UserDTO mockUserDTO(String name) { + return UserDTO.builder() + .withName(name) + .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + private GroupDTO mockGroupDTO(String name) { + return GroupDTO.builder() + .withName(name) + .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + private void assertUser(User expected, User actual) { + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(expected.roles(), actual.roles()); + } + + private void assertGroup(Group expected, Group actual) { + Assertions.assertEquals(expected.name(), actual.name()); + Assertions.assertEquals(expected.roles(), actual.roles()); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java new file mode 100644 index 00000000000..a524520f256 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** Represents a response for a delete operation. */ +@ToString +@EqualsAndHashCode(callSuper = true) +public class DeleteResponse extends BaseResponse { + + @JsonProperty("deleted") + private final boolean deleted; + + /** + * Constructor for DeleteResponse. + * + * @param deleted Whether the delete operation was successful. + */ + public DeleteResponse(boolean deleted) { + super(0); + this.deleted = deleted; + } + + /** Default constructor for DeleteResponse (used by Jackson deserializer). */ + public DeleteResponse() { + super(); + this.deleted = false; + } + + /** + * Returns whether the delete operation was successful. + * + * @return True if the delete operation was successful, otherwise false. + */ + public boolean deleted() { + return deleted; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index 131153f4512..35c3ea99f47 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.exceptions.GroupAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.NoSuchUserException; import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; @@ -46,13 +47,15 @@ public AccessControlManager(EntityStore store, IdGenerator idGenerator, Config c * Adds a new User. * * @param metalake The Metalake of the User. - * @param name The name of the User. + * @param user The name of the User. * @return The added User instance. - * @throws UserAlreadyExistsException If a User with the same identifier already exists. + * @throws UserAlreadyExistsException If a User with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If adding the User encounters storage issues. */ - public User addUser(String metalake, String name) throws UserAlreadyExistsException { - return doWithNonAdminLock(() -> userGroupManager.addUser(metalake, name)); + public User addUser(String metalake, String user) + throws UserAlreadyExistsException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> userGroupManager.addUser(metalake, user)); } /** @@ -60,10 +63,12 @@ public User addUser(String metalake, String name) throws UserAlreadyExistsExcept * * @param metalake The Metalake of the User. * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` otherwise. + * @return `true` if the User was successfully removed, `false` only when there's no such user, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the User encounters storage issues. */ - public boolean removeUser(String metalake, String user) { + public boolean removeUser(String metalake, String user) throws NoSuchMetalakeException { return doWithNonAdminLock(() -> userGroupManager.removeUser(metalake, user)); } @@ -73,10 +78,12 @@ public boolean removeUser(String metalake, String user) { * @param metalake The Metalake of the User. * @param user The name of the User. * @return The getting User instance. - * @throws NoSuchUserException If the User with the given identifier does not exist. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If getting the User encounters storage issues. */ - public User getUser(String metalake, String user) throws NoSuchUserException { + public User getUser(String metalake, String user) + throws NoSuchUserException, NoSuchMetalakeException { return doWithNonAdminLock(() -> userGroupManager.getUser(metalake, user)); } @@ -86,10 +93,12 @@ public User getUser(String metalake, String user) throws NoSuchUserException { * @param metalake The Metalake of the Group. * @param group The name of the Group. * @return The Added Group instance. - * @throws GroupAlreadyExistsException If a Group with the same identifier already exists. + * @throws GroupAlreadyExistsException If a Group with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If adding the Group encounters storage issues. */ - public Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { + public Group addGroup(String metalake, String group) + throws GroupAlreadyExistsException, NoSuchMetalakeException { return doWithNonAdminLock(() -> userGroupManager.addGroup(metalake, group)); } @@ -98,10 +107,12 @@ public Group addGroup(String metalake, String group) throws GroupAlreadyExistsEx * * @param metalake The Metalake of the Group. * @param group THe name of the Group. - * @return `true` if the Group was successfully removed, `false` otherwise. + * @return `true` if the Group was successfully removed, `false` only when there's no such group, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the Group encounters storage issues. */ - public boolean removeGroup(String metalake, String group) { + public boolean removeGroup(String metalake, String group) throws NoSuchMetalakeException { return doWithNonAdminLock(() -> userGroupManager.removeGroup(metalake, group)); } @@ -111,10 +122,12 @@ public boolean removeGroup(String metalake, String group) { * @param metalake The Metalake of the Group. * @param group The name of the Group. * @return The getting Group instance. - * @throws NoSuchGroupException If the Group with the given identifier does not exist. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If getting the Group encounters storage issues. */ - public Group getGroup(String metalake, String group) throws NoSuchGroupException { + public Group getGroup(String metalake, String group) + throws NoSuchGroupException, NoSuchMetalakeException { return doWithNonAdminLock(() -> userGroupManager.getGroup(metalake, group)); } @@ -187,10 +200,10 @@ public boolean revokeRoleFromUser(String metalake, String role, String user) { * * @param user The name of the User. * @return The added User instance. - * @throws UserAlreadyExistsException If a User with the same identifier already exists. + * @throws UserAlreadyExistsException If a metalake admin with the same name already exists. * @throws RuntimeException If adding the User encounters storage issues. */ - public User addMetalakeAdmin(String user) { + public User addMetalakeAdmin(String user) throws UserAlreadyExistsException { return doWithAdminLock(() -> adminManager.addMetalakeAdmin(user)); } @@ -198,7 +211,8 @@ public User addMetalakeAdmin(String user) { * Removes a metalake admin. * * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` otherwise. + * @return `true` if the User was successfully removed, `false` only when there's no such metalake + * admin, otherwise it will throw an exception. * @throws RuntimeException If removing the User encounters storage issues. */ public boolean removeMetalakeAdmin(String user) { @@ -234,7 +248,8 @@ public boolean isMetalakeAdmin(String user) { * @param securableObject The securable object of the Role. * @param privileges The privileges of the Role. * @return The created Role instance. - * @throws RoleAlreadyExistsException If a Role with the same identifier already exists. + * @throws RoleAlreadyExistsException If a Role with the same name already exists. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( @@ -243,34 +258,38 @@ public Role createRole( Map properties, SecurableObject securableObject, List privileges) - throws RoleAlreadyExistsException { + throws RoleAlreadyExistsException, NoSuchMetalakeException { return doWithNonAdminLock( () -> roleManager.createRole(metalake, role, properties, securableObject, privileges)); } /** - * Loads a Role. + * Gets a Role. * * @param metalake The Metalake of the Role. * @param role The name of the Role. - * @return The loading Role instance. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RuntimeException If loading the Role encounters storage issues. + * @return The getting Role instance. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If getting the Role encounters storage issues. */ - public Role loadRole(String metalake, String role) throws NoSuchRoleException { - return doWithNonAdminLock(() -> roleManager.loadRole(metalake, role)); + public Role getRole(String metalake, String role) + throws NoSuchRoleException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> roleManager.getRole(metalake, role)); } /** - * Drops a Role. + * Deletes a Role. * * @param metalake The Metalake of the Role. * @param role The name of the Role. - * @return `true` if the Role was successfully dropped, `false` otherwise. - * @throws RuntimeException If dropping the User encounters storage issues. + * @return `true` if the Role was successfully deleted, `false` only when there's no such role, + * otherwise it will throw an exception. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If deleting the Role encounters storage issues. */ - public boolean dropRole(String metalake, String role) { - return doWithNonAdminLock(() -> roleManager.dropRole(metalake, role)); + public boolean deleteRole(String metalake, String role) throws NoSuchMetalakeException { + return doWithNonAdminLock(() -> roleManager.deleteRole(metalake, role)); } @VisibleForTesting diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java index 4453e68d255..a77eb191336 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java @@ -44,7 +44,7 @@ class PermissionManager { boolean grantRoleToUser(String metalake, String role, String user) { try { - RoleEntity roleEntity = roleManager.loadRole(metalake, role); + RoleEntity roleEntity = roleManager.getRole(metalake, role); store.update( AuthorizationUtils.ofUser(metalake, user), @@ -98,7 +98,7 @@ boolean grantRoleToUser(String metalake, String role, String user) { boolean grantRoleToGroup(String metalake, String role, String group) { try { - RoleEntity roleEntity = roleManager.loadRole(metalake, role); + RoleEntity roleEntity = roleManager.getRole(metalake, role); store.update( AuthorizationUtils.ofGroup(metalake, group), @@ -152,7 +152,7 @@ boolean grantRoleToGroup(String metalake, String role, String group) { boolean revokeRoleFromGroup(String metalake, String role, String group) { try { - RoleEntity roleEntity = roleManager.loadRole(metalake, role); + RoleEntity roleEntity = roleManager.getRole(metalake, role); AtomicBoolean removed = new AtomicBoolean(true); @@ -212,7 +212,7 @@ boolean revokeRoleFromGroup(String metalake, String role, String group) { boolean revokeRoleFromUser(String metalake, String role, String user) { try { - RoleEntity roleEntity = roleManager.loadRole(metalake, role); + RoleEntity roleEntity = roleManager.getRole(metalake, role); AtomicBoolean removed = new AtomicBoolean(true); store.update( diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index ce396599967..71d72c36e41 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -108,7 +108,7 @@ RoleEntity createRole( } } - RoleEntity loadRole(String metalake, String role) throws NoSuchRoleException { + RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { try { AuthorizationUtils.checkMetalakeExists(metalake); return getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); @@ -118,7 +118,7 @@ RoleEntity loadRole(String metalake, String role) throws NoSuchRoleException { } } - boolean dropRole(String metalake, String role) { + boolean deleteRole(String metalake, String role) { try { AuthorizationUtils.checkMetalakeExists(metalake); NameIdentifier ident = AuthorizationUtils.ofRole(metalake, role); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java index e65115c3c51..6b69395be53 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java @@ -245,9 +245,9 @@ public void testLoadRole() { accessControlManager.createRole( "metalake", "loadRole", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); - Role cachedRole = accessControlManager.loadRole("metalake", "loadRole"); + Role cachedRole = accessControlManager.getRole("metalake", "loadRole"); accessControlManager.getRoleManager().getCache().invalidateAll(); - Role role = accessControlManager.loadRole("metalake", "loadRole"); + Role role = accessControlManager.getRole("metalake", "loadRole"); // Verify the cached roleEntity is correct Assertions.assertEquals(role, cachedRole); @@ -258,8 +258,7 @@ public void testLoadRole() { // Test load non-existed group Throwable exception = Assertions.assertThrows( - NoSuchRoleException.class, - () -> accessControlManager.loadRole("metalake", "not-exist")); + NoSuchRoleException.class, () -> accessControlManager.getRole("metalake", "not-exist")); Assertions.assertTrue(exception.getMessage().contains("Role not-exist does not exist")); } @@ -271,11 +270,11 @@ public void testDropRole() { "metalake", "testDrop", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); // Test drop role - boolean dropped = accessControlManager.dropRole("metalake", "testDrop"); + boolean dropped = accessControlManager.deleteRole("metalake", "testDrop"); Assertions.assertTrue(dropped); // Test drop non-existed role - boolean dropped1 = accessControlManager.dropRole("metalake", "no-exist"); + boolean dropped1 = accessControlManager.deleteRole("metalake", "no-exist"); Assertions.assertFalse(dropped1); } diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java index 00dd6127bc4..cf38825fe3f 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -117,7 +117,7 @@ public static void tearDown() throws IOException { } @Test - public void testAddRoleToUser() { + public void testGrantRoleToUser() { String notExist = "not-exist"; User user = accessControlManager.getUser(METALAKE, USER); @@ -153,7 +153,7 @@ public void testAddRoleToUser() { } @Test - public void testRemoveRoleFromUser() { + public void testRevokeRoleFromUser() { String notExist = "not-exist"; Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); @@ -179,7 +179,7 @@ public void testRemoveRoleFromUser() { } @Test - public void testAddRoleToGroup() { + public void testGrantRoleToGroup() { String notExist = "not-exist"; Group group = accessControlManager.getGroup(METALAKE, GROUP); @@ -216,7 +216,7 @@ public void testAddRoleToGroup() { } @Test - public void testRemoveRoleFormGroup() { + public void testRevokeRoleFormGroup() { String notExist = "not-exist"; Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); @@ -261,7 +261,7 @@ public void testDropRole() throws IOException { entityStore.put(roleEntity, true); Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, anotherRole, USER)); Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, anotherRole, GROUP)); - accessControlManager.dropRole(METALAKE, anotherRole); + accessControlManager.deleteRole(METALAKE, anotherRole); Group group = accessControlManager.getGroup(METALAKE, GROUP); Assertions.assertTrue(group.roles().isEmpty()); } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java index 78a4f56aa0a..6daacfc822e 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java @@ -13,5 +13,6 @@ public enum OperationType { /** This is a special operation type that is used to get a partition from a table. */ GET, ADD, - REMOVE + REMOVE, + DELETE } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java index c89f27d0a7c..14b8d331179 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java @@ -11,7 +11,7 @@ import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.dto.requests.RoleCreateRequest; -import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.DeleteResponse; import com.datastrato.gravitino.dto.responses.RoleResponse; import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.metrics.MetricNames; @@ -44,18 +44,18 @@ public RoleOperations() { @GET @Path("{role}") @Produces("application/vnd.gravitino.v1+json") - @Timed(name = "load-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "load-role", absolute = true) - public Response loadRole(@PathParam("metalake") String metalake, @PathParam("role") String role) { + @Timed(name = "get-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-role", absolute = true) + public Response getRole(@PathParam("metalake") String metalake, @PathParam("role") String role) { try { return Utils.doAs( httpRequest, () -> Utils.ok( new RoleResponse( - DTOConverters.toDTO(accessControlManager.loadRole(metalake, role))))); + DTOConverters.toDTO(accessControlManager.getRole(metalake, role))))); } catch (Exception e) { - return ExceptionHandlers.handleRoleException(OperationType.LOAD, role, metalake, e); + return ExceptionHandlers.handleRoleException(OperationType.GET, role, metalake, e); } } @@ -63,7 +63,7 @@ public Response loadRole(@PathParam("metalake") String metalake, @PathParam("rol @Produces("application/vnd.gravitino.v1+json") @Timed(name = "create-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-role", absolute = true) - public Response creatRole(@PathParam("metalake") String metalake, RoleCreateRequest request) { + public Response createRole(@PathParam("metalake") String metalake, RoleCreateRequest request) { try { return Utils.doAs( httpRequest, @@ -88,21 +88,22 @@ public Response creatRole(@PathParam("metalake") String metalake, RoleCreateRequ @DELETE @Path("{role}") @Produces("application/vnd.gravitino.v1+json") - @Timed(name = "drop-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "drop-role", absolute = true) - public Response dropRole(@PathParam("metalake") String metalake, @PathParam("role") String role) { + @Timed(name = "delete-role." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-role", absolute = true) + public Response deleteRole( + @PathParam("metalake") String metalake, @PathParam("role") String role) { try { return Utils.doAs( httpRequest, () -> { - boolean dropped = accessControlManager.dropRole(metalake, role); - if (!dropped) { - LOG.warn("Failed to drop role {} under metalake {}", role, metalake); + boolean deteted = accessControlManager.deleteRole(metalake, role); + if (!deteted) { + LOG.warn("Failed to delete role {} under metalake {}", role, metalake); } - return Utils.ok(new DropResponse(dropped)); + return Utils.ok(new DeleteResponse(deteted)); }); } catch (Exception e) { - return ExceptionHandlers.handleRoleException(OperationType.DROP, role, metalake, e); + return ExceptionHandlers.handleRoleException(OperationType.DELETE, role, metalake, e); } } } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java index 17d74cc02bb..59397bce665 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java @@ -20,7 +20,7 @@ import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.dto.authorization.RoleDTO; import com.datastrato.gravitino.dto.requests.RoleCreateRequest; -import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.DeleteResponse; import com.datastrato.gravitino.dto.responses.ErrorConstants; import com.datastrato.gravitino.dto.responses.ErrorResponse; import com.datastrato.gravitino.dto.responses.RoleResponse; @@ -176,10 +176,10 @@ public void testCreateRole() { } @Test - public void testLoadRole() { + public void testGetRole() { Role role = buildRole("role1"); - when(manager.loadRole(any(), any())).thenReturn(role); + when(manager.getRole(any(), any())).thenReturn(role); Response resp = target("/metalakes/metalake1/roles/role1") @@ -198,7 +198,7 @@ public void testLoadRole() { Assertions.assertEquals(Lists.newArrayList(Privileges.LoadCatalog.get()), roleDTO.privileges()); // Test to throw NoSuchMetalakeException - doThrow(new NoSuchMetalakeException("mock error")).when(manager).loadRole(any(), any()); + doThrow(new NoSuchMetalakeException("mock error")).when(manager).getRole(any(), any()); Response resp1 = target("/metalakes/metalake1/roles/role1") .request(MediaType.APPLICATION_JSON_TYPE) @@ -212,7 +212,7 @@ public void testLoadRole() { Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResponse.getType()); // Test to throw NoSuchRoleException - doThrow(new NoSuchRoleException("mock error")).when(manager).loadRole(any(), any()); + doThrow(new NoSuchRoleException("mock error")).when(manager).getRole(any(), any()); Response resp2 = target("/metalakes/metalake1/roles/role1") .request(MediaType.APPLICATION_JSON_TYPE) @@ -226,7 +226,7 @@ public void testLoadRole() { Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse1.getType()); // Test to throw internal RuntimeException - doThrow(new RuntimeException("mock error")).when(manager).loadRole(any(), any()); + doThrow(new RuntimeException("mock error")).when(manager).getRole(any(), any()); Response resp3 = target("/metalakes/metalake1/roles/role1") .request(MediaType.APPLICATION_JSON_TYPE) @@ -254,8 +254,8 @@ private Role buildRole(String role) { } @Test - public void testDropRole() { - when(manager.dropRole(any(), any())).thenReturn(true); + public void testDeleteRole() { + when(manager.deleteRole(any(), any())).thenReturn(true); Response resp = target("/metalakes/metalake1/roles/role1") @@ -264,12 +264,12 @@ public void testDropRole() { .delete(); Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); - DropResponse dropResponse = resp.readEntity(DropResponse.class); - Assertions.assertEquals(0, dropResponse.getCode()); - Assertions.assertTrue(dropResponse.dropped()); + DeleteResponse deleteResponse = resp.readEntity(DeleteResponse.class); + Assertions.assertEquals(0, deleteResponse.getCode()); + Assertions.assertTrue(deleteResponse.deleted()); - // Test when failed to drop role - when(manager.dropRole(any(), any())).thenReturn(false); + // Test when failed to delete role + when(manager.deleteRole(any(), any())).thenReturn(false); Response resp2 = target("/metalakes/metalake1/roles/role1") .request(MediaType.APPLICATION_JSON_TYPE) @@ -277,11 +277,11 @@ public void testDropRole() { .delete(); Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); - DropResponse dropResponse2 = resp2.readEntity(DropResponse.class); - Assertions.assertEquals(0, dropResponse2.getCode()); - Assertions.assertFalse(dropResponse2.dropped()); + DeleteResponse deleteResponse2 = resp2.readEntity(DeleteResponse.class); + Assertions.assertEquals(0, deleteResponse2.getCode()); + Assertions.assertFalse(deleteResponse2.deleted()); - doThrow(new RuntimeException("mock error")).when(manager).dropRole(any(), any()); + doThrow(new RuntimeException("mock error")).when(manager).deleteRole(any(), any()); Response resp3 = target("/metalakes/metalake1/roles/role1") .request(MediaType.APPLICATION_JSON_TYPE) From 9e6e1899d0598246a9c44bfb2a7b70e2aa8ee005 Mon Sep 17 00:00:00 2001 From: Xun Liu Date: Thu, 18 Apr 2024 20:44:45 +0800 Subject: [PATCH 059/106] [#2292] improvement(PyClient): Fix unstable Python client integration test (#2994) ### What changes were proposed in this pull request? 1. Add `skipPyClientITs` param in Gradle build script. Let's Python client integration test not running in the backend ITs 2. Combined Gradle build command of Python client test and integration test 3. improvement FilesetCatalog integration test codes. 4. Test mode + Test Principle: Every Python ITs class base on the `IntegrationTestEnv`, `IntegrationTestEnv` will automatic start and stop Gravitino server to support ITs, But when you run multiple ITs class at same time, The first test class that finishes running will shut down the Gravitino server, which will cause other test classes to fail if they can't connect to the Gravitino server. + Run test in the IDE: Through `IntegrationTestEnv` class automatic start and stop Gravitino server to support ITs. + Run test in the Github Action: Manual start and stop Gravitino server, and set `EXTERNAL_START_GRAVITINO` environment variable + Run test in the Gradle command `:client:client-python:test`: Gradle automatic start and stop Gravitino server, and set `EXTERNAL_START_GRAVITINO` environment variable. ### Why are the changes needed? Fix: #2292 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Through CI check. --- .../workflows/backend-integration-test.yml | 2 +- .github/workflows/python-integration-test.yml | 18 ++++----- clients/client-python/build.gradle.kts | 39 +++++++++---------- .../tests/integration/integration_test_env.py | 24 +++++++----- .../tests/integration/test_fileset_catalog.py | 11 +++--- 5 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 3cd93760e91..68664e85d2b 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -95,7 +95,7 @@ jobs: - name: Backend Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} + ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPyClientITs - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 95e18493fa6..2d9a6f9dd49 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -57,18 +57,18 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - name: Python Client Integration Test id: integrationTest run: | - #./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - #for pythonVersion in "3.8" "3.9" "3.10" "3.11" - #do - # ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest - #done + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + + for pythonVersion in "3.8" "3.9" "3.10" "3.11" + do + echo "Use Python version ${pythonVersion} to test the Python client." + ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:test + # Clean Gravitino database to clean test data + rm -rf ./distribution/package/data + done - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index 21b89367a1f..03bf90f687a 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -26,9 +26,9 @@ fun gravitinoServer(operation: String) { val exitCode = process.waitFor() if (exitCode == 0) { val currentContext = process.inputStream.bufferedReader().readText() - println("Current docker context is: $currentContext") + println("Gravitino server status: $currentContext") } else { - println("checkOrbStackStatus Command execution failed with exit code $exitCode") + println("Gravitino server execution failed with exit code $exitCode") } } @@ -39,26 +39,25 @@ tasks { } val test by registering(VenvTask::class) { - dependsOn(pipInstall) - venvExec = "python" - args = listOf("-m", "unittest") - workingDir = projectDir.resolve(".") - } - - val integrationTest by registering(VenvTask::class) { - doFirst() { - gravitinoServer("start") - } + val skipPyClientITs = project.hasProperty("skipPyClientITs") + if (!skipPyClientITs) { + doFirst { + gravitinoServer("start") + } - dependsOn(pipInstall) - venvExec = "python" - args = listOf("-m", "unittest") - workingDir = projectDir.resolve(".") - environment = mapOf("PROJECT_VERSION" to project.version, - "GRADLE_START_GRAVITINO" to "True") + dependsOn(pipInstall) + venvExec = "python" + args = listOf("-m", "unittest") + workingDir = projectDir.resolve(".") + environment = mapOf( + "PROJECT_VERSION" to project.version, + "GRAVITINO_HOME" to project.rootDir.path + "/distribution/package", + "START_EXTERNAL_GRAVITINO" to "true" + ) - doLast { - gravitinoServer("stop") + doLast { + gravitinoServer("stop") + } } } diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index 73cbaed5e01..e02206bf07c 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -20,11 +20,11 @@ def get_gravitino_server_version(): response.close() return True except requests.exceptions.RequestException as e: - logger.warning("Failed to access the server: {}", e) + logger.warning("Failed to access the Gravitino server") return False -def check_gravitino_server_status(): +def check_gravitino_server_status() -> bool: gravitino_server_running = False for i in range(5): logger.info("Monitoring Gravitino server status. Attempt %s", i + 1) @@ -53,15 +53,21 @@ class IntegrationTestEnv(unittest.TestCase): def setUpClass(cls): _init_logging() - if os.environ.get('GRADLE_START_GRAVITINO') is not None: - logger.info('Manual start gravitino server [%s].', check_gravitino_server_status()) + if os.environ.get('START_EXTERNAL_GRAVITINO') is not None: + """Maybe Gravitino server already startup by Gradle test command or developer manual startup.""" + if not check_gravitino_server_status(): + logger.error("ERROR: Can't find online Gravitino server!") return - current_path = os.getcwd() - cls.gravitino_startup_script = os.path.join(current_path, '../../../distribution/package/bin/gravitino.sh') + GravitinoHome = os.environ.get('GRAVITINO_HOME') + if GravitinoHome is None: + logger.error('Gravitino Python client integration test must configure `GRAVITINO_HOME`') + quit(0) + + cls.gravitino_startup_script = os.path.join(GravitinoHome, 'bin/gravitino.sh') if not os.path.exists(cls.gravitino_startup_script): logger.error("Can't find Gravitino startup script: %s, " - "Please execute `./gradlew compileDistribution -x test` in the gravitino project root " + "Please execute `./gradlew compileDistribution -x test` in the Gravitino project root " "directory.", cls.gravitino_startup_script) quit(0) @@ -78,11 +84,9 @@ def setUpClass(cls): logger.error("ERROR: Can't start Gravitino server!") quit(0) - cls.clean_test_date() - @classmethod def tearDownClass(cls): - if os.environ.get('GRADLE_START_GRAVITINO') is not None: + if os.environ.get('START_EXTERNAL_GRAVITINO') is not None: return logger.info("Stop integration test environment...") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 20d20970392..240547928f5 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -3,6 +3,7 @@ This software is licensed under the Apache License version 2. """ import logging +from random import random, randint from gravitino.api.catalog import Catalog from gravitino.api.fileset import Fileset @@ -20,11 +21,11 @@ class TestFilesetCatalog(IntegrationTestEnv): catalog: Catalog = None metalake: GravitinoMetalake = None - metalake_name: str = "testMetalake" - catalog_name: str = "testCatalog" - schema_name: str = "testSchema" - fileset_name: str = "testFileset1" - fileset_alter_name: str = "testFilesetAlter" + metalake_name: str = "testMetalake" + str(randint(1, 100)) + catalog_name: str = "testCatalog" + str(randint(1, 100)) + schema_name: str = "testSchema" + str(randint(1, 100)) + fileset_name: str = "testFileset1" + str(randint(1, 100)) + fileset_alter_name: str = "testFilesetAlter" + str(randint(1, 100)) provider: str = "hadoop" metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) From 3c39fa97d7e19f056cbefe2f6f760114cb2b47c8 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Fri, 19 Apr 2024 10:14:51 +0800 Subject: [PATCH 060/106] [#3005] fix(web): fix table properties displaying format (#3019) ### What changes were proposed in this pull request? fix table properties displaying format - partitioning: - image - image - indexes: - image - image - distribution: - image - image - sortOrders: - image - image ### Why are the changes needed? Fix: #3005 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? local --- .../rightContent/tabsContent/TabsContent.js | 29 +++++++++++++------ .../tabsContent/tableView/TableView.js | 12 ++++++-- web/src/lib/store/metalakes/index.js | 4 +-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js index 699d3bf069b..bdea15e7763 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js @@ -7,7 +7,7 @@ import { Inconsolata } from 'next/font/google' -import { useState, useEffect } from 'react' +import { useState, useEffect, Fragment } from 'react' import { styled, Box, Divider, List, ListItem, ListItemText, Stack, Tab, Typography } from '@mui/material' import Tooltip, { tooltipClasses } from '@mui/material/Tooltip' @@ -163,10 +163,16 @@ const TabsContent = () => { - {item.items.map(i => { + {item.items.map((it, idx) => { return ( - - {item.type === 'sortOrders' ? i.text : i.fields} + + {item.type === 'sortOrders' ? it.text : it.fields.join('.')} ) })} @@ -210,11 +216,16 @@ const TabsContent = () => { textOverflow: 'ellipsis' }} > - - {item.type === 'sortOrders' - ? item.items.map(i => i.text) - : item.items.map(i => i.fields)} - + {item.items.map((it, idx) => { + return ( + + + {it.fields.join('.')} + + {idx < item.items.length - 1 && , } + + ) + })} } /> diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js index 1074e86aa43..71a2ea1464f 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js @@ -111,10 +111,16 @@ const TableView = () => { - {items.map(i => { + {items.map((it, idx) => { return ( - - {i.text || i.fields} + + {it.text || it.fields} ) })} diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js index eeba3508a4c..b9e505fc319 100644 --- a/web/src/lib/store/metalakes/index.js +++ b/web/src/lib/store/metalakes/index.js @@ -609,7 +609,7 @@ export const getTableDetails = createAsyncThunk( items: partitioning.map(i => { let fields = i.fieldName || [] let sub = '' - let last = i.fieldName.join('.') + let last = i.fieldName switch (i.strategy) { case 'bucket': @@ -680,7 +680,7 @@ export const getTableDetails = createAsyncThunk( fields: i.fieldNames, name: i.name, indexType: i.indexType, - text: `${i.name}(${i.fieldNames.join(',')})` + text: `${i.name}(${i.fieldNames.join('.')})` } }) } From 7f1f3a9af44748a7c4644713f540b0dd44511a09 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Fri, 19 Apr 2024 11:27:45 +0800 Subject: [PATCH 061/106] [#3010] fix(web): fix gradle build issue (#3013) ### What changes were proposed in this pull request? fix gradle build will update pnpm-lock.yaml ### Why are the changes needed? Fix: #3010 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? LOCAL --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index b8a79d08acc..20d08c02d96 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -407,6 +407,7 @@ subprojects { plugins.apply(NodePlugin::class) configure { version.set("20.9.0") + pnpmVersion.set("9.x") nodeProjectDir.set(file("$rootDir/.node")) download.set(true) } From bbcbe6726c112449891f0586b2b5de67e89d5809 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 19 Apr 2024 11:28:48 +0800 Subject: [PATCH 062/106] [#3012] Fix(build): Fix compile issue after #2866 is merged (#3020) ### What changes were proposed in this pull request? The proposal changes the task to the function and makes sure the function will not be skipped, also explicitly add the file to jar to make sure it will be executed every time. ### Why are the changes needed? Fix: #3012 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual test. --- common/build.gradle.kts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 5a019da140a..9b7cecbbefc 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -56,7 +56,7 @@ fun getGitCommitId(): String { } val propertiesFile = "src/main/resources/project.properties" -val writeProjectPropertiesFile = tasks.register("writeProjectPropertiesFile") { +fun writeProjectPropertiesFile() { val propertiesFile = file(propertiesFile) if (propertiesFile.exists()) { propertiesFile.delete() @@ -85,13 +85,18 @@ val writeProjectPropertiesFile = tasks.register("writeProjectPropertiesFile") { tasks { jar { - dependsOn(writeProjectPropertiesFile) doFirst() { + writeProjectPropertiesFile() if (!file(propertiesFile).exists()) { throw GradleException("$propertiesFile file not generated!") } } + + from("src/main/resources") { + include("project.properties").duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } } + clean { delete("$propertiesFile") } From 6c8b3c46092cbd835247fb612b3e35ba806bb41f Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Fri, 19 Apr 2024 13:05:29 +0800 Subject: [PATCH 063/106] [#2996][#3011] fix(web): fix refresh issue and kafka topic property issue (#3014) ### What changes were proposed in this pull request? Load catalogs first when refresh schema or table level page image Fix kafka topic property issue ### Why are the changes needed? Fix: #2996, Fix: #3011 ### Does this PR introduce _any_ user-facing change? When schema, table or fileset page network error, refresh the page ### How was this patch tested? image --- .../app/metalakes/metalake/MetalakeTree.js | 4 +-- .../app/metalakes/metalake/MetalakeView.js | 35 ++++++++++++------- .../rightContent/CreateCatalogDialog.js | 2 +- .../tabsContent/detailsView/DetailsView.js | 19 +++++----- web/src/lib/store/metalakes/index.js | 28 --------------- 5 files changed, 36 insertions(+), 52 deletions(-) diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index 58141684290..9435bf7e648 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -79,7 +79,7 @@ const MetalakeTree = props => { case 'fileset': { if (store.selectedNodes.includes(nodeProps.data.key)) { const pathArr = extractPlaceholder(nodeProps.data.key) - const [metalake, catalog, schema, fileset] = pathArr + const [metalake, catalog, type, schema, fileset] = pathArr dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) } break @@ -87,7 +87,7 @@ const MetalakeTree = props => { case 'topic': { if (store.selectedNodes.includes(nodeProps.data.key)) { const pathArr = extractPlaceholder(nodeProps.data.key) - const [metalake, catalog, schema, topic] = pathArr + const [metalake, catalog, type, schema, topic] = pathArr dispatch(getTopicDetails({ init: true, metalake, catalog, schema, topic })) } break diff --git a/web/src/app/metalakes/metalake/MetalakeView.js b/web/src/app/metalakes/metalake/MetalakeView.js index 44c3e2e2daf..992e6af46b4 100644 --- a/web/src/app/metalakes/metalake/MetalakeView.js +++ b/web/src/app/metalakes/metalake/MetalakeView.js @@ -9,7 +9,7 @@ import { useEffect } from 'react' import { Box } from '@mui/material' -import { useAppDispatch } from '@/lib/hooks/useStore' +import { useAppDispatch, useAppSelector } from '@/lib/hooks/useStore' import { useSearchParams } from 'next/navigation' import MetalakePageLeftBar from './MetalakePageLeftBar' import RightContent from './rightContent/RightContent' @@ -32,8 +32,8 @@ import { const MetalakeView = () => { const dispatch = useAppDispatch() const searchParams = useSearchParams() - const paramsSize = [...searchParams.keys()].length + const store = useAppSelector(state => state.metalakes) useEffect(() => { const routeParams = { @@ -54,11 +54,18 @@ const MetalakeView = () => { } if (paramsSize === 3 && catalog) { + if (!store.catalogs.length) { + dispatch(fetchCatalogs({ metalake })) + } dispatch(fetchSchemas({ init: true, page: 'catalogs', metalake, catalog, type })) dispatch(getCatalogDetails({ metalake, catalog, type })) } if (paramsSize === 4 && catalog && type && schema) { + if (!store.catalogs.length) { + dispatch(fetchCatalogs({ metalake })) + dispatch(fetchSchemas({ metalake, catalog, type })) + } switch (type) { case 'relational': dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema })) @@ -75,16 +82,20 @@ const MetalakeView = () => { dispatch(getSchemaDetails({ metalake, catalog, schema })) } - if (paramsSize === 5 && catalog && schema && table) { - dispatch(getTableDetails({ init: true, metalake, catalog, schema, table })) - } - - if (paramsSize === 5 && catalog && schema && fileset) { - dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) - } - - if (paramsSize === 5 && catalog && schema && topic) { - dispatch(getTopicDetails({ init: true, metalake, catalog, schema, topic })) + if (paramsSize === 5 && catalog && schema) { + if (!store.catalogs.length) { + dispatch(fetchCatalogs({ metalake })) + dispatch(fetchSchemas({ metalake, catalog, type })) + } + if (table) { + dispatch(getTableDetails({ init: true, metalake, catalog, schema, table })) + } + if (fileset) { + dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset })) + } + if (topic) { + dispatch(getTopicDetails({ init: true, metalake, catalog, schema, topic })) + } } } diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index f3005fcd4f5..cfab9e650db 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -396,7 +396,7 @@ const CreateCatalogDialog = props => { setInnerProps(propsItems) setValue('propItems', propsItems) } - }, [open, data, setValue]) + }, [open, data, setValue, type]) return (

diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js index a15b287a259..84728a83cf5 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js @@ -23,7 +23,7 @@ const DetailsView = () => { const audit = activatedItem?.audit || {} - const properties = Object.keys(activatedItem?.properties || []) + let properties = Object.keys(activatedItem?.properties || []) .filter(key => !['partition-count', 'replication-factor'].includes(key)) .map(item => { return { @@ -32,14 +32,15 @@ const DetailsView = () => { } }) if (paramsSize === 5 && searchParams.get('topic')) { - properties.unshift({ - key: 'replication-factor', - value: JSON.stringify(activatedItem?.properties['replication-factor'])?.replace(/^"|"$/g, '') - }) - properties.unshift({ - key: 'partition-count', - value: JSON.stringify(activatedItem?.properties['partition-count'])?.replace(/^"|"$/g, '') - }) + const topicPros = Object.keys(activatedItem?.properties || []) + .filter(key => ['partition-count', 'replication-factor'].includes(key)) + .map(item => { + return { + key: item, + value: JSON.stringify(activatedItem?.properties[item]).replace(/^"|"$/g, '') + } + }) + properties = [...topicPros, ...properties] } const renderFieldText = ({ value, linkBreak = false, isDate = false }) => { diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js index b9e505fc319..30742dea554 100644 --- a/web/src/lib/store/metalakes/index.js +++ b/web/src/lib/store/metalakes/index.js @@ -492,10 +492,6 @@ export const fetchSchemas = createAsyncThunk( ) } - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch(setExpandedNodes([`{{${metalake}}}`, `{{${metalake}}}{{${catalog}}}{{${type}}}`])) return { schemas, page, init } @@ -567,10 +563,6 @@ export const fetchTables = createAsyncThunk( ) } - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, @@ -688,10 +680,6 @@ export const getTableDetails = createAsyncThunk( dispatch(setTableProps(tableProps)) - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, @@ -753,10 +741,6 @@ export const fetchFilesets = createAsyncThunk( ) } - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, @@ -786,10 +770,6 @@ export const getFilesetDetails = createAsyncThunk( const { fileset: resFileset } = res - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, @@ -851,10 +831,6 @@ export const fetchTopics = createAsyncThunk( ) } - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, @@ -884,10 +860,6 @@ export const getTopicDetails = createAsyncThunk( const { topic: resTopic } = res - if (getState().metalakes.metalakeTree.length === 0) { - dispatch(fetchCatalogs({ metalake })) - } - dispatch( setExpandedNodes([ `{{${metalake}}}`, From b547801cdd75f82e272b455edb0ec42396f7d5b7 Mon Sep 17 00:00:00 2001 From: lwyang <1670906161@qq.com> Date: Fri, 19 Apr 2024 16:15:03 +0800 Subject: [PATCH 064/106] [#3030] fix(test): adjust the order of assertion parameters (#3040) ### What changes were proposed in this pull request? assertion arguments should be in the in correct order: expected value, actual value ### Why are the changes needed? Fix: #3030 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? exist ut Co-authored-by: yangliwei --- .../gravitino/integration/test/web/ui/CatalogsPageTest.java | 2 +- .../gravitino/integration/test/web/ui/MetalakePageTest.java | 2 +- .../spark/connector/hive/TestHivePropertiesConverter.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java index 466500447c9..b31631030ba 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java @@ -251,7 +251,7 @@ public void testCreateKafkaCatalog() throws InterruptedException { @Order(7) public void testRefreshPage() { driver.navigate().refresh(); - Assertions.assertEquals(driver.getTitle(), WEB_TITLE); + Assertions.assertEquals(WEB_TITLE, driver.getTitle()); Assertions.assertTrue(catalogsPage.verifyRefreshPage()); List catalogsNames = Arrays.asList( diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/MetalakePageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/MetalakePageTest.java index 271be49e0ad..d288a8a2932 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/MetalakePageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/MetalakePageTest.java @@ -125,7 +125,7 @@ public void testLinkToCatalogsPage() throws InterruptedException { public void testRefreshPage() { driver.navigate().refresh(); - Assertions.assertEquals(driver.getTitle(), WEB_TITLE); + Assertions.assertEquals(WEB_TITLE, driver.getTitle()); Assertions.assertTrue(metalakePage.verifyRefreshPage()); } diff --git a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/hive/TestHivePropertiesConverter.java b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/hive/TestHivePropertiesConverter.java index 83bde5416a5..e8e830a9378 100644 --- a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/hive/TestHivePropertiesConverter.java +++ b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/hive/TestHivePropertiesConverter.java @@ -25,7 +25,7 @@ void testTableFormat() { hivePropertiesConverter.toGravitinoTableProperties( ImmutableMap.of(HivePropertiesConstants.SPARK_HIVE_STORED_AS, "PARQUET")); Assertions.assertEquals( - hiveProperties.get(HivePropertiesConstants.GRAVITINO_HIVE_FORMAT), "PARQUET"); + "PARQUET", hiveProperties.get(HivePropertiesConstants.GRAVITINO_HIVE_FORMAT)); Assertions.assertThrowsExactly( NotSupportedException.class, () -> @@ -84,8 +84,8 @@ void testExternalTable() { hivePropertiesConverter.toGravitinoTableProperties( ImmutableMap.of(HivePropertiesConstants.SPARK_HIVE_EXTERNAL, "true")); Assertions.assertEquals( - hiveProperties.get(HivePropertiesConstants.GRAVITINO_HIVE_TABLE_TYPE), - HivePropertiesConstants.GRAVITINO_HIVE_EXTERNAL_TABLE); + HivePropertiesConstants.GRAVITINO_HIVE_EXTERNAL_TABLE, + hiveProperties.get(HivePropertiesConstants.GRAVITINO_HIVE_TABLE_TYPE)); hiveProperties = hivePropertiesConverter.toSparkTableProperties( From 2f31fb6a393e80aad4c94ad9a9ae49ac8bb9d456 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Fri, 19 Apr 2024 16:38:20 +0800 Subject: [PATCH 065/106] [#3042] [MINOR] fix(docs): Fix the authentication content format in gvfs docs (#3043) ### What changes were proposed in this pull request? Fix the format of authentication content in gvfs docs. ![image](https://github.com/datastrato/gravitino/assets/26177232/217288ae-3bc5-4e5e-8dd9-ed0380041992) ### Why are the changes needed? Fix: #3042 Co-authored-by: xiaojiebao --- docs/how-to-use-gvfs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index cea93de5e81..be0bc3e6fba 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -234,9 +234,9 @@ Currently, Gravitino Virtual File System supports two kinds of authentication ty The type of `simple` is the default authentication type in Gravitino Virtual File System. -### How to use simple authentication +### How to use authentication -#### Using `simple` authentication type +#### Using `simple` authentication First, make sure that your Gravitino server is also configured to use the `simple` authentication mode. @@ -260,7 +260,7 @@ Path filesetPath = new Path("gvfs://fileset/test_catalog/test_schema/test_filese FileSystem fs = filesetPath.getFileSystem(conf); ``` -#### Using OAuth authentication +#### Using `OAuth` authentication If you want to use `oauth2` authentication for the Gravitino client in the Gravitino Virtual File System, please refer to this document to complete the configuration of the Gravitino server and the OAuth server: [Security](./security.md). From bb4d18328ed67804629c87bde721cca219a1aaa2 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Fri, 19 Apr 2024 17:43:00 +0800 Subject: [PATCH 066/106] [#2948] Improve(trino-connector): Update all the testers for using simple catalog name (#3004) ### What changes were proposed in this pull request? Make using simple catalog name as default and update all the testers ### Why are the changes needed? Fix: #2948 ### Does this PR introduce _any_ user-facing change? The related documentation will updated by another PR ### How was this patch tested? UT IT --- docs/trino-connector/configuration.md | 2 +- .../test/trino/TrinoConnectorIT.java | 231 +++++++----------- .../bugs/00002_alter_table_mysql.sql | 16 -- .../testsets/hive/00000_create_table.sql | 8 +- .../testsets/hive/00001_select_table.sql | 24 +- .../testsets/hive/00002_alter_table.sql | 40 +-- .../testsets/hive/00002_alter_table.txt | 12 +- .../testsets/hive/00005_catalog.sql | 6 +- .../testsets/hive/00005_catalog.txt | 4 +- .../testsets/hive/00006_datatype.sql | 6 +- .../testsets/hive/00006_datatype.txt | 2 +- .../testsets/hive/00007_varchar.sql | 30 +-- .../testsets/hive/00007_varchar.txt | 12 +- .../jdbc-mysql/00000_create_table.sql | 50 ++-- .../jdbc-mysql/00000_create_table.txt | 10 +- .../jdbc-mysql/00001_select_table.sql | 26 +- .../testsets/jdbc-mysql/00002_alter_table.sql | 50 ++-- .../testsets/jdbc-mysql/00002_alter_table.txt | 16 +- .../testsets/jdbc-mysql/00003_use.sql | 10 +- .../jdbc-mysql/00004_query_pushdown.sql | 6 +- .../jdbc-mysql/00004_query_pushdown.txt | 10 +- .../jdbc-mysql/00005_create_catalog.sql | 8 +- .../jdbc-mysql/00005_create_catalog.txt | 4 +- .../testsets/jdbc-mysql/00006_datatype.sql | 6 +- .../testsets/jdbc-mysql/00006_datatype.txt | 4 +- .../testsets/jdbc-mysql/00007_varchar.sql | 30 +-- .../testsets/jdbc-mysql/00007_varchar.txt | 12 +- .../jdbc-mysql/00008_alter_catalog.sql | 8 +- .../jdbc-mysql/00008_alter_catalog.txt | 8 +- .../jdbc-postgresql/00000_create_table.sql | 8 +- .../jdbc-postgresql/00001_select_table.sql | 24 +- .../jdbc-postgresql/00002_alter_table.sql | 40 +-- .../jdbc-postgresql/00002_alter_table.txt | 12 +- .../jdbc-postgresql/00003_join_pushdown.sql | 22 +- .../jdbc-postgresql/00004_query_pushdown.sql | 6 +- .../jdbc-postgresql/00004_query_pushdown.txt | 10 +- .../jdbc-postgresql/00006_datatype.sql | 6 +- .../jdbc-postgresql/00006_datatype.txt | 4 +- .../jdbc-postgresql/00007_varchar.sql | 30 +-- .../jdbc-postgresql/00007_varchar.txt | 12 +- .../lakehouse-iceberg/00000_create_table.sql | 42 ++-- .../lakehouse-iceberg/00000_create_table.txt | 12 +- .../lakehouse-iceberg/00001_select_table.sql | 24 +- .../lakehouse-iceberg/00002_alter_table.sql | 40 +-- .../lakehouse-iceberg/00002_alter_table.txt | 12 +- .../lakehouse-iceberg/00006_datatype.sql | 6 +- .../lakehouse-iceberg/00006_datatype.txt | 4 +- .../lakehouse-iceberg/00007_varchar.sql | 10 +- .../lakehouse-iceberg/00007_varchar.txt | 2 +- .../testsets/tpcds/catalog_mysql_prepare.sql | 4 +- .../testsets/tpch/catalog_hive_prepare.sql | 4 +- .../testsets/tpch/catalog_iceberg_prepare.sql | 4 +- .../testsets/tpch/catalog_mysql_prepare.sql | 4 +- .../tpch/catalog_postgresql_prepare.sql | 4 +- .../trino/connector/GravitinoConfig.java | 2 +- .../connector/GravitinoConnectorFactory.java | 5 +- .../catalog/CatalogConnectorFactory.java | 31 +-- .../catalog/CatalogConnectorManager.java | 82 +++++-- .../connector/metadata/GravitinoCatalog.java | 6 +- .../table/GravitinoSystemTableCatalog.java | 2 +- .../trino/connector/GravitinoMockServer.java | 46 ++-- .../TestCreateGravitinoConnector.java | 36 ++- .../connector/TestGravitinoConnector.java | 68 ++---- ...itinoConnectorWithMetalakeCatalogName.java | 149 +++++++++++ ...avitinoConnectorWithSimpleCatalogName.java | 92 ------- 65 files changed, 771 insertions(+), 745 deletions(-) delete mode 100644 integration-test/src/test/resources/trino-ci-testset/bugs/00002_alter_table_mysql.sql create mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java delete mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 5b1065abe7d..cd1e4c6684f 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -11,4 +11,4 @@ This software is licensed under the Apache License version 2." | connector.name | string | (none) | The `connector.name` defines the name of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | | gravitino.metalake | string | (none) | The `gravitino.metalake` defines which metalake in Gravitino server the Trino connector uses. Trino connector should set it at start, the value of `gravitino.metalake` needs to be a valid name, Trino connector can detect and load the metalake with catalogs, schemas and tables once created and keep in sync. | Yes | 0.2.0 | | gravitino.uri | string | http://localhost:8090 | The `gravitino.uri` defines the connection URL of the Gravitino server, the default value is `http://localhost:8090`. Trino connector can detect and connect to Gravitino server once it is ready, no need to start Gravitino server beforehand. | Yes | 0.2.0 | -| gravitino.simplify-catalog-names | boolean | false | The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. If you set it to true, Trino will configure only one Graviton catalog. | NO | 0.5.0 | +| gravitino.simplify-catalog-names | boolean | true | The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. If you set it to true, Trino will configure only one Graviton catalog. | NO | 0.5.0 | diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 0067763c4b4..1f8c90ce769 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -131,10 +131,9 @@ public static void stopDockerContainer() { public static void createSchema() throws TException, InterruptedException { String sql1 = String.format( - "CREATE SCHEMA \"%s.%s\".%s WITH (\n" + "CREATE SCHEMA %s.%s WITH (\n" + " location = 'hdfs://%s:%d/user/hive/warehouse/%s.db'\n" + ")", - metalakeName, catalogName, databaseName, containerSuite.getHiveContainer().getContainerIpAddress(), @@ -149,17 +148,13 @@ public static void createSchema() throws TException, InterruptedException { containerSuite .getTrinoContainer() .executeQuerySQL( - String.format( - "show schemas from \"%s.%s\" like '%s'", - metalakeName, catalogName, databaseName)); + String.format("show schemas from %s like '%s'", catalogName, databaseName)); Assertions.assertEquals(r.get(0).get(0), databaseName); } @Test public void testShowSchemas() { - String sql = - String.format( - "SHOW SCHEMAS FROM \"%s.%s\" LIKE '%s'", metalakeName, catalogName, databaseName); + String sql = String.format("SHOW SCHEMAS FROM %s LIKE '%s'", catalogName, databaseName); ArrayList> queryData = containerSuite.getTrinoContainer().executeQuerySQL(sql); Assertions.assertEquals(queryData.get(0).get(0), databaseName); @@ -169,7 +164,7 @@ public void testShowSchemas() { public void testCreateTable() throws TException, InterruptedException { String sql3 = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (\n" + "CREATE TABLE %s.%s.%s (\n" + " col1 varchar,\n" + " col2 varchar,\n" + " col3 varchar\n" @@ -177,7 +172,7 @@ public void testCreateTable() throws TException, InterruptedException { + "WITH (\n" + " format = 'TEXTFILE'\n" + ")", - metalakeName, catalogName, databaseName, tab1Name); + catalogName, databaseName, tab1Name); containerSuite.getTrinoContainer().executeUpdateSQL(sql3); // Verify in Gravitino Server @@ -192,9 +187,7 @@ public void testCreateTable() throws TException, InterruptedException { private void testShowTable() { String sql = - String.format( - "SHOW TABLES FROM \"%s.%s\".%s LIKE '%s'", - metalakeName, catalogName, databaseName, tab1Name); + String.format("SHOW TABLES FROM %s.%s LIKE '%s'", catalogName, databaseName, tab1Name); ArrayList> queryData = containerSuite.getTrinoContainer().executeQuerySQL(sql); Assertions.assertEquals(queryData.get(0).get(0), tab1Name); @@ -205,9 +198,7 @@ private void verifySchemaAndTable(String dbName, String tableName) { ArrayList> r = containerSuite .getTrinoContainer() - .executeQuerySQL( - String.format( - "show schemas from \"%s.%s\" like '%s'", metalakeName, catalogName, dbName)); + .executeQuerySQL(String.format("show schemas from %s like '%s'", catalogName, dbName)); Assertions.assertEquals(r.get(0).get(0), dbName); // Compare table @@ -215,16 +206,14 @@ private void verifySchemaAndTable(String dbName, String tableName) { containerSuite .getTrinoContainer() .executeQuerySQL( - String.format( - "show create table \"%s.%s\".%s.%s", - metalakeName, catalogName, dbName, tableName)); + String.format("show create table %s.%s.%s", catalogName, dbName, tableName)); Assertions.assertTrue(r.get(0).get(0).contains(tableName)); } public void testScenarioTable1() throws TException, InterruptedException { String sql3 = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (\n" + "CREATE TABLE %s.%s.%s (\n" + " user_name varchar,\n" + " gender varchar,\n" + " age varchar,\n" @@ -238,7 +227,7 @@ public void testScenarioTable1() throws TException, InterruptedException { + "WITH (\n" + " format = 'TEXTFILE'\n" + ")", - metalakeName, catalogName, databaseName, scenarioTab1Name); + catalogName, databaseName, scenarioTab1Name); containerSuite.getTrinoContainer().executeUpdateSQL(sql3); // Verify in Gravitino Server @@ -258,8 +247,8 @@ public void testScenarioTable1() throws TException, InterruptedException { StringBuilder sql5 = new StringBuilder( String.format( - "INSERT INTO \"%s.%s\".%s.%s (user_name, gender, age, phone) VALUES", - metalakeName, catalogName, databaseName, scenarioTab1Name)); + "INSERT INTO %s.%s.%s (user_name, gender, age, phone) VALUES", + catalogName, databaseName, scenarioTab1Name)); int index = 0; for (ArrayList record : table1Data) { sql5.append( @@ -276,8 +265,8 @@ public void testScenarioTable1() throws TException, InterruptedException { // Select data from table1 and verify it String sql6 = String.format( - "SELECT user_name, gender, age, phone FROM \"%s.%s\".%s.%s ORDER BY user_name", - metalakeName, catalogName, databaseName, scenarioTab1Name); + "SELECT user_name, gender, age, phone FROM %s.%s.%s ORDER BY user_name", + catalogName, databaseName, scenarioTab1Name); ArrayList> table1QueryData = containerSuite.getTrinoContainer().executeQuerySQL(sql6); Assertions.assertEquals(table1Data, table1QueryData); @@ -286,7 +275,7 @@ public void testScenarioTable1() throws TException, InterruptedException { public void testScenarioTable2() throws TException, InterruptedException { String sql4 = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (\n" + "CREATE TABLE %s.%s.%s (\n" + " user_name varchar,\n" + " consumer varchar,\n" + " recharge varchar,\n" @@ -297,7 +286,7 @@ public void testScenarioTable2() throws TException, InterruptedException { + "WITH (\n" + " format = 'TEXTFILE'\n" + ")", - metalakeName, catalogName, databaseName, scenarioTab2Name); + catalogName, databaseName, scenarioTab2Name); containerSuite.getTrinoContainer().executeUpdateSQL(sql4); // Verify in Gravitino Server @@ -318,8 +307,8 @@ public void testScenarioTable2() throws TException, InterruptedException { StringBuilder sql7 = new StringBuilder( String.format( - "INSERT INTO \"%s.%s\".%s.%s (user_name, consumer, recharge, event_time) VALUES", - metalakeName, catalogName, databaseName, scenarioTab2Name)); + "INSERT INTO %s.%s.%s (user_name, consumer, recharge, event_time) VALUES", + catalogName, databaseName, scenarioTab2Name)); for (ArrayList record : table2Data) { sql7.append( String.format( @@ -335,8 +324,8 @@ public void testScenarioTable2() throws TException, InterruptedException { // Select data from table1 and verify it String sql8 = String.format( - "SELECT user_name, consumer, recharge, event_time FROM \"%s.%s\".%s.%s ORDER BY user_name", - metalakeName, catalogName, databaseName, scenarioTab2Name); + "SELECT user_name, consumer, recharge, event_time FROM %s.%s.%s ORDER BY user_name", + catalogName, databaseName, scenarioTab2Name); ArrayList> table2QueryData = containerSuite.getTrinoContainer().executeQuerySQL(sql8); Assertions.assertEquals(table2Data, table2QueryData); @@ -349,11 +338,11 @@ public void testScenarioJoinTwoTable() throws TException, InterruptedException { String sql9 = String.format( - "SELECT * FROM (SELECT t1.user_name as user_name, gender, age, phone, consumer, recharge, event_time FROM \"%1$s.%2$s\".%3$s.%4$s AS t1\n" + "SELECT * FROM (SELECT t1.user_name as user_name, gender, age, phone, consumer, recharge, event_time FROM %1s.%2$s.%3$s AS t1\n" + "JOIN\n" - + " (SELECT user_name, consumer, recharge, event_time FROM \"%1$s.%2$s\".%3$s.%5$s) AS t2\n" + + " (SELECT user_name, consumer, recharge, event_time FROM %1$s.%2$s.%4$s) AS t2\n" + " ON t1.user_name = t2.user_name) ORDER BY user_name", - metalakeName, catalogName, databaseName, scenarioTab1Name, scenarioTab2Name); + catalogName, databaseName, scenarioTab1Name, scenarioTab2Name); ArrayList> joinQueryData = containerSuite.getTrinoContainer().executeQuerySQL(sql9); ArrayList> joinData = new ArrayList<>(); @@ -379,8 +368,8 @@ void testHiveSchemaCreatedByTrino() { String createSchemaSql = String.format( - "CREATE SCHEMA \"%s.%s\".%s with( location = 'hdfs://localhost:9000/user/hive/warehouse/hive_schema_1123123')", - metalakeName, catalogName, schemaName); + "CREATE SCHEMA %s.%s with( location = 'hdfs://localhost:9000/user/hive/warehouse/hive_schema_1123123')", + catalogName, schemaName); containerSuite.getTrinoContainer().executeUpdateSQL(createSchemaSql); Schema schema = @@ -395,17 +384,16 @@ void testHiveTableCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); - String createSchemaSql = - String.format("CREATE SCHEMA \"%s.%s\".%s", metalakeName, catalogName, schemaName); + String createSchemaSql = String.format("CREATE SCHEMA %s.%s", catalogName, schemaName); containerSuite.getTrinoContainer().executeUpdateSQL(createSchemaSql); String createTableSql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar)" + "CREATE TABLE %s.%s.%s (id int, name varchar)" + " with ( serde_name = '123455', location = 'hdfs://localhost:9000/user/hive/warehouse/hive_schema.db/hive_table'" + ", partitioned_by = ARRAY['name'], bucketed_by = ARRAY['id'], bucket_count = 50, sorted_by = ARRAY['name']" + ")", - metalakeName, catalogName, schemaName, tableName); + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(createTableSql); Table table = @@ -473,8 +461,7 @@ void testHiveSchemaCreatedByGravitino() throws InterruptedException { .put("location", "hdfs://localhost:9000/user/hive/warehouse/hive_schema_1223445.db") .build()); - String sql = - String.format("show create schema \"%s.%s\".%s", metalakeName, catalogName, schemaName); + String sql = String.format("show create schema %s.%s", catalogName, schemaName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load schema created by gravitino: " + sql); @@ -623,27 +610,25 @@ void testColumnTypeNotNullByTrino() throws InterruptedException { .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) .build()); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30)); String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); - String createSchemaSql = - String.format("CREATE SCHEMA \"%s.%s\".%s", metalakeName, catalogName, schemaName); + String createSchemaSql = String.format("CREATE SCHEMA %s.%s", catalogName, schemaName); containerSuite.getTrinoContainer().executeUpdateSQL(createSchemaSql); - sql = String.format("show create schema \"%s.%s\".%s", metalakeName, catalogName, schemaName); + sql = String.format("show create schema %s.%s", catalogName, schemaName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30)); String createTableSql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int not null, name varchar not null)", - metalakeName, catalogName, schemaName, tableName); + "CREATE TABLE %s.%s.%s (id int not null, name varchar not null)", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(createTableSql); String showCreateTableSql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); ArrayList> rs = containerSuite.getTrinoContainer().executeQuerySQL(showCreateTableSql); Assertions.assertTrue(rs.get(0).get(0).toLowerCase(Locale.ENGLISH).contains("not null")); @@ -652,8 +637,7 @@ void testColumnTypeNotNullByTrino() throws InterruptedException { .getTrinoContainer() .executeUpdateSQL( String.format( - "insert into \"%s.%s\".%s.%s values(1, 'a')", - metalakeName, catalogName, schemaName, tableName)); + "insert into %s.%s.%s values(1, 'a')", catalogName, schemaName, tableName)); Assertions.assertThrows( RuntimeException.class, () -> @@ -661,8 +645,8 @@ void testColumnTypeNotNullByTrino() throws InterruptedException { .getTrinoContainer() .executeUpdateSQL( String.format( - "insert into \"%s.%s\".%s.%s values(null, 'a')", - metalakeName, catalogName, schemaName, tableName))); + "insert into %s.%s.%s values(null, 'a')", + catalogName, schemaName, tableName))); Assertions.assertThrows( RuntimeException.class, () -> @@ -670,8 +654,8 @@ void testColumnTypeNotNullByTrino() throws InterruptedException { .getTrinoContainer() .executeUpdateSQL( String.format( - "insert into \"%s.%s\".%s.%s values(1, null)", - metalakeName, catalogName, schemaName, tableName))); + "insert into %s.%s.%s values(1, null)", + catalogName, schemaName, tableName))); Assertions.assertThrows( RuntimeException.class, () -> @@ -679,8 +663,8 @@ void testColumnTypeNotNullByTrino() throws InterruptedException { .getTrinoContainer() .executeUpdateSQL( String.format( - "insert into \"%s.%s\".%s.%s values(null, null)", - metalakeName, catalogName, schemaName, tableName))); + "insert into %s.%s.%s values(null, null)", + catalogName, schemaName, tableName))); catalog .asTableCatalog() @@ -749,9 +733,7 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); Assertions.assertNotNull(table); - String sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + String sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load table created by gravitino: " + sql); @@ -777,13 +759,11 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { tableName = GravitinoITUtils.genRandomName("table_format1").toLowerCase(); sql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar) with (format = 'ORC')", - metalakeName, catalogName, schemaName, tableName); + "CREATE TABLE %s.%s.%s (id int, name varchar) with (format = 'ORC')", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30), "Trino fail to create table:" + tableName); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); @@ -797,12 +777,10 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { tableName = GravitinoITUtils.genRandomName("table_format2").toLowerCase(); sql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar) with (format = 'ORC', input_format = 'org.apache.hadoop.mapred.TextInputFormat')", - metalakeName, catalogName, schemaName, tableName); + "CREATE TABLE %s.%s.%s (id int, name varchar) with (format = 'ORC', input_format = 'org.apache.hadoop.mapred.TextInputFormat')", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30), "Trino fail to create table:" + tableName); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); @@ -816,12 +794,10 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { tableName = GravitinoITUtils.genRandomName("table_format3").toLowerCase(); sql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar) with (format = 'ORC', output_format = 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat')", - metalakeName, catalogName, schemaName, tableName); + "CREATE TABLE %s.%s.%s (id int, name varchar) with (format = 'ORC', output_format = 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat')", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30), "Trino fail to create table:" + tableName); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); @@ -860,7 +836,7 @@ void testHiveCatalogCreatedByGravitino() throws InterruptedException { Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); @@ -868,7 +844,7 @@ void testHiveCatalogCreatedByGravitino() throws InterruptedException { // Because we assign 'hive.target-max-file-size' a wrong value, trino can't load the catalog String data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); - Assertions.assertEquals(metalakeName + "." + catalogName, data); + Assertions.assertEquals(catalogName, data); } @Test @@ -899,7 +875,7 @@ void testWrongHiveCatalogProperty() throws InterruptedException { Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); checkTrinoHasLoaded(sql, 6); // Because we assign 'hive.target-max-file-size' a wrong value, trino can't load the catalog Assertions.assertTrue(containerSuite.getTrinoContainer().executeQuerySQL(sql).isEmpty()); @@ -966,9 +942,7 @@ void testIcebergTableAndSchemaCreatedByGravitino() throws InterruptedException { .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); Assertions.assertNotNull(table); - String sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + String sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { @@ -985,8 +959,8 @@ void testIcebergTableAndSchemaCreatedByGravitino() throws InterruptedException { String tableCreatedByTrino = GravitinoITUtils.genRandomName("table").toLowerCase(); String createTableSql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar) with (partitioning = ARRAY['name'], sorted_by = ARRAY['id'])", - metalakeName, catalogName, schemaName, tableCreatedByTrino); + "CREATE TABLE %s.%s.%s (id int, name varchar) with (partitioning = ARRAY['name'], sorted_by = ARRAY['id'])", + catalogName, schemaName, tableCreatedByTrino); containerSuite.getTrinoContainer().executeUpdateSQL(createTableSql); table = @@ -1007,14 +981,12 @@ void testIcebergTableAndSchemaCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); - String createSchemaSql = - String.format("CREATE SCHEMA \"%s.%s\".%s", metalakeName, catalogName, schemaName); + String createSchemaSql = String.format("CREATE SCHEMA %s.%s", catalogName, schemaName); containerSuite.getTrinoContainer().executeUpdateSQL(createSchemaSql); String createTableSql = String.format( - "CREATE TABLE \"%s.%s\".%s.%s (id int, name varchar)", - metalakeName, catalogName, schemaName, tableName); + "CREATE TABLE %s.%s.%s (id int, name varchar)", catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(createTableSql); Table table = @@ -1061,14 +1033,14 @@ void testIcebergCatalogCreatedByGravitino() throws InterruptedException { Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); Assertions.assertEquals("root", catalog.properties().get("jdbc-user")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); } String data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); - Assertions.assertEquals(metalakeName + "." + catalogName, data); + Assertions.assertEquals(catalogName, data); catalog .asSchemas() @@ -1077,12 +1049,10 @@ void testIcebergCatalogCreatedByGravitino() throws InterruptedException { "Created by gravitino client", ImmutableMap.builder().build()); - sql = - String.format("show schemas in \"%s.%s\" like '%s'", metalakeName, catalogName, schemaName); + sql = String.format("show schemas in %s like '%s'", catalogName, schemaName); Assertions.assertTrue(checkTrinoHasLoaded(sql, 30)); - final String sql1 = - String.format("drop schema \"%s.%s\".%s cascade", metalakeName, catalogName, schemaName); + final String sql1 = String.format("drop schema %s.%s cascade", catalogName, schemaName); // Will fail because the iceberg catalog does not support cascade drop TrinoContainer trinoContainer = containerSuite.getTrinoContainer(); Assertions.assertThrows( @@ -1091,8 +1061,7 @@ void testIcebergCatalogCreatedByGravitino() throws InterruptedException { trinoContainer.executeUpdateSQL(sql1); }); - final String sql2 = - String.format("show schemas in \"%s.%s\" like '%s'", metalakeName, catalogName, schemaName); + final String sql2 = String.format("show schemas in %s like '%s'", catalogName, schemaName); success = checkTrinoHasLoaded(sql2, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql2); @@ -1104,8 +1073,7 @@ void testIcebergCatalogCreatedByGravitino() throws InterruptedException { .asSchemas() .dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), true); Assertions.assertFalse(success); - final String sql3 = - String.format("show schemas in \"%s.%s\" like '%s'", metalakeName, catalogName, schemaName); + final String sql3 = String.format("show schemas in %s like '%s'", catalogName, schemaName); success = checkTrinoHasLoaded(sql3, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); @@ -1143,14 +1111,14 @@ void testMySQLCatalogCreatedByGravitino() throws InterruptedException { Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); Assertions.assertEquals("root", catalog.properties().get("jdbc-user")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); } String data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); - Assertions.assertEquals(metalakeName + "." + catalogName, data); + Assertions.assertEquals(catalogName, data); } @Test @@ -1186,14 +1154,14 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); Assertions.assertEquals("root", catalog.properties().get("jdbc-user")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); } String data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); - Assertions.assertEquals(metalakeName + "." + catalogName, data); + Assertions.assertEquals(catalogName, data); Schema schema = catalog @@ -1220,9 +1188,7 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); Assertions.assertNotNull(table); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); success = checkTrinoHasLoaded(sql, 30); if (!success) { @@ -1256,9 +1222,7 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { new Index[] { Indexes.createMysqlPrimaryKey(new String[][] {new String[] {"IntegerType"}}) }); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); success = checkTrinoHasLoaded(sql, 30); if (!success) { @@ -1304,46 +1268,43 @@ void testMySQLTableCreatedByTrino() throws InterruptedException { Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); Assertions.assertEquals("root", catalog.properties().get("jdbc-user")); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); if (!success) { Assertions.fail("Trino fail to load catalogs created by gravitino: " + sql); } String data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); - Assertions.assertEquals(metalakeName + "." + catalogName, data); + Assertions.assertEquals(catalogName, data); // Create schema - sql = String.format("create schema \"%s.%s\".%s", metalakeName, catalogName, schemaName); + sql = String.format("create schema %s.%s", catalogName, schemaName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); // create table sql = String.format( - "create table \"%s.%s\".%s.%s (id int, name varchar)", - metalakeName, catalogName, schemaName, tableName); + "create table %s.%s.%s (id int, name varchar)", catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); // Add a not null column sql = String.format( - "alter table \"%s.%s\".%s.%s add column age int not null comment 'age of users'", - metalakeName, catalogName, schemaName, tableName); + "alter table %s.%s.%s add column age int not null comment 'age of users'", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); sql = String.format( - "alter table \"%s.%s\".%s.%s add column address varchar(20) not null comment 'address of users'", - metalakeName, catalogName, schemaName, tableName); + "alter table %s.%s.%s add column address varchar(20) not null comment 'address of users'", + catalogName, schemaName, tableName); containerSuite.getTrinoContainer().executeUpdateSQL(sql); catalog .asTableCatalog() .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); Assertions.assertTrue(data.contains("age integer NOT NULL")); @@ -1353,57 +1314,45 @@ void testMySQLTableCreatedByTrino() throws InterruptedException { String tableName1 = "t112"; sql = String.format( - "create table \"%s.%s\".%s.%s (id int, t1name varchar)", - metalakeName, catalogName, schemaName, tableName1); + "create table %s.%s.%s (id int, t1name varchar)", catalogName, schemaName, tableName1); containerSuite.getTrinoContainer().executeUpdateSQL(sql); String tableName2 = "t212"; sql = String.format( - "create table \"%s.%s\".%s.%s (id int, t2name varchar)", - metalakeName, catalogName, schemaName, tableName2); + "create table %s.%s.%s (id int, t2name varchar)", catalogName, schemaName, tableName2); containerSuite.getTrinoContainer().executeUpdateSQL(sql); String tableName3 = "t_12"; sql = String.format( - "create table \"%s.%s\".%s.%s (id int, t3name varchar)", - metalakeName, catalogName, schemaName, tableName3); + "create table %s.%s.%s (id int, t3name varchar)", catalogName, schemaName, tableName3); containerSuite.getTrinoContainer().executeUpdateSQL(sql); String tableName4 = "_1__"; sql = String.format( - "create table \"%s.%s\".%s.%s (id int, t4name varchar)", - metalakeName, catalogName, schemaName, tableName4); + "create table %s.%s.%s (id int, t4name varchar)", catalogName, schemaName, tableName4); containerSuite.getTrinoContainer().executeUpdateSQL(sql); // Get table tableName1 - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName1); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName1); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); data.contains("t1name varchar"); // Get table tableName2 - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName2); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName2); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); data.contains("t2name varchar"); // Get table tableName3 - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName3); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName3); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); data.contains("t3name varchar"); // Get table tableName4 - sql = - String.format( - "show create table \"%s.%s\".%s.%s", metalakeName, catalogName, schemaName, tableName4); + sql = String.format("show create table %s.%s.%s", catalogName, schemaName, tableName4); data = containerSuite.getTrinoContainer().executeQuerySQL(sql).get(0).get(0); data.contains("t4name varchar"); } @@ -1440,7 +1389,7 @@ void testDropCatalogAndCreateAgain() throws InterruptedException { .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) .build()); - String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); Assertions.assertTrue(success, "Trino should load the catalog: " + sql); diff --git a/integration-test/src/test/resources/trino-ci-testset/bugs/00002_alter_table_mysql.sql b/integration-test/src/test/resources/trino-ci-testset/bugs/00002_alter_table_mysql.sql deleted file mode 100644 index b26d9fe6828..00000000000 --- a/integration-test/src/test/resources/trino-ci-testset/bugs/00002_alter_table_mysql.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE SCHEMA "test.jdbc-mysql".gt_db1; - -CREATE TABLE "test.jdbc-mysql".gt_db1.tb01 ( - name varchar, - salary int -); - -alter table "test.jdbc-mysql".gt_db1.tb01 rename column name to s; -show tables from "test.jdbc-mysql".gt_db1; - -comment on table "test.jdbc-mysql".gt_db1.tb01 is 'test table comments'; -show create table "test.jdbc-mysql".gt_db1.tb01; - -comment on column "test.jdbc-mysql".gt_db1.tb01.s is 'test column comments'; -show create table "test.jdbc-mysql".gt_db1.tb01; - diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00000_create_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00000_create_table.sql index 5df0dda6cd9..e37f3fb7365 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00000_create_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00000_create_table.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_hive".gt_db1; +CREATE SCHEMA gt_hive.gt_db1; -CREATE TABLE "test.gt_hive".gt_db1.tb01 ( +CREATE TABLE gt_hive.gt_db1.tb01 ( name varchar, salary int ) @@ -8,6 +8,6 @@ WITH ( format = 'TEXTFILE' ); -drop table "test.gt_hive".gt_db1.tb01; +drop table gt_hive.gt_db1.tb01; -drop schema "test.gt_hive".gt_db1; \ No newline at end of file +drop schema gt_hive.gt_db1; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00001_select_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00001_select_table.sql index 5bd054d44d9..4057536a827 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00001_select_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00001_select_table.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_hive".gt_db1; +CREATE SCHEMA gt_hive.gt_db1; -CREATE TABLE "test.gt_hive".gt_db1.tb01 ( +CREATE TABLE gt_hive.gt_db1.tb01 ( name varchar, salary int ) @@ -8,13 +8,13 @@ WITH ( format = 'TEXTFILE' ); -insert into "test.gt_hive".gt_db1.tb01(name, salary) values ('sam', 11); -insert into "test.gt_hive".gt_db1.tb01(name, salary) values ('jerry', 13); -insert into "test.gt_hive".gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); +insert into gt_hive.gt_db1.tb01(name, salary) values ('sam', 11); +insert into gt_hive.gt_db1.tb01(name, salary) values ('jerry', 13); +insert into gt_hive.gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); -select * from "test.gt_hive".gt_db1.tb01 order by name; +select * from gt_hive.gt_db1.tb01 order by name; -CREATE TABLE "test.gt_hive".gt_db1.tb02 ( +CREATE TABLE gt_hive.gt_db1.tb02 ( name varchar, salary int ) @@ -22,12 +22,12 @@ WITH ( format = 'TEXTFILE' ); -insert into "test.gt_hive".gt_db1.tb02(name, salary) select distinct * from "test.gt_hive".gt_db1.tb01 order by name; +insert into gt_hive.gt_db1.tb02(name, salary) select * from gt_hive.gt_db1.tb01 order by name; -select * from "test.gt_hive".gt_db1.tb02 order by name; +select * from gt_hive.gt_db1.tb02 order by name; -drop table "test.gt_hive".gt_db1.tb02; +drop table gt_hive.gt_db1.tb02; -drop table "test.gt_hive".gt_db1.tb01; +drop table gt_hive.gt_db1.tb01; -drop schema "test.gt_hive".gt_db1; +drop schema gt_hive.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.sql index dfca4f16e19..9c9e29b4ea8 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_hive".gt_db1; +CREATE SCHEMA gt_hive.gt_db1; -CREATE TABLE "test.gt_hive".gt_db1.tb01 ( +CREATE TABLE gt_hive.gt_db1.tb01 ( name varchar, salary int, city int @@ -9,30 +9,30 @@ WITH ( format = 'TEXTFILE' ); -alter table "test.gt_hive".gt_db1.tb01 rename to "test.gt_hive".gt_db1.tb03; -show tables from "test.gt_hive".gt_db1; +alter table gt_hive.gt_db1.tb01 rename to gt_hive.gt_db1.tb03; +show tables from gt_hive.gt_db1; -alter table "test.gt_hive".gt_db1.tb03 rename to "test.gt_hive".gt_db1.tb01; -show tables from "test.gt_hive".gt_db1; +alter table gt_hive.gt_db1.tb03 rename to gt_hive.gt_db1.tb01; +show tables from gt_hive.gt_db1; -alter table "test.gt_hive".gt_db1.tb01 drop column city; -show create table "test.gt_hive".gt_db1.tb01; +alter table gt_hive.gt_db1.tb01 drop column city; +show create table gt_hive.gt_db1.tb01; -alter table "test.gt_hive".gt_db1.tb01 rename column name to s; -show create table "test.gt_hive".gt_db1.tb01; +alter table gt_hive.gt_db1.tb01 rename column name to s; +show create table gt_hive.gt_db1.tb01; -alter table "test.gt_hive".gt_db1.tb01 alter column s set data type varchar(256); -show create table "test.gt_hive".gt_db1.tb01; +alter table gt_hive.gt_db1.tb01 alter column s set data type varchar(256); +show create table gt_hive.gt_db1.tb01; -comment on table "test.gt_hive".gt_db1.tb01 is 'test table comments'; -show create table "test.gt_hive".gt_db1.tb01; +comment on table gt_hive.gt_db1.tb01 is 'test table comments'; +show create table gt_hive.gt_db1.tb01; -comment on column "test.gt_hive".gt_db1.tb01.s is 'test column comments'; -show create table "test.gt_hive".gt_db1.tb01; +comment on column gt_hive.gt_db1.tb01.s is 'test column comments'; +show create table gt_hive.gt_db1.tb01; -alter table "test.gt_hive".gt_db1.tb01 add column city varchar comment 'aaa'; -show create table "test.gt_hive".gt_db1.tb01; +alter table gt_hive.gt_db1.tb01 add column city varchar comment 'aaa'; +show create table gt_hive.gt_db1.tb01; -drop table "test.gt_hive".gt_db1.tb01; +drop table gt_hive.gt_db1.tb01; -drop schema "test.gt_hive".gt_db1; +drop schema gt_hive.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.txt index c759dfe1c95..d07f2e32e9e 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00002_alter_table.txt @@ -12,7 +12,7 @@ RENAME TABLE DROP COLUMN -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( name varchar, salary integer ) @@ -30,7 +30,7 @@ WITH ( RENAME COLUMN -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( s varchar, salary integer ) @@ -48,7 +48,7 @@ WITH ( SET COLUMN TYPE -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( s varchar(256), salary integer ) @@ -66,7 +66,7 @@ WITH ( COMMENT -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( s varchar(256), salary integer ) @@ -84,7 +84,7 @@ WITH ( COMMENT -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( s varchar(256) COMMENT 'test column comments', salary integer ) @@ -102,7 +102,7 @@ WITH ( ADD COLUMN -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( s varchar(256) COMMENT 'test column comments', salary integer, city varchar COMMENT 'aaa' diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.sql index 91daf2a79a6..c33cba46da2 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.sql @@ -7,11 +7,11 @@ call gravitino.system.create_catalog( ) ); -show catalogs like 'test.gt_hive_xxx1'; +show catalogs like 'gt_hive_xxx1'; CALL gravitino.system.drop_catalog('gt_hive_xxx1'); -show catalogs like 'test.gt_hive_xxx1'; +show catalogs like 'gt_hive_xxx1'; call gravitino.system.create_catalog( 'gt_hive_xxx1', @@ -22,6 +22,6 @@ call gravitino.system.create_catalog( ) ); -show catalogs like 'test.gt_hive_xxx1'; +show catalogs like 'gt_hive_xxx1'; CALL gravitino.system.drop_catalog('gt_hive_xxx1'); diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.txt index 45ac31e08ab..a9fb382877a 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00005_catalog.txt @@ -1,6 +1,6 @@ CALL -"test.gt_hive_xxx1" +"gt_hive_xxx1" CALL @@ -8,6 +8,6 @@ CALL CALL -"test.gt_hive_xxx1" +"gt_hive_xxx1" CALL diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.sql index a58417f6710..45941886073 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_hive".gt_db1; +CREATE SCHEMA gt_hive.gt_db1; -USE "test.gt_hive".gt_db1; +USE gt_hive.gt_db1; -- Unsupported Type: TIME CREATE TABLE tb01 ( @@ -32,4 +32,4 @@ select * from tb01 order by f1; drop table tb01; -drop schema "test.gt_hive".gt_db1 cascade; +drop schema gt_hive.gt_db1 cascade; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.txt index e4269caa0f1..33bedfdc751 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00006_datatype.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_hive"".gt_db1.tb01 ( +"CREATE TABLE gt_hive.gt_db1.tb01 ( f1 varchar(200), f2 char(20), f3 varbinary, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.sql index 54dfd8b6f67..555f5d82b80 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.sql @@ -1,47 +1,47 @@ -CREATE SCHEMA "test.gt_hive".varchar_db1; +CREATE SCHEMA gt_hive.varchar_db1; -USE "test.gt_hive".varchar_db1; +USE gt_hive.varchar_db1; CREATE TABLE tb01 (id int, name char(20)); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb01; +SHOW CREATE TABLE gt_hive.varchar_db1.tb01; CREATE TABLE tb02 (id int, name char(255)); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb02; +SHOW CREATE TABLE gt_hive.varchar_db1.tb02; CREATE TABLE tb03 (id int, name char(256)); CREATE TABLE tb04 (id int, name varchar(250)); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb04; +SHOW CREATE TABLE gt_hive.varchar_db1.tb04; CREATE TABLE tb05 (id int, name varchar(65535)); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb05; +SHOW CREATE TABLE gt_hive.varchar_db1.tb05; CREATE TABLE tb06 (id int, name char); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb06; +SHOW CREATE TABLE gt_hive.varchar_db1.tb06; CREATE TABLE tb07 (id int, name varchar); -SHOW CREATE TABLE "test.gt_hive".varchar_db1.tb07; +SHOW CREATE TABLE gt_hive.varchar_db1.tb07; CREATE TABLE tb08 (id int, name varchar(65536)); -drop table "test.gt_hive".varchar_db1.tb01; +drop table gt_hive.varchar_db1.tb01; -drop table "test.gt_hive".varchar_db1.tb02; +drop table gt_hive.varchar_db1.tb02; -drop table "test.gt_hive".varchar_db1.tb04; +drop table gt_hive.varchar_db1.tb04; -drop table "test.gt_hive".varchar_db1.tb05; +drop table gt_hive.varchar_db1.tb05; -drop table "test.gt_hive".varchar_db1.tb06; +drop table gt_hive.varchar_db1.tb06; -drop table "test.gt_hive".varchar_db1.tb07; +drop table gt_hive.varchar_db1.tb07; -drop schema "test.gt_hive".varchar_db1; +drop schema gt_hive.varchar_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.txt index b69411fb752..77d6e774295 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00007_varchar.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb01 ( +"CREATE TABLE gt_hive.varchar_db1.tb01 ( id integer, name char(20) ) @@ -20,7 +20,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb02 ( +"CREATE TABLE gt_hive.varchar_db1.tb02 ( id integer, name char(255) ) @@ -38,7 +38,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb04 ( +"CREATE TABLE gt_hive.varchar_db1.tb04 ( id integer, name varchar(250) ) @@ -54,7 +54,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb05 ( +"CREATE TABLE gt_hive.varchar_db1.tb05 ( id integer, name varchar(65535) ) @@ -70,7 +70,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb06 ( +"CREATE TABLE gt_hive.varchar_db1.tb06 ( id integer, name char(1) ) @@ -86,7 +86,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_hive"".varchar_db1.tb07 ( +"CREATE TABLE gt_hive.varchar_db1.tb07 ( id integer, name varchar ) diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.sql index 09e50a1becb..e3804cde4a2 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.sql @@ -1,61 +1,61 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -SHOW SCHEMAS FROM "test.gt_mysql" like 'gt_db1'; +SHOW SCHEMAS FROM gt_mysql like 'gt_db1'; -SHOW CREATE SCHEMA "test.gt_mysql".gt_db1; +SHOW CREATE SCHEMA gt_mysql.gt_db1; -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -CREATE SCHEMA IF NOT EXISTS "test.gt_mysql".gt_db1; +CREATE SCHEMA IF NOT EXISTS gt_mysql.gt_db1; -CREATE SCHEMA IF NOT EXISTS "test.gt_mysql".gt_db2; +CREATE SCHEMA IF NOT EXISTS gt_mysql.gt_db2; -SHOW SCHEMAS FROM "test.gt_mysql" like 'gt_db2'; +SHOW SCHEMAS FROM gt_mysql like 'gt_db2'; -CREATE TABLE "test.gt_mysql".gt_db1.tb01 ( +CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary int ); -SHOW CREATE TABLE "test.gt_mysql".gt_db1.tb01; +SHOW CREATE TABLE gt_mysql.gt_db1.tb01; -SHOW tables FROM "test.gt_mysql".gt_db1 like 'tb01'; +SHOW tables FROM gt_mysql.gt_db1 like 'tb01'; -CREATE TABLE "test.gt_mysql".gt_db1.tb01 ( +CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary int ); -CREATE TABLE IF NOT EXISTS "test.gt_mysql".gt_db1.tb01 ( +CREATE TABLE IF NOT EXISTS gt_mysql.gt_db1.tb01 ( name varchar(200), salary int ); -CREATE TABLE IF NOT EXISTS "test.gt_mysql".gt_db1.tb02 ( +CREATE TABLE IF NOT EXISTS gt_mysql.gt_db1.tb02 ( name varchar(200), salary int ); -SHOW tables FROM "test.gt_mysql".gt_db1 like 'tb02'; +SHOW tables FROM gt_mysql.gt_db1 like 'tb02'; -DROP TABLE "test.gt_mysql".gt_db1.tb01; +DROP TABLE gt_mysql.gt_db1.tb01; -SHOW tables FROM "test.gt_mysql".gt_db1 like 'tb01'; +SHOW tables FROM gt_mysql.gt_db1 like 'tb01'; -DROP TABLE "test.gt_mysql".gt_db1.tb01; +DROP TABLE gt_mysql.gt_db1.tb01; -DROP TABLE IF EXISTS "test.gt_mysql".gt_db1.tb01; +DROP TABLE IF EXISTS gt_mysql.gt_db1.tb01; -DROP TABLE IF EXISTS "test.gt_mysql".gt_db1.tb02; +DROP TABLE IF EXISTS gt_mysql.gt_db1.tb02; -SHOW tables FROM "test.gt_mysql".gt_db1 like 'tb02'; +SHOW tables FROM gt_mysql.gt_db1 like 'tb02'; -DROP SCHEMA "test.gt_mysql".gt_db1; +DROP SCHEMA gt_mysql.gt_db1; -SHOW SCHEMAS FROM "test.gt_mysql" like 'gt_db1'; +SHOW SCHEMAS FROM gt_mysql like 'gt_db1'; -DROP SCHEMA IF EXISTS "test.gt_mysql".gt_db1; +DROP SCHEMA IF EXISTS gt_mysql.gt_db1; -DROP SCHEMA IF EXISTS "test.gt_mysql".gt_db2; +DROP SCHEMA IF EXISTS gt_mysql.gt_db2; -SHOW SCHEMAS FROM "test.gt_mysql" like 'gt_db2' +SHOW SCHEMAS FROM gt_mysql like 'gt_db2' diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.txt index 7b78fbe07b6..43da1446fc8 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00000_create_table.txt @@ -2,9 +2,9 @@ CREATE SCHEMA "gt_db1" -"CREATE SCHEMA ""test.gt_mysql"".gt_db1" +"CREATE SCHEMA gt_mysql.gt_db1" - Schema 'test.gt_mysql.gt_db1' already exists + Schema 'gt_mysql.gt_db1' already exists CREATE SCHEMA @@ -14,7 +14,7 @@ CREATE SCHEMA CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary integer ) @@ -25,7 +25,7 @@ WITH ( "tb01" - Table 'test.gt_mysql.gt_db1.tb01' already exists + Table 'gt_mysql.gt_db1.tb01' already exists CREATE TABLE @@ -37,7 +37,7 @@ DROP TABLE - Table 'test.gt_mysql.gt_db1.tb01' does not exist + Table 'gt_mysql.gt_db1.tb01' does not exist DROP TABLE diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00001_select_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00001_select_table.sql index 51fc9066bdd..ddf2105de46 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00001_select_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00001_select_table.sql @@ -1,29 +1,29 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -CREATE TABLE "test.gt_mysql".gt_db1.tb01 ( +CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary int ); -insert into "test.gt_mysql".gt_db1.tb01(name, salary) values ('sam', 11); -insert into "test.gt_mysql".gt_db1.tb01(name, salary) values ('jerry', 13); -insert into "test.gt_mysql".gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); +insert into gt_mysql.gt_db1.tb01(name, salary) values ('sam', 11); +insert into gt_mysql.gt_db1.tb01(name, salary) values ('jerry', 13); +insert into gt_mysql.gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); -select * from "test.gt_mysql".gt_db1.tb01 order by name; +select * from gt_mysql.gt_db1.tb01 order by name; -CREATE TABLE "test.gt_mysql".gt_db1.tb02 ( +CREATE TABLE gt_mysql.gt_db1.tb02 ( name varchar(200), salary int ); -insert into "test.gt_mysql".gt_db1.tb02(name, salary) select distinct * from "test.gt_mysql".gt_db1.tb01 order by name; +insert into gt_mysql.gt_db1.tb02(name, salary) select * from gt_mysql.gt_db1.tb01 order by name; -select * from "test.gt_mysql".gt_db1.tb02 order by name; +select * from gt_mysql.gt_db1.tb02 order by name; -select * from "test.gt_mysql".gt_db1.tb01 join "test.gt_mysql".gt_db1.tb02 t on tb01.salary = t.salary order by tb01.name; +select * from gt_mysql.gt_db1.tb01 join gt_mysql.gt_db1.tb02 t on tb01.salary = t.salary order by tb01.name; -drop table "test.gt_mysql".gt_db1.tb02; +drop table gt_mysql.gt_db1.tb02; -drop table "test.gt_mysql".gt_db1.tb01; +drop table gt_mysql.gt_db1.tb01; -drop schema "test.gt_mysql".gt_db1; +drop schema gt_mysql.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql index 0ce7326cadf..b3af09a6580 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql @@ -1,42 +1,42 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -CREATE TABLE "test.gt_mysql".gt_db1.tb01 ( +CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary int, city int ); -alter table "test.gt_mysql".gt_db1.tb01 rename to "test.gt_mysql".gt_db1.tb03; -show tables from "test.gt_mysql".gt_db1; +alter table gt_mysql.gt_db1.tb01 rename to gt_mysql.gt_db1.tb03; +show tables from gt_mysql.gt_db1; -alter table "test.gt_mysql".gt_db1.tb03 rename to "test.gt_mysql".gt_db1.tb01; -show tables from "test.gt_mysql".gt_db1; +alter table gt_mysql.gt_db1.tb03 rename to gt_mysql.gt_db1.tb01; +show tables from gt_mysql.gt_db1; -alter table "test.gt_mysql".gt_db1.tb01 drop column city; -show create table "test.gt_mysql".gt_db1.tb01; +alter table gt_mysql.gt_db1.tb01 drop column city; +show create table gt_mysql.gt_db1.tb01; -alter table "test.gt_mysql".gt_db1.tb01 alter column salary set data type bigint; -show create table "test.gt_mysql".gt_db1.tb01; +alter table gt_mysql.gt_db1.tb01 alter column salary set data type bigint; +show create table gt_mysql.gt_db1.tb01; -comment on column "test.gt_mysql".gt_db1.tb01.name is 'test column comments'; -show create table "test.gt_mysql".gt_db1.tb01; +comment on column gt_mysql.gt_db1.tb01.name is 'test column comments'; +show create table gt_mysql.gt_db1.tb01; -comment on table "test.gt_mysql".gt_db1.tb01 is 'test table comments'; -show create table "test.gt_mysql".gt_db1.tb01; +comment on table gt_mysql.gt_db1.tb01 is 'test table comments'; +show create table gt_mysql.gt_db1.tb01; -alter table "test.gt_mysql".gt_db1.tb01 rename column name to s; -show create table "test.gt_mysql".gt_db1.tb01; +alter table gt_mysql.gt_db1.tb01 rename column name to s; +show create table gt_mysql.gt_db1.tb01; --- alter table "test.gt_mysql".gt_db1.tb01 add column city varchar(50) not null comment 'aaa'; -alter table "test.gt_mysql".gt_db1.tb01 add column city varchar(50) comment 'aaa'; -show create table "test.gt_mysql".gt_db1.tb01; +-- alter table gt_mysql.gt_db1.tb01 add column city varchar(50) not null comment 'aaa'; +alter table gt_mysql.gt_db1.tb01 add column city varchar(50) comment 'aaa'; +show create table gt_mysql.gt_db1.tb01; -alter table "test.gt_mysql".gt_db1.tb01 add column age int not null comment 'age of users'; -show create table "test.gt_mysql".gt_db1.tb01; +alter table gt_mysql.gt_db1.tb01 add column age int not null comment 'age of users'; +show create table gt_mysql.gt_db1.tb01; -alter table "test.gt_mysql".gt_db1.tb01 add column address varchar(200) not null comment 'address of users'; -show create table "test.gt_mysql".gt_db1.tb01; +alter table gt_mysql.gt_db1.tb01 add column address varchar(200) not null comment 'address of users'; +show create table gt_mysql.gt_db1.tb01; -drop table "test.gt_mysql".gt_db1.tb01; +drop table gt_mysql.gt_db1.tb01; -drop schema "test.gt_mysql".gt_db1; +drop schema gt_mysql.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt index 2ace13f5db0..3aa3144935c 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt @@ -12,7 +12,7 @@ RENAME TABLE DROP COLUMN -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary integer ) @@ -23,7 +23,7 @@ WITH ( SET COLUMN TYPE -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200), salary bigint ) @@ -34,7 +34,7 @@ WITH ( COMMENT -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200) COMMENT 'test column comments', salary bigint ) @@ -45,7 +45,7 @@ WITH ( COMMENT -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( name varchar(200) COMMENT 'test column comments', salary bigint ) @@ -56,7 +56,7 @@ WITH ( RENAME COLUMN -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( s varchar(200) COMMENT 'test column comments', salary bigint ) @@ -67,7 +67,7 @@ WITH ( ADD COLUMN -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( s varchar(200) COMMENT 'test column comments', salary bigint, city varchar(50) COMMENT 'aaa' @@ -79,7 +79,7 @@ WITH ( ADD COLUMN -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( s varchar(200) COMMENT 'test column comments', salary bigint, city varchar(50) COMMENT 'aaa', @@ -92,7 +92,7 @@ WITH ( ADD COLUMN -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( s varchar(200) COMMENT 'test column comments', salary bigint, city varchar(50) COMMENT 'aaa', diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00003_use.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00003_use.sql index 0b568e506a2..7d2594b0579 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00003_use.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00003_use.sql @@ -1,13 +1,13 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -USE "test.gt_mysql".gt_db1; +USE gt_mysql.gt_db1; CREATE TABLE tb01 ( name varchar(200), salary int ); -show tables from "test.gt_mysql".gt_db1; +show tables from gt_mysql.gt_db1; show tables; @@ -15,11 +15,11 @@ use tpch.tiny; show tables; -USE "test.gt_mysql".gt_db1; +USE gt_mysql.gt_db1; show tables; drop table tb01; -drop schema "test.gt_mysql".gt_db1; +drop schema gt_mysql.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql index 770e5a25304..a0fb5a60813 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -USE "test.gt_mysql".gt_db1; +USE gt_mysql.gt_db1; CREATE TABLE customer ( custkey bigint NOT NULL, @@ -48,4 +48,4 @@ drop table customer; drop table orders; -drop schema "test.gt_mysql".gt_db1; +drop schema gt_mysql.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt index 3d4c52bc990..5e8e51a098d 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt @@ -12,35 +12,35 @@ INSERT: 15000 rows "Trino version: % % - └─ TableScan[table = test.gt_mysql:gt_db1.customer gt_db1.customer limit=10 columns=[custkey:bigint:BIGINT]] + └─ TableScan[table = gt_mysql:gt_db1.customer gt_db1.customer limit=10 columns=[custkey:bigint:BIGINT]] Layout: [custkey:bigint] % " "Trino version: % % - └─ ScanFilter[table = test.gt_mysql:gt_db1:customer, filterPredicate = ""$like""(""phone"", ""$literal$""(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAEAAAALAAAAAAsAAAAGAAAAJTIzNDIlAA==')))] + └─ ScanFilter[table = gt_mysql:gt_db1:customer, filterPredicate = ""$like""(""phone"", ""$literal$""(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAEAAAALAAAAAAsAAAAGAAAAJTIzNDIlAA==')))] Layout: [custkey:bigint, name:varchar(25), address:varchar(40), nationkey:bigint, phone:varchar(15), acctbal:decimal(12,2), mktsegment:varchar(10), comment:varchar(117)] % " "Trino version: % % - └─ TableScan[table = test.gt_mysql:Query[SELECT sum(`totalprice`) AS `_pfgnrtd_0` FROM `gt_db1`.`orders`] columns=[_pfgnrtd_0:decimal(38,2):decimal]] + └─ TableScan[table = gt_mysql:Query[SELECT sum(`totalprice`) AS `_pfgnrtd_0` FROM `gt_db1`.`orders`] columns=[_pfgnrtd_0:decimal(38,2):decimal]] Layout: [_pfgnrtd:decimal(38,2)] % " "Trino version: % % - └─ TableScan[table = test.gt_mysql:Query[SELECT `orderdate`, sum(`totalprice`) AS `_pfgnrtd_0` FROM `gt_db1`.`orders` GROUP BY `orderdate`] sortOrder=[orderdate:date:DATE ASC NULLS LAST] limit=10 columns=[orderdate:date:DATE, _pfgnrtd_0:decimal(38,2):decimal]] + └─ TableScan[table = gt_mysql:Query[SELECT `orderdate`, sum(`totalprice`) AS `_pfgnrtd_0` FROM `gt_db1`.`orders` GROUP BY `orderdate`] sortOrder=[orderdate:date:DATE ASC NULLS LAST] limit=10 columns=[orderdate:date:DATE, _pfgnrtd_0:decimal(38,2):decimal]] Layout: [orderdate:date, _pfgnrtd:decimal(38,2)] % " "Trino version: % % - └─ TableScan[table = test.gt_mysql:Query[SELECT % INNER JOIN %] limit=10 columns=%]] + └─ TableScan[table = gt_mysql:Query[SELECT % INNER JOIN %] limit=10 columns=%]] % " diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.sql index 8236cca3508..dec476d9590 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.sql @@ -7,7 +7,7 @@ call gravitino.system.create_catalog( ) ); -show catalogs like 'test.gt_mysql_xxx1'; +show catalogs like 'gt_mysql_xxx1'; call gravitino.system.create_catalog( 'gt_mysql_xxx1', @@ -30,7 +30,7 @@ call gravitino.system.create_catalog( CALL gravitino.system.drop_catalog('gt_mysql_xxx1'); -show catalogs like 'test.gt_mysql_xxx1'; +show catalogs like 'gt_mysql_xxx1'; CALL gravitino.system.drop_catalog('gt_mysql_xxx1'); @@ -45,9 +45,9 @@ call gravitino.system.create_catalog( ) ); -show catalogs like 'test.gt_mysql_xxx1'; +show catalogs like 'gt_mysql_xxx1'; CALL gravitino.system.drop_catalog( catalog => 'gt_mysql_xxx1', ignore_not_exist => true); -show catalogs like 'test.gt_mysql_xxx1'; +show catalogs like 'gt_mysql_xxx1'; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.txt index 198a59bacb9..d17eaa68f90 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00005_create_catalog.txt @@ -1,6 +1,6 @@ CALL -"test.gt_mysql_xxx1" +"gt_mysql_xxx1" Catalog test.gt_mysql_xxx1 already exists. @@ -16,7 +16,7 @@ CALL CALL -"test.gt_mysql_xxx1" +"gt_mysql_xxx1" CALL diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.sql index 531d5d9921c..f6f0ec755fb 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_mysql".gt_db1; +CREATE SCHEMA gt_mysql.gt_db1; -USE "test.gt_mysql".gt_db1; +USE gt_mysql.gt_db1; -- Unsupported Type: BOOLEAN CREATE TABLE tb01 ( @@ -68,4 +68,4 @@ drop table tb01; drop table tb02; -drop schema "test.gt_mysql".gt_db1 cascade; +drop schema gt_mysql.gt_db1 cascade; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.txt index d2629bda0d5..fb63415a854 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00006_datatype.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb01 ( +"CREATE TABLE gt_mysql.gt_db1.tb01 ( f1 varchar(200), f2 char(20), f3 varbinary, @@ -34,7 +34,7 @@ INSERT: 1 row CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".gt_db1.tb02 ( +"CREATE TABLE gt_mysql.gt_db1.tb02 ( f1 varchar(200) NOT NULL, f2 char(20) NOT NULL, f3 varbinary NOT NULL, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.sql index a0143719f78..b1277532b4d 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.sql @@ -1,44 +1,44 @@ -CREATE SCHEMA "test.gt_mysql".varchar_db1; +CREATE SCHEMA gt_mysql.varchar_db1; -USE "test.gt_mysql".varchar_db1; +USE gt_mysql.varchar_db1; CREATE TABLE tb01 (id int, name char(20)); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb01; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb01; CREATE TABLE tb02 (id int, name char(255)); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb02; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb02; CREATE TABLE tb03 (id int, name char(256)); CREATE TABLE tb04 (id int, name varchar(250)); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb04; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb04; CREATE TABLE tb05 (id int, name varchar(256)); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb05; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb05; CREATE TABLE tb06 (id int, name char); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb06; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb06; CREATE TABLE tb07 (id int, name varchar); -SHOW CREATE TABLE "test.gt_mysql".varchar_db1.tb07; +SHOW CREATE TABLE gt_mysql.varchar_db1.tb07; -drop table "test.gt_mysql".varchar_db1.tb01; +drop table gt_mysql.varchar_db1.tb01; -drop table "test.gt_mysql".varchar_db1.tb02; +drop table gt_mysql.varchar_db1.tb02; -drop table "test.gt_mysql".varchar_db1.tb04; +drop table gt_mysql.varchar_db1.tb04; -drop table "test.gt_mysql".varchar_db1.tb05; +drop table gt_mysql.varchar_db1.tb05; -drop table "test.gt_mysql".varchar_db1.tb06; +drop table gt_mysql.varchar_db1.tb06; -drop table "test.gt_mysql".varchar_db1.tb07; +drop table gt_mysql.varchar_db1.tb07; -drop schema "test.gt_mysql".varchar_db1; +drop schema gt_mysql.varchar_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.txt index cbad718d69b..a239050a776 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00007_varchar.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb01 ( +"CREATE TABLE gt_mysql.varchar_db1.tb01 ( id integer, name char(20) ) @@ -15,7 +15,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb02 ( +"CREATE TABLE gt_mysql.varchar_db1.tb02 ( id integer, name char(255) ) @@ -28,7 +28,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb04 ( +"CREATE TABLE gt_mysql.varchar_db1.tb04 ( id integer, name varchar(250) ) @@ -39,7 +39,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb05 ( +"CREATE TABLE gt_mysql.varchar_db1.tb05 ( id integer, name varchar(256) ) @@ -50,7 +50,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb06 ( +"CREATE TABLE gt_mysql.varchar_db1.tb06 ( id integer, name char(1) ) @@ -61,7 +61,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_mysql"".varchar_db1.tb07 ( +"CREATE TABLE gt_mysql.varchar_db1.tb07 ( id integer, name varchar ) diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql index dba2caf829e..46d8b8c8034 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql @@ -7,7 +7,7 @@ call gravitino.system.create_catalog( ) ); -select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1'; +select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', @@ -17,7 +17,7 @@ call gravitino.system.alter_catalog( ) ); -select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1'; +select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', @@ -25,7 +25,7 @@ call gravitino.system.alter_catalog( array['join-pushdown.strategy'] ); -select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1'; +select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( catalog => 'gt_mysql_xxx1', @@ -36,6 +36,6 @@ call gravitino.system.alter_catalog( remove_properties => array['test_key'] ); -select * from gravitino.system.catalog where name = 'test.gt_mysql_xxx1'; +select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.drop_catalog('gt_mysql_xxx1'); \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt index 6f62f062010..b0e3aac0dcc 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt @@ -1,17 +1,17 @@ CALL -"test.gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" CALL -"test.gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" CALL -"test.gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" CALL -"test.gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" +"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" CALL \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00000_create_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00000_create_table.sql index 0a2f81a7ef9..9705111cd44 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00000_create_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00000_create_table.sql @@ -1,10 +1,10 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -CREATE TABLE "test.gt_postgresql".gt_db1.tb01 ( +CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary int ); -drop table "test.gt_postgresql".gt_db1.tb01; +drop table gt_postgresql.gt_db1.tb01; -drop schema "test.gt_postgresql".gt_db1; \ No newline at end of file +drop schema gt_postgresql.gt_db1; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00001_select_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00001_select_table.sql index e595cfabd11..b084c6568b3 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00001_select_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00001_select_table.sql @@ -1,27 +1,27 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -CREATE TABLE "test.gt_postgresql".gt_db1.tb01 ( +CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary int ); -insert into "test.gt_postgresql".gt_db1.tb01(name, salary) values ('sam', 11); -insert into "test.gt_postgresql".gt_db1.tb01(name, salary) values ('jerry', 13); -insert into "test.gt_postgresql".gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); +insert into gt_postgresql.gt_db1.tb01(name, salary) values ('sam', 11); +insert into gt_postgresql.gt_db1.tb01(name, salary) values ('jerry', 13); +insert into gt_postgresql.gt_db1.tb01(name, salary) values ('bob', 14), ('tom', 12); -select * from "test.gt_postgresql".gt_db1.tb01 order by name; +select * from gt_postgresql.gt_db1.tb01 order by name; -CREATE TABLE "test.gt_postgresql".gt_db1.tb02 ( +CREATE TABLE gt_postgresql.gt_db1.tb02 ( name varchar, salary int ); -insert into "test.gt_postgresql".gt_db1.tb02(name, salary) select distinct * from "test.gt_postgresql".gt_db1.tb01 order by name; +insert into gt_postgresql.gt_db1.tb02(name, salary) select * from gt_postgresql.gt_db1.tb01 order by name; -select * from "test.gt_postgresql".gt_db1.tb02 order by name; +select * from gt_postgresql.gt_db1.tb02 order by name; -drop table "test.gt_postgresql".gt_db1.tb02; +drop table gt_postgresql.gt_db1.tb02; -drop table "test.gt_postgresql".gt_db1.tb01; +drop table gt_postgresql.gt_db1.tb01; -drop schema "test.gt_postgresql".gt_db1; \ No newline at end of file +drop schema gt_postgresql.gt_db1; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.sql index 96214720c43..da78532d10e 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.sql @@ -1,35 +1,35 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -CREATE TABLE "test.gt_postgresql".gt_db1.tb01 ( +CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary int, city int ); -alter table "test.gt_postgresql".gt_db1.tb01 rename to "test.gt_postgresql".gt_db1.tb03; -show tables from "test.gt_postgresql".gt_db1; +alter table gt_postgresql.gt_db1.tb01 rename to gt_postgresql.gt_db1.tb03; +show tables from gt_postgresql.gt_db1; -alter table "test.gt_postgresql".gt_db1.tb03 rename to "test.gt_postgresql".gt_db1.tb01; -show tables from "test.gt_postgresql".gt_db1; +alter table gt_postgresql.gt_db1.tb03 rename to gt_postgresql.gt_db1.tb01; +show tables from gt_postgresql.gt_db1; -alter table "test.gt_postgresql".gt_db1.tb01 drop column city; -show create table "test.gt_postgresql".gt_db1.tb01; +alter table gt_postgresql.gt_db1.tb01 drop column city; +show create table gt_postgresql.gt_db1.tb01; -alter table "test.gt_postgresql".gt_db1.tb01 alter column salary set data type bigint; -show create table "test.gt_postgresql".gt_db1.tb01; +alter table gt_postgresql.gt_db1.tb01 alter column salary set data type bigint; +show create table gt_postgresql.gt_db1.tb01; -comment on table "test.gt_postgresql".gt_db1.tb01 is 'test table comments'; -show create table "test.gt_postgresql".gt_db1.tb01; +comment on table gt_postgresql.gt_db1.tb01 is 'test table comments'; +show create table gt_postgresql.gt_db1.tb01; -alter table "test.gt_postgresql".gt_db1.tb01 rename column name to s; -show create table "test.gt_postgresql".gt_db1.tb01; +alter table gt_postgresql.gt_db1.tb01 rename column name to s; +show create table gt_postgresql.gt_db1.tb01; -comment on column "test.gt_postgresql".gt_db1.tb01.s is 'test column comments'; -show create table "test.gt_postgresql".gt_db1.tb01; +comment on column gt_postgresql.gt_db1.tb01.s is 'test column comments'; +show create table gt_postgresql.gt_db1.tb01; -alter table "test.gt_postgresql".gt_db1.tb01 add column city varchar comment 'aaa'; -show create table "test.gt_postgresql".gt_db1.tb01; +alter table gt_postgresql.gt_db1.tb01 add column city varchar comment 'aaa'; +show create table gt_postgresql.gt_db1.tb01; -drop table "test.gt_postgresql".gt_db1.tb01; +drop table gt_postgresql.gt_db1.tb01; -drop schema "test.gt_postgresql".gt_db1; +drop schema gt_postgresql.gt_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.txt index 72fb2754ff4..e34adddb64f 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00002_alter_table.txt @@ -12,7 +12,7 @@ RENAME TABLE DROP COLUMN -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary integer ) @@ -20,7 +20,7 @@ COMMENT ''" SET COLUMN TYPE -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary bigint ) @@ -28,7 +28,7 @@ COMMENT ''" COMMENT -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( name varchar, salary bigint ) @@ -36,7 +36,7 @@ COMMENT 'test table comments'" RENAME COLUMN -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( s varchar, salary bigint ) @@ -44,7 +44,7 @@ COMMENT 'test table comments'" COMMENT -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( s varchar COMMENT 'test column comments', salary bigint ) @@ -52,7 +52,7 @@ COMMENT 'test table comments'" ADD COLUMN -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( s varchar COMMENT 'test column comments', salary bigint, city varchar COMMENT 'aaa' diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql index aca90645814..a0c47d71625 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql @@ -1,15 +1,15 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -use "test.gt_postgresql".gt_db1; +use gt_postgresql.gt_db1; -CREATE TABLE "test.gt_postgresql".gt_db1.employee_performance ( +CREATE TABLE gt_postgresql.gt_db1.employee_performance ( employee_id integer, evaluation_date date, rating integer ) COMMENT 'comment'; -CREATE TABLE "test.gt_postgresql".gt_db1.employees ( +CREATE TABLE gt_postgresql.gt_db1.employees ( employee_id integer, department_id integer, job_title varchar(100), @@ -20,7 +20,7 @@ CREATE TABLE "test.gt_postgresql".gt_db1.employees ( ) COMMENT 'comment'; -INSERT INTO "test.gt_postgresql".gt_db1.employee_performance (employee_id, evaluation_date, rating) VALUES +INSERT INTO gt_postgresql.gt_db1.employee_performance (employee_id, evaluation_date, rating) VALUES (1, DATE '2018-02-24', 4), (1, DATE '2016-12-25', 7), (1, DATE '2023-04-07', 4), @@ -32,7 +32,7 @@ INSERT INTO "test.gt_postgresql".gt_db1.employee_performance (employee_id, evalu (3, DATE '2021-01-05', 6), (3, DATE '2014-10-24', 4); -INSERT INTO "test.gt_postgresql".gt_db1.employees (employee_id, department_id, job_title, given_name, family_name, birth_date, hire_date) VALUES +INSERT INTO gt_postgresql.gt_db1.employees (employee_id, department_id, job_title, given_name, family_name, birth_date, hire_date) VALUES (1, 1, 'Manager', 'Gregory', 'Smith', DATE '1968-04-15', DATE '2014-06-04'), (2, 1, 'Sales Assistant', 'Owen', 'Rivers', DATE '1988-08-13', DATE '2021-02-05'), (3, 1, 'Programmer', 'Avram', 'Lawrence', DATE '1969-11-21', DATE '2010-09-29'), @@ -48,14 +48,14 @@ SELECT given_name, family_name, rating -FROM "test.gt_postgresql".gt_db1.employee_performance AS p -JOIN "test.gt_postgresql".gt_db1.employees AS e +FROM gt_postgresql.gt_db1.employee_performance AS p +JOIN gt_postgresql.gt_db1.employees AS e ON p.employee_id = e.employee_id ORDER BY rating DESC, given_name LIMIT 10; -drop table "test.gt_postgresql".gt_db1.employee_performance; -drop table "test.gt_postgresql".gt_db1.employees; +drop table gt_postgresql.gt_db1.employee_performance; +drop table gt_postgresql.gt_db1.employees; -drop schema "test.gt_postgresql".gt_db1; \ No newline at end of file +drop schema gt_postgresql.gt_db1; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql index d5a16ec5880..59760a2a31e 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -use "test.gt_postgresql".gt_db1; +use gt_postgresql.gt_db1; CREATE TABLE customer ( custkey bigint NOT NULL, @@ -48,4 +48,4 @@ drop table customer; drop table orders; -drop schema "test.gt_postgresql".gt_db1;; +drop schema gt_postgresql.gt_db1;; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt index d5fe30a8584..a823283fe1e 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt @@ -12,35 +12,35 @@ INSERT: 15000 rows "Trino version: % % - └─ TableScan[table = test.gt_postgresql:gt_db1.customer gt_db1.customer limit=10 columns=[custkey:bigint:int8]] + └─ TableScan[table = gt_postgresql:gt_db1.customer gt_db1.customer limit=10 columns=[custkey:bigint:int8]] Layout: [custkey:bigint] % " "Trino version: % % - └─ TableScan[table = test.gt_postgresql:gt_db1.customer gt_db1.customer constraints=[ParameterizedExpression[expression=(""phone"") LIKE (?), parameters=[QueryParameter{jdbcType=Optional.empty, type=varchar(6), value=Optional[Slice{base=[B@%, baseOffset=0, length=6}]}]]] limit=10] + └─ TableScan[table = gt_postgresql:gt_db1.customer gt_db1.customer constraints=[ParameterizedExpression[expression=(""phone"") LIKE (?), parameters=[QueryParameter{jdbcType=Optional.empty, type=varchar(6), value=Optional[Slice{base=[B@%, baseOffset=0, length=6}]}]]] limit=10] Layout: [custkey:bigint, name:varchar(25), address:varchar(40), nationkey:bigint, phone:varchar(15), acctbal:decimal(12,2), mktsegment:varchar(10), comment:varchar(117)] % " "Trino version: % % - └─ TableScan[table = test.gt_postgresql:Query[SELECT sum(""totalprice"") AS ""_pfgnrtd_0"" FROM ""gt_db1"".""orders""] columns=[_pfgnrtd_0:decimal(38,2):decimal]] + └─ TableScan[table = gt_postgresql:Query[SELECT sum(""totalprice"") AS ""_pfgnrtd_0"" FROM ""gt_db1"".""orders""] columns=[_pfgnrtd_0:decimal(38,2):decimal]] Layout: [_pfgnrtd:decimal(38,2)] % " "Trino version: % % - └─ TableScan[table = test.gt_postgresql:Query[SELECT ""orderdate"", sum(""totalprice"") AS ""_pfgnrtd_0"" FROM ""gt_db1"".""orders"" GROUP BY ""orderdate""] sortOrder=[orderdate:date:date ASC NULLS LAST] limit=10 columns=[orderdate:date:date, _pfgnrtd_0:decimal(38,2):decimal]] + └─ TableScan[table = gt_postgresql:Query[SELECT ""orderdate"", sum(""totalprice"") AS ""_pfgnrtd_0"" FROM ""gt_db1"".""orders"" GROUP BY ""orderdate""] sortOrder=[orderdate:date:date ASC NULLS LAST] limit=10 columns=[orderdate:date:date, _pfgnrtd_0:decimal(38,2):decimal]] Layout: [orderdate:date, _pfgnrtd:decimal(38,2)] % " "Trino version: % % - TableScan[table = test.gt_postgresql:Query[SELECT % INNER JOIN %] limit=10 columns=%] + TableScan[table = gt_postgresql:Query[SELECT % INNER JOIN %] limit=10 columns=%] % " diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.sql index d67997d2b8c..0f3327f1045 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_postgresql".gt_db1; +CREATE SCHEMA gt_postgresql.gt_db1; -USE "test.gt_postgresql".gt_db1; +USE gt_postgresql.gt_db1; -- Unsupported Type: TINYINT CREATE TABLE tb01 ( @@ -69,4 +69,4 @@ drop table tb01; drop table tb02; -drop schema "test.gt_postgresql".gt_db1 cascade; +drop schema gt_postgresql.gt_db1 cascade; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.txt index 4c3789a3513..42a88b77053 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00006_datatype.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb01 ( +"CREATE TABLE gt_postgresql.gt_db1.tb01 ( f1 varchar(200), f2 char(20), f3 varbinary, @@ -31,7 +31,7 @@ INSERT: 1 row CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".gt_db1.tb02 ( +"CREATE TABLE gt_postgresql.gt_db1.tb02 ( f1 varchar(200) NOT NULL, f2 char(20) NOT NULL, f3 varbinary NOT NULL, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.sql index b21f99f9d2a..8e911501beb 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.sql @@ -1,46 +1,46 @@ -CREATE SCHEMA "test.gt_postgresql".varchar_db1; +CREATE SCHEMA gt_postgresql.varchar_db1; -USE "test.gt_postgresql".varchar_db1; +USE gt_postgresql.varchar_db1; CREATE TABLE tb01 (id int, name char(20)); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb01; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb01; CREATE TABLE tb02 (id int, name char(65536)); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb02; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb02; CREATE TABLE tb03 (id int, name char(65537)); CREATE TABLE tb04 (id int, name varchar(250)); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb04; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb04; CREATE TABLE tb05 (id int, name varchar(10485760)); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb05; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb05; CREATE TABLE tb06 (id int, name varchar(10485761)); CREATE TABLE tb06 (id int, name char); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb06; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb06; CREATE TABLE tb07 (id int, name varchar); -SHOW CREATE TABLE "test.gt_postgresql".varchar_db1.tb07; +SHOW CREATE TABLE gt_postgresql.varchar_db1.tb07; -drop table "test.gt_postgresql".varchar_db1.tb01; +drop table gt_postgresql.varchar_db1.tb01; -drop table "test.gt_postgresql".varchar_db1.tb02; +drop table gt_postgresql.varchar_db1.tb02; -drop table "test.gt_postgresql".varchar_db1.tb04; +drop table gt_postgresql.varchar_db1.tb04; -drop table "test.gt_postgresql".varchar_db1.tb05; +drop table gt_postgresql.varchar_db1.tb05; -drop table "test.gt_postgresql".varchar_db1.tb06; +drop table gt_postgresql.varchar_db1.tb06; -drop table "test.gt_postgresql".varchar_db1.tb07; +drop table gt_postgresql.varchar_db1.tb07; -drop schema "test.gt_postgresql".varchar_db1; +drop schema gt_postgresql.varchar_db1; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.txt index f7776c53bec..ad1e5b13084 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00007_varchar.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb01 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb01 ( id integer, name char(20) ) @@ -12,7 +12,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb02 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb02 ( id integer, name char(65536) ) @@ -22,7 +22,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb04 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb04 ( id integer, name varchar(250) ) @@ -30,7 +30,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb05 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb05 ( id integer, name varchar(10485760) ) @@ -40,7 +40,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb06 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb06 ( id integer, name char(1) ) @@ -48,7 +48,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_postgresql"".varchar_db1.tb07 ( +"CREATE TABLE gt_postgresql.varchar_db1.tb07 ( id integer, name varchar ) diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql index 83e47dda274..a2936fb1316 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql @@ -1,13 +1,13 @@ -CREATE SCHEMA "test.gt_iceberg".gt_db2; +CREATE SCHEMA gt_iceberg.gt_db2; -CREATE TABLE "test.gt_iceberg".gt_db2.tb01( +CREATE TABLE gt_iceberg.gt_db2.tb01( name varchar, salary int ); -show create table "test.gt_iceberg".gt_db2.tb01; +show create table gt_iceberg.gt_db2.tb01; -CREATE TABLE "test.gt_iceberg".gt_db2.tb02 ( +CREATE TABLE gt_iceberg.gt_db2.tb02 ( name varchar, salary int ) with ( @@ -15,9 +15,9 @@ CREATE TABLE "test.gt_iceberg".gt_db2.tb02 ( sorted_by = ARRAY['salary'] ); -show create table "test.gt_iceberg".gt_db2.tb02; +show create table gt_iceberg.gt_db2.tb02; -CREATE TABLE "test.gt_iceberg".gt_db2.tb03 ( +CREATE TABLE gt_iceberg.gt_db2.tb03 ( name varchar, salary int ) with ( @@ -25,7 +25,7 @@ CREATE TABLE "test.gt_iceberg".gt_db2.tb03 ( sorted_by = ARRAY['salary_wrong_name'] ); -CREATE TABLE "test.gt_iceberg".gt_db2.tb03 ( +CREATE TABLE gt_iceberg.gt_db2.tb03 ( name varchar, salary int ) with ( @@ -33,46 +33,46 @@ CREATE TABLE "test.gt_iceberg".gt_db2.tb03 ( sorted_by = ARRAY['name'] ); -show create table "test.gt_iceberg".gt_db2.tb03; +show create table gt_iceberg.gt_db2.tb03; -CREATE TABLE "test.gt_iceberg".gt_db2.tb04 ( +CREATE TABLE gt_iceberg.gt_db2.tb04 ( name varchar, salary int ) with ( sorted_by = ARRAY['name'] ); -show create table "test.gt_iceberg".gt_db2.tb04; +show create table gt_iceberg.gt_db2.tb04; -CREATE TABLE "test.gt_iceberg".gt_db2.tb05 ( +CREATE TABLE gt_iceberg.gt_db2.tb05 ( name varchar, salary int ) with ( partitioning = ARRAY['name'] ); -show create table "test.gt_iceberg".gt_db2.tb05; +show create table gt_iceberg.gt_db2.tb05; -CREATE TABLE "test.gt_iceberg".gt_db2.tb06 ( +CREATE TABLE gt_iceberg.gt_db2.tb06 ( name varchar, salary int ) with ( location = '${hdfs_uri}/user/iceberg/warehouse/TrinoQueryIT/gt_iceberg/gt_db2/tb06' ); -show create table "test.gt_iceberg".gt_db2.tb06; +show create table gt_iceberg.gt_db2.tb06; -drop table "test.gt_iceberg".gt_db2.tb01; +drop table gt_iceberg.gt_db2.tb01; -drop table "test.gt_iceberg".gt_db2.tb02; +drop table gt_iceberg.gt_db2.tb02; -drop table "test.gt_iceberg".gt_db2.tb03; +drop table gt_iceberg.gt_db2.tb03; -drop table "test.gt_iceberg".gt_db2.tb04; +drop table gt_iceberg.gt_db2.tb04; -drop table "test.gt_iceberg".gt_db2.tb05; +drop table gt_iceberg.gt_db2.tb05; -drop table "test.gt_iceberg".gt_db2.tb06; +drop table gt_iceberg.gt_db2.tb06; -drop schema "test.gt_iceberg".gt_db2; \ No newline at end of file +drop schema gt_iceberg.gt_db2; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt index a8fd39e3409..c0ba8a4044b 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt @@ -2,7 +2,7 @@ CREATE SCHEMA CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( name varchar, salary integer ) @@ -10,7 +10,7 @@ COMMENT ''" CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb02 ( +"CREATE TABLE gt_iceberg.gt_db2.tb02 ( name varchar, salary integer ) @@ -24,7 +24,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb03 ( +"CREATE TABLE gt_iceberg.gt_db2.tb03 ( name varchar, salary integer ) @@ -36,7 +36,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb04 ( +"CREATE TABLE gt_iceberg.gt_db2.tb04 ( name varchar, salary integer ) @@ -47,7 +47,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb05 ( +"CREATE TABLE gt_iceberg.gt_db2.tb05 ( name varchar, salary integer ) @@ -58,7 +58,7 @@ WITH ( CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb06 ( +"CREATE TABLE gt_iceberg.gt_db2.tb06 ( name varchar, salary integer ) diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00001_select_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00001_select_table.sql index 9ec231fd164..16c2b23ee27 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00001_select_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00001_select_table.sql @@ -1,28 +1,28 @@ -CREATE SCHEMA "test.gt_iceberg".gt_db2; +CREATE SCHEMA gt_iceberg.gt_db2; -CREATE TABLE "test.gt_iceberg".gt_db2.tb01 ( +CREATE TABLE gt_iceberg.gt_db2.tb01 ( name varchar, salary int ); -insert into "test.gt_iceberg".gt_db2.tb01(name, salary) values ('sam', 11); -insert into "test.gt_iceberg".gt_db2.tb01(name, salary) values ('jerry', 13); -insert into "test.gt_iceberg".gt_db2.tb01(name, salary) values ('bob', 14), ('tom', 12); +insert into gt_iceberg.gt_db2.tb01(name, salary) values ('sam', 11); +insert into gt_iceberg.gt_db2.tb01(name, salary) values ('jerry', 13); +insert into gt_iceberg.gt_db2.tb01(name, salary) values ('bob', 14), ('tom', 12); -select * from "test.gt_iceberg".gt_db2.tb01 order by name; +select * from gt_iceberg.gt_db2.tb01 order by name; -CREATE TABLE "test.gt_iceberg".gt_db2.tb02 ( +CREATE TABLE gt_iceberg.gt_db2.tb02 ( name varchar, salary int ); -insert into "test.gt_iceberg".gt_db2.tb02(name, salary) select distinct * from "test.gt_iceberg".gt_db2.tb01 order by name; +insert into gt_iceberg.gt_db2.tb02(name, salary) select * from gt_iceberg.gt_db2.tb01 order by name; -select * from "test.gt_iceberg".gt_db2.tb02 order by name; +select * from gt_iceberg.gt_db2.tb02 order by name; -drop table "test.gt_iceberg".gt_db2.tb02; +drop table gt_iceberg.gt_db2.tb02; -drop table "test.gt_iceberg".gt_db2.tb01; +drop table gt_iceberg.gt_db2.tb01; -drop schema "test.gt_iceberg".gt_db2; +drop schema gt_iceberg.gt_db2; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.sql index 3450a23a2d2..b31831e3570 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.sql @@ -1,35 +1,35 @@ -CREATE SCHEMA "test.gt_iceberg".gt_db2; +CREATE SCHEMA gt_iceberg.gt_db2; -CREATE TABLE "test.gt_iceberg".gt_db2.tb01 ( +CREATE TABLE gt_iceberg.gt_db2.tb01 ( name varchar, salary int, city int ); -alter table "test.gt_iceberg".gt_db2.tb01 rename to "test.gt_iceberg".gt_db2.tb03; -show tables from "test.gt_iceberg".gt_db2; +alter table gt_iceberg.gt_db2.tb01 rename to gt_iceberg.gt_db2.tb03; +show tables from gt_iceberg.gt_db2; -alter table "test.gt_iceberg".gt_db2.tb03 rename to "test.gt_iceberg".gt_db2.tb01; -show tables from "test.gt_iceberg".gt_db2; +alter table gt_iceberg.gt_db2.tb03 rename to gt_iceberg.gt_db2.tb01; +show tables from gt_iceberg.gt_db2; -alter table "test.gt_iceberg".gt_db2.tb01 drop column city; -show create table "test.gt_iceberg".gt_db2.tb01; +alter table gt_iceberg.gt_db2.tb01 drop column city; +show create table gt_iceberg.gt_db2.tb01; -alter table "test.gt_iceberg".gt_db2.tb01 rename column name to s; -show create table "test.gt_iceberg".gt_db2.tb01; +alter table gt_iceberg.gt_db2.tb01 rename column name to s; +show create table gt_iceberg.gt_db2.tb01; -alter table "test.gt_iceberg".gt_db2.tb01 alter column salary set data type bigint; -show create table "test.gt_iceberg".gt_db2.tb01; +alter table gt_iceberg.gt_db2.tb01 alter column salary set data type bigint; +show create table gt_iceberg.gt_db2.tb01; -comment on table "test.gt_iceberg".gt_db2.tb01 is 'test table comments'; -show create table "test.gt_iceberg".gt_db2.tb01; +comment on table gt_iceberg.gt_db2.tb01 is 'test table comments'; +show create table gt_iceberg.gt_db2.tb01; -comment on column "test.gt_iceberg".gt_db2.tb01.s is 'test column comments'; -show create table "test.gt_iceberg".gt_db2.tb01; +comment on column gt_iceberg.gt_db2.tb01.s is 'test column comments'; +show create table gt_iceberg.gt_db2.tb01; -alter table "test.gt_iceberg".gt_db2.tb01 add column city varchar comment 'aaa'; -show create table "test.gt_iceberg".gt_db2.tb01; +alter table gt_iceberg.gt_db2.tb01 add column city varchar comment 'aaa'; +show create table gt_iceberg.gt_db2.tb01; -drop table "test.gt_iceberg".gt_db2.tb01; +drop table gt_iceberg.gt_db2.tb01; -drop schema "test.gt_iceberg".gt_db2; +drop schema gt_iceberg.gt_db2; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt index e979156e512..969d40b0eb8 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt @@ -12,7 +12,7 @@ RENAME TABLE DROP COLUMN -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( name varchar, salary integer ) @@ -20,7 +20,7 @@ COMMENT ''" RENAME COLUMN -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( s varchar, salary integer ) @@ -28,7 +28,7 @@ COMMENT ''" SET COLUMN TYPE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( s varchar, salary bigint ) @@ -36,7 +36,7 @@ COMMENT ''" COMMENT -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( s varchar, salary bigint ) @@ -44,7 +44,7 @@ COMMENT 'test table comments'" COMMENT -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( s varchar COMMENT 'test column comments', salary bigint ) @@ -52,7 +52,7 @@ COMMENT 'test table comments'" ADD COLUMN -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( s varchar COMMENT 'test column comments', salary bigint, city varchar COMMENT 'aaa' diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql index c3d7890550a..72854588809 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_iceberg".gt_db2; +CREATE SCHEMA gt_iceberg.gt_db2; -USE "test.gt_iceberg".gt_db2; +USE gt_iceberg.gt_db2; -- Unsupported Type: TINYINT, SMALLINT CREATE TABLE tb01 ( @@ -64,4 +64,4 @@ drop table tb01; drop table tb02; -drop schema "test.gt_iceberg".gt_db2; +drop schema gt_iceberg.gt_db2; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt index 8ff0979aa96..915539adaf5 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt @@ -4,7 +4,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( +"CREATE TABLE gt_iceberg.gt_db2.tb01 ( f1 varchar, f3 varbinary, f4 decimal(10, 3), @@ -30,7 +30,7 @@ INSERT: 1 row CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".gt_db2.tb02 ( +"CREATE TABLE gt_iceberg.gt_db2.tb02 ( f1 varchar NOT NULL, f3 varbinary NOT NULL, f4 decimal(10, 3) NOT NULL, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql index 3e19bf5a15b..ab17351747b 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA "test.gt_iceberg".varchar_db2; +CREATE SCHEMA gt_iceberg.varchar_db2; -USE "test.gt_iceberg".varchar_db2; +USE gt_iceberg.varchar_db2; CREATE TABLE tb01 (id int, name char(20)); @@ -10,9 +10,9 @@ CREATE TABLE tb03 (id int, name varchar(233)); CREATE TABLE tb04 (id int, name varchar); -SHOW CREATE TABLE "test.gt_iceberg".varchar_db2.tb04; +SHOW CREATE TABLE gt_iceberg.varchar_db2.tb04; -drop table "test.gt_iceberg".varchar_db2.tb04; +drop table gt_iceberg.varchar_db2.tb04; -drop schema "test.gt_iceberg".varchar_db2; +drop schema gt_iceberg.varchar_db2; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt index 58e38b89f4a..c7f7ab14e44 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt @@ -10,7 +10,7 @@ USE CREATE TABLE -"CREATE TABLE ""test.gt_iceberg"".varchar_db2.tb04 ( +"CREATE TABLE gt_iceberg.varchar_db2.tb04 ( id integer, name varchar ) diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpcds/catalog_mysql_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpcds/catalog_mysql_prepare.sql index 81ca5c14654..86bd2f72a6e 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpcds/catalog_mysql_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpcds/catalog_mysql_prepare.sql @@ -9,8 +9,8 @@ call gravitino.system.create_catalog( show catalogs; -create schema "test.gt_mysql1".gt_tpcds; -use "test.gt_mysql1".gt_tpcds; +create schema gt_mysql1.gt_tpcds; +use gt_mysql1.gt_tpcds; CREATE TABLE call_center ( cc_call_center_sk bigint, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_hive_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_hive_prepare.sql index cc015bccbdf..2fb4294af7d 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_hive_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_hive_prepare.sql @@ -7,8 +7,8 @@ call gravitino.system.create_catalog( ) ); -create schema "test.gt_hive2".gt_tpch; -use "test.gt_hive2".gt_tpch; +create schema gt_hive2.gt_tpch; +use gt_hive2.gt_tpch; CREATE TABLE customer ( custkey bigint, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql index dfd439612d3..a5a0669f839 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql @@ -7,8 +7,8 @@ call gravitino.system.create_catalog( ) ); -create schema "test.gt_iceberg2".gt_tpch2; -use "test.gt_iceberg2".gt_tpch2; +create schema gt_iceberg2.gt_tpch2; +use gt_iceberg2.gt_tpch2; CREATE TABLE customer ( custkey bigint, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_mysql_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_mysql_prepare.sql index 3376ab82977..5f42569f2fc 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_mysql_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_mysql_prepare.sql @@ -7,8 +7,8 @@ call gravitino.system.create_catalog( ) ); -create schema "test.gt_mysql2".gt_tpch; -use "test.gt_mysql2".gt_tpch; +create schema gt_mysql2.gt_tpch; +use gt_mysql2.gt_tpch; CREATE TABLE customer ( custkey bigint NOT NULL, diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_postgresql_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_postgresql_prepare.sql index d4b8444a016..bacbc98ea0d 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_postgresql_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_postgresql_prepare.sql @@ -7,8 +7,8 @@ call gravitino.system.create_catalog( ) ); -create schema "test.gt_postgresql2".gt_tpch; -use "test.gt_postgresql2".gt_tpch; +create schema gt_postgresql2.gt_tpch; +use gt_postgresql2.gt_tpch; CREATE TABLE customer ( custkey bigint NOT NULL, diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java index 63b68bb1626..978e8d1f548 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java @@ -28,7 +28,7 @@ public class GravitinoConfig { new ConfigEntry( "gravitino.simplify-catalog-names", "Omit metalake prefix for catalog names", - "false", + "true", false); public GravitinoConfig(Map requiredConfig) { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java index 1c7a43f09fe..84f84ec2665 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java @@ -64,8 +64,7 @@ public Connector create( try { CatalogInjector catalogInjector = new CatalogInjector(); catalogInjector.init(context); - CatalogConnectorFactory catalogConnectorFactory = - new CatalogConnectorFactory(catalogInjector); + CatalogConnectorFactory catalogConnectorFactory = new CatalogConnectorFactory(); catalogConnectorManager = new CatalogConnectorManager(catalogInjector, catalogConnectorFactory); @@ -95,7 +94,7 @@ public Connector create( if (Strings.isNullOrEmpty(metalake)) { throw new TrinoException(GRAVITINO_METALAKE_NOT_EXISTS, "No gravitino metalake selected"); } - if (config.simplifyCatalogNames() && catalogConnectorManager.getCatalogs().size() > 1) { + if (config.simplifyCatalogNames() && !catalogConnectorManager.getUsedMetalakes().isEmpty()) { throw new TrinoException( GRAVITINO_MISSING_CONFIG, "Multiple metalakes are not supported when setting gravitino.simplify-catalog-names = true"); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java index ad141b6386a..1b8ab569247 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java @@ -4,10 +4,8 @@ */ package com.datastrato.gravitino.trino.connector.catalog; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_CATALOG_PROVIDER; -import com.datastrato.gravitino.client.GravitinoMetalake; import com.datastrato.gravitino.trino.connector.catalog.hive.HiveConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.iceberg.IcebergConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.jdbc.mysql.MySQLConnectorAdapter; @@ -15,7 +13,6 @@ import com.datastrato.gravitino.trino.connector.catalog.memory.MemoryConnectorAdapter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.TrinoException; -import io.trino.spi.connector.Connector; import java.util.HashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,12 +21,9 @@ public class CatalogConnectorFactory { private static final Logger LOG = LoggerFactory.getLogger(CatalogConnectorFactory.class); - private final CatalogInjector catalogInjector; private final HashMap catalogBuilders = new HashMap<>(); - public CatalogConnectorFactory(CatalogInjector catalogInjector) { - this.catalogInjector = catalogInjector; - + public CatalogConnectorFactory() { catalogBuilders.put("hive", new CatalogConnectorContext.Builder(new HiveConnectorAdapter())); catalogBuilders.put( "memory", new CatalogConnectorContext.Builder(new MemoryConnectorAdapter())); @@ -41,8 +35,8 @@ public CatalogConnectorFactory(CatalogInjector catalogInjector) { "jdbc-postgresql", new CatalogConnectorContext.Builder(new PostgreSQLConnectorAdapter())); } - public CatalogConnectorContext loadCatalogConnector( - GravitinoMetalake metalake, GravitinoCatalog catalog) { + public CatalogConnectorContext.Builder createCatalogConnectorContextBuilder( + GravitinoCatalog catalog) { String catalogProvider = catalog.getProvider(); CatalogConnectorContext.Builder builder = catalogBuilders.get(catalogProvider); if (builder == null) { @@ -52,23 +46,6 @@ public CatalogConnectorContext loadCatalogConnector( } // Avoid using the same builder object to prevent catalog creation errors. - builder = builder.clone(); - - try { - Connector internalConnector = - catalogInjector.createConnector(catalog.getFullName(), builder.buildConfig(catalog)); - - return builder - .withMetalake(metalake) - .withCatalog(catalog) - .withInternalConnector(internalConnector) - .build(); - - } catch (Exception e) { - String message = - String.format("Failed to create internal catalog connector. The catalog is: %s", catalog); - LOG.error(message, e); - throw new TrinoException(GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, message, e); - } + return builder.clone(); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index 9ba6fefeb77..ac1da69bfbe 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -6,8 +6,10 @@ import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CATALOG_ALREADY_EXISTS; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CATALOG_NOT_EXISTS; +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_METALAKE_NOT_EXISTS; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_OPERATION_FAILED; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION; import com.datastrato.gravitino.Catalog; @@ -25,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.trino.spi.TrinoException; +import io.trino.spi.connector.Connector; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -166,7 +169,7 @@ public void loadCatalogs(GravitinoMetalake metalake) { Catalog catalog = metalake.loadCatalog(nameIdentifier); GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(metalake.name(), catalog, config.simplifyCatalogNames()); - if (catalogConnectors.containsKey(gravitinoCatalog.getFullName())) { + if (catalogConnectors.containsKey(getTrinoCatalogName(gravitinoCatalog))) { // Reload catalogs that have been updated in Gravitino server. reloadCatalog(metalake, gravitinoCatalog); @@ -183,13 +186,13 @@ public void loadCatalogs(GravitinoMetalake metalake) { } private void reloadCatalog(GravitinoMetalake metalake, GravitinoCatalog catalog) { - GravitinoCatalog oldCatalog = catalogConnectors.get(catalog.getFullName()).getCatalog(); + GravitinoCatalog oldCatalog = catalogConnectors.get(getTrinoCatalogName(catalog)).getCatalog(); if (!catalog.getLastModifiedTime().isAfter(oldCatalog.getLastModifiedTime())) { return; } - catalogInjector.removeCatalogConnector(catalog.getFullName()); - catalogConnectors.remove(catalog.getFullName()); + catalogInjector.removeCatalogConnector((getTrinoCatalogName(catalog))); + catalogConnectors.remove(getTrinoCatalogName(catalog)); loadCatalogImpl(metalake, catalog); LOG.info("Update catalog '{}' in metalake {} successfully.", catalog, metalake.name()); @@ -197,16 +200,27 @@ private void reloadCatalog(GravitinoMetalake metalake, GravitinoCatalog catalog) private void loadCatalog(GravitinoMetalake metalake, GravitinoCatalog catalog) { loadCatalogImpl(metalake, catalog); - LOG.info( - "Load catalog {} in metalake {} successfully.", catalog.getFullName(), metalake.name()); + LOG.info("Load catalog {} in metalake {} successfully.", catalog, metalake.name()); } private void loadCatalogImpl(GravitinoMetalake metalake, GravitinoCatalog catalog) { - CatalogConnectorContext catalogConnectorContext = - catalogConnectorFactory.loadCatalogConnector(metalake, catalog); + CatalogConnectorContext.Builder builder = + catalogConnectorFactory.createCatalogConnectorContextBuilder(catalog); + try { + Connector internalConnector = + catalogInjector.createConnector( + getTrinoCatalogName(catalog), builder.buildConfig(catalog)); - catalogConnectors.put(catalog.getFullName(), catalogConnectorContext); - catalogInjector.injectCatalogConnector(catalog.getFullName()); + builder.withMetalake(metalake).withCatalog(catalog).withInternalConnector(internalConnector); + } catch (Exception e) { + String message = + String.format("Failed to create internal catalog connector. The catalog is: %s", catalog); + LOG.error(message, e); + throw new TrinoException(GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, message, e); + } + + catalogConnectors.put(getTrinoCatalogName(catalog), builder.build()); + catalogInjector.injectCatalogConnector(getTrinoCatalogName(catalog)); } private void unloadCatalog(GravitinoMetalake metalake, String catalogFullName) { @@ -228,6 +242,14 @@ public void shutdown() { throw new NotImplementedException(); } + public String getTrinoCatalogName(String metalake, String catalog) { + return config.simplifyCatalogNames() ? catalog : metalake + "." + catalog; + } + + public String getTrinoCatalogName(GravitinoCatalog catalog) { + return getTrinoCatalogName(catalog.getMetalake(), catalog.getName()); + } + public void createCatalog( String metalakeName, String catalogName, @@ -235,7 +257,7 @@ public void createCatalog( Map properties, boolean ignoreExist) { NameIdentifier catalog = NameIdentifier.of(metalakeName, catalogName); - if (catalogConnectors.containsKey(catalog.toString())) { + if (catalogConnectors.containsKey(getTrinoCatalogName(metalakeName, catalogName))) { if (!ignoreExist) { throw new TrinoException( GRAVITINO_CATALOG_ALREADY_EXISTS, String.format("Catalog %s already exists.", catalog)); @@ -249,11 +271,15 @@ public void createCatalog( metalake.createCatalog( catalog, Catalog.Type.RELATIONAL, provider, "Trino created", properties); - LOG.info("Create catalog {} in metalake {} successfully.", catalog, metalake); + LOG.info("Create catalog {} in metalake {} successfully.", catalogName, metalake); Future future = executorService.submit(this::loadMetalake); future.get(30, TimeUnit.SECONDS); + if (!catalogConnectors.containsKey(getTrinoCatalogName(metalakeName, catalogName))) { + throw new TrinoException( + GRAVITINO_OPERATION_FAILED, "Create catalog failed due to the loading process fails"); + } } catch (NoSuchMetalakeException e) { throw new TrinoException( GRAVITINO_METALAKE_NOT_EXISTS, "Metalake " + metalakeName + " not exists."); @@ -286,11 +312,15 @@ public void dropCatalog(String metalakeName, String catalogName, boolean ignoreN throw new TrinoException( GRAVITINO_UNSUPPORTED_OPERATION, "Drop catalog " + catalog + " does not support."); } - LOG.info("Drop catalog {} in metalake {} successfully.", catalog, metalake); + LOG.info("Drop catalog {} in metalake {} successfully.", catalogName, metalake); Future future = executorService.submit(this::loadMetalake); future.get(30, TimeUnit.SECONDS); + if (catalogConnectors.containsKey(getTrinoCatalogName(metalakeName, catalogName))) { + throw new TrinoException( + GRAVITINO_OPERATION_FAILED, "Drop catalog failed due to the reloading process fails"); + } } catch (NoSuchMetalakeException e) { throw new TrinoException( GRAVITINO_METALAKE_NOT_EXISTS, "Metalake " + metalakeName + " not exists."); @@ -305,10 +335,10 @@ public void alterCatalog( String catalogName, Map setProperties, List removeProperties) { + NameIdentifier catalog = NameIdentifier.of(metalakeName, catalogName); try { - NameIdentifier catalogNameId = NameIdentifier.of(metalakeName, catalogName); CatalogConnectorContext catalogConnectorContext = - catalogConnectors.get(catalogNameId.toString()); + catalogConnectors.get(getTrinoCatalogName(metalakeName, catalogName)); GravitinoCatalog oldCatalog = catalogConnectorContext.getCatalog(); List changes = new ArrayList<>(); @@ -341,19 +371,27 @@ public void alterCatalog( GravitinoMetalake metalake = gravitinoClient.loadMetalake(NameIdentifier.ofMetalake(metalakeName)); - metalake.alterCatalog( - NameIdentifier.of(metalakeName, catalogName), - changes.toArray(changes.toArray(new CatalogChange[0]))); + metalake.alterCatalog(catalog, changes.toArray(changes.toArray(new CatalogChange[0]))); Future future = executorService.submit(this::loadMetalake); future.get(30, TimeUnit.SECONDS); + catalogConnectorContext = + catalogConnectors.get(getTrinoCatalogName(metalakeName, catalogName)); + if (catalogConnectorContext == null + || catalogConnectorContext + .getCatalog() + .getLastModifiedTime() + .equals(oldCatalog.getLastModifiedTime())) { + throw new TrinoException( + GRAVITINO_OPERATION_FAILED, "Update catalog failed due to the reloading process fails"); + } + } catch (NoSuchMetalakeException e) { throw new TrinoException( GRAVITINO_METALAKE_NOT_EXISTS, "Metalake " + metalakeName + " not exists."); } catch (NoSuchCatalogException e) { - throw new TrinoException( - GRAVITINO_CATALOG_NOT_EXISTS, "Catalog " + catalogName + " not exists."); + throw new TrinoException(GRAVITINO_CATALOG_NOT_EXISTS, "Catalog " + catalog + " not exists."); } catch (Exception e) { throw new TrinoException( GRAVITINO_UNSUPPORTED_OPERATION, "alter catalog failed. " + e.getMessage(), e); @@ -367,4 +405,8 @@ public void addMetalake(String metalake) { "Multiple metalakes are not supported when setting gravitino.simplify-catalog-names = true"); usedMetalakes.add(metalake); } + + public Set getUsedMetalakes() { + return usedMetalakes; + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java index 19c2fc81659..4dd61b1c368 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -18,12 +18,10 @@ public class GravitinoCatalog { private final String metalake; private final Catalog catalog; - private final boolean usingSimpleName; public GravitinoCatalog(String metalake, Catalog catalog, boolean usingSimpleName) { this.metalake = metalake; this.catalog = catalog; - this.usingSimpleName = usingSimpleName; } public String getProvider() { @@ -34,8 +32,8 @@ public String getName() { return catalog.name(); } - public String getFullName() { - return usingSimpleName ? catalog.name() : metalake + "." + catalog.name(); + public String getMetalake() { + return metalake; } public NameIdentifier geNameIdentifier() { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java index 2996f8a9688..63db9900ed7 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java @@ -52,7 +52,7 @@ public Page loadPageData() { for (GravitinoCatalog catalog : catalogs) { Preconditions.checkNotNull(catalog, "catalog should not be null"); - VARCHAR.writeString(nameColumnBuilder, catalog.getFullName()); + VARCHAR.writeString(nameColumnBuilder, catalog.getName()); VARCHAR.writeString(providerColumnBuilder, catalog.getProvider()); try { VARCHAR.writeString( diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 0b10579dcf4..66d1fa9821b 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -65,10 +65,6 @@ public class GravitinoMockServer implements AutoCloseable { CatalogConnectorManager catalogConnectorManager; private GeneralDataTypeTransformer dataTypeTransformer = new HiveDataTypeTransformer(); - public GravitinoMockServer() { - this(false); - } - public GravitinoMockServer(boolean simpleCatalogName) { this.simpleCatalogName = simpleCatalogName; createMetalake(NameIdentifier.ofMetalake(testMetalake)); @@ -241,12 +237,13 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector(catalogConnectorManager.getTrinoCatalogName(catalog)) .getMetadataAdapter(); GravitinoSchema schema = new GravitinoSchema(schemaName.name(), properties, ""); metadata.createSchema(null, schemaName.name(), emptyMap(), null); @@ -269,7 +266,8 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); metadata.dropSchema(null, nameIdentifier.name(), cascade); @@ -286,7 +284,8 @@ public NameIdentifier[] answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); return metadata.listSchemaNames(null).stream() @@ -307,7 +306,8 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); memoryConnector.getMetadata(null, null); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -316,7 +316,7 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector(catalogConnectorManager.getTrinoCatalogName(catalog)) .getMetadataAdapter(); GravitinoSchema gravitinoSchema = @@ -361,7 +361,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { tableName.schema(), tableName.table(), columns, comment, properties); CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector(catalogConnectorManager.getTrinoCatalogName(catalog)) .getMetadataAdapter(); ConnectorTableMetadata tableMetadata = metadataAdapter.getTableMetadata(gravitinoTable); @@ -369,7 +369,8 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); metadata.createTable(null, tableMetadata, false); @@ -388,7 +389,8 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); memoryConnector.getMetadata(null, null); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -415,7 +417,8 @@ public NameIdentifier[] answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); ArrayList tableNames = new ArrayList<>(); @@ -441,7 +444,8 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); return metadata.getTableHandle( @@ -463,7 +467,8 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); @@ -477,7 +482,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector(catalogConnectorManager.getTrinoCatalogName(catalog)) .getMetadataAdapter(); GravitinoTable gravitinoTable = metadataAdapter.createTable(tableMetadata); @@ -505,7 +510,8 @@ public Table answer(InvocationOnMock invocation) throws Throwable { MemoryConnector memoryConnector = (MemoryConnector) catalogConnectorManager - .getCatalogConnector(catalog.getFullName()) + .getCatalogConnector( + catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); ConnectorTableHandle tableHandle = @@ -542,7 +548,9 @@ void doAlterTable( GravitinoColumn column = new GravitinoColumn(fieldName, addColumn.getDataType(), -1, "", true); CatalogConnectorMetadataAdapter metadataAdapter = - catalogConnectorManager.getCatalogConnector(catalog.getFullName()).getMetadataAdapter(); + catalogConnectorManager + .getCatalogConnector(catalogConnectorManager.getTrinoCatalogName(catalog)) + .getMetadataAdapter(); metadata.addColumn(null, tableHandle, metadataAdapter.getColumnMetadata(column)); } else if (tableChange instanceof TableChange.DeleteColumn) { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java index 5b7013b9b75..5f6aa375830 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java @@ -19,7 +19,7 @@ public class TestCreateGravitinoConnector { GravitinoMockServer server; @Test - public void testCreateSimpleCatalogNameConnector() throws Exception { + public void testCreateConnectorsWithEnableSimpleCatalog() throws Exception { server = new GravitinoMockServer(true); Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); @@ -28,6 +28,7 @@ public void testCreateSimpleCatalogNameConnector() throws Exception { TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); queryRunner.installPlugin(gravitinoPlugin); + // test create two connector and set gravitino.simplify-catalog-names = true { // create a gravitino connector named gravitino using metalake test HashMap properties = new HashMap<>(); @@ -42,6 +43,7 @@ public void testCreateSimpleCatalogNameConnector() throws Exception { HashMap properties = new HashMap<>(); properties.put("gravitino.metalake", "test1"); properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "true"); try { queryRunner.createCatalog("test1", "gravitino", properties); } catch (Exception e) { @@ -51,4 +53,36 @@ public void testCreateSimpleCatalogNameConnector() throws Exception { server.close(); } + + @Test + public void testCreateConnectorsWithDisableSimpleCatalog() throws Exception { + server = new GravitinoMockServer(false); + Session session = testSessionBuilder().setCatalog("gravitino").build(); + QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); + + GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); + TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); + queryRunner.installPlugin(gravitinoPlugin); + + // test create two connector and set gravitino.simplify-catalog-names = false + { + // create a gravitino connector named gravitino using metalake test + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "false"); + queryRunner.createCatalog("test0", "gravitino", properties); + } + + { + // Test failed to create catalog with different metalake + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test1"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "false"); + queryRunner.createCatalog("test1", "gravitino", properties); + } + + server.close(); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index c6054909975..93fb75da8c7 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -9,7 +9,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; -import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; import io.trino.Session; @@ -19,7 +18,6 @@ import io.trino.testing.MaterializedResult; import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -37,7 +35,7 @@ public class TestGravitinoConnector extends AbstractTestQueryFramework { @Override protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer()); + server = closeAfterClass(new GravitinoMockServer(true)); GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); @@ -50,21 +48,11 @@ protected QueryRunner createQueryRunner() throws Exception { queryRunner.installPlugin(gravitinoPlugin); queryRunner.installPlugin(new MemoryPlugin()); - { - // create a gravitino connector named gravitino using metalake test - HashMap properties = new HashMap<>(); - properties.put("gravitino.metalake", "test"); - properties.put("gravitino.uri", "http://127.0.0.1:8090"); - queryRunner.createCatalog("gravitino", "gravitino", properties); - } - - { - // create a gravitino connector named test1 using metalake test1 - HashMap properties = new HashMap<>(); - properties.put("gravitino.metalake", "test1"); - properties.put("gravitino.uri", "http://127.0.0.1:8090"); - queryRunner.createCatalog("test1", "gravitino", properties); - } + // create a gravitino connector named gravitino using metalake test + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + queryRunner.createCatalog("gravitino", "gravitino", properties); CatalogConnectorManager catalogConnectorManager = gravitinoPlugin.getCatalogConnectorManager(); @@ -82,14 +70,14 @@ protected QueryRunner createQueryRunner() throws Exception { @Test public void testCreateSchema() { - String catalogName = "test.memory"; + String catalogName = "memory"; String schemaName = "db_01"; - String fullSchemaName = String.format("\"%s\".%s", catalogName, schemaName); - assertThat(computeActual("show schemas from \"test.memory\"").getOnlyColumnAsSet()) + String fullSchemaName = String.format("%s.%s", catalogName, schemaName); + assertThat(computeActual("show schemas from " + catalogName).getOnlyColumnAsSet()) .doesNotContain(schemaName); assertUpdate("create schema " + fullSchemaName); - assertThat(computeActual("show schemas from \"test.memory\"").getOnlyColumnAsSet()) + assertThat(computeActual("show schemas from \"memory\"").getOnlyColumnAsSet()) .contains(schemaName); assertThat((String) computeScalar("show create schema " + fullSchemaName)) @@ -108,7 +96,7 @@ public void testCreateSchema() { @Test public void testCreateTable() { - String fullSchemaName = "\"test.memory\".db_01"; + String fullSchemaName = "memory.db_01"; String tableName = "tb_01"; String fullTableName = fullSchemaName + "." + tableName; @@ -127,13 +115,13 @@ public void testCreateTable() { .startsWith(format("CREATE TABLE %s", fullTableName)); // cleanup - assertUpdate("drop table" + fullTableName); + assertUpdate("drop table " + fullTableName); assertUpdate("drop schema " + fullSchemaName); } @Test public void testInsert() throws Exception { - String fullTableName = "\"test.memory\".db_01.tb_01"; + String fullTableName = "\"memory\".db_01.tb_01"; createTestTable(fullTableName); // insert some data. assertUpdate(String.format("insert into %s (a, b) values ('ice', 12)", fullTableName), 1); @@ -152,8 +140,8 @@ public void testInsert() throws Exception { @Test public void testInsertIntoSelect() throws Exception { - String fullTableName1 = "\"test.memory\".db_01.tb_01"; - String fullTableName2 = "\"test.memory\".db_01.tb_02"; + String fullTableName1 = "\"memory\".db_01.tb_01"; + String fullTableName2 = "\"memory\".db_01.tb_02"; createTestTable(fullTableName1); createTestTable(fullTableName2); @@ -171,8 +159,8 @@ public void testInsertIntoSelect() throws Exception { @Test public void testAlterTable() throws Exception { - String fullTableName1 = "\"test.memory\".db_01.tb_01"; - String fullTableName2 = "\"test.memory\".db_01.tb_02"; + String fullTableName1 = "\"memory\".db_01.tb_01"; + String fullTableName2 = "\"memory\".db_01.tb_02"; createTestTable(fullTableName1); // test rename table @@ -223,35 +211,25 @@ public void testAlterTable() throws Exception { public void testCreateCatalog() throws Exception { // testing the catalogs assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory"); // testing the gravitino connector framework works. assertThat(computeActual("select * from system.jdbc.tables")); // test metalake named test. the connector name is gravitino assertUpdate("call gravitino.system.create_catalog('memory1', 'memory', Map())"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory1"); assertUpdate("call gravitino.system.drop_catalog('memory1')"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("memory1"); assertUpdate( "call gravitino.system.create_catalog(" + "catalog=>'memory1', provider=>'memory', properties => Map(array['max_ttl'], array['10']), ignore_exist => true)"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory1"); assertUpdate( "call gravitino.system.drop_catalog(catalog => 'memory1', ignore_not_exist => true)"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); - - // test metalake named test1. the connnector name is test1 - GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); - gravitinoClient.createMetalake(NameIdentifier.ofMetalake("test1"), "", Collections.emptyMap()); - - assertUpdate("call test1.system.create_catalog('memory1', 'memory', Map())"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1.memory1"); - assertUpdate("call test1.system.drop_catalog('memory1')"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test1.memory1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("memory1"); } @Test @@ -260,7 +238,7 @@ public void testSystemTable() throws Exception { assertEquals(expectedResult.getRowCount(), 1); List expectedRows = expectedResult.getMaterializedRows(); MaterializedRow row = expectedRows.get(0); - assertEquals(row.getField(0), "test.memory"); + assertEquals(row.getField(0), "memory"); assertEquals(row.getField(1), "memory"); assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java new file mode 100644 index 00000000000..ed9d4457a0e --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java @@ -0,0 +1,149 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector; + +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; +import io.trino.Session; +import io.trino.plugin.memory.MemoryPlugin; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedResult; +import io.trino.testing.MaterializedRow; +import io.trino.testing.QueryRunner; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.annotations.Test; + +public class TestGravitinoConnectorWithMetalakeCatalogName extends AbstractTestQueryFramework { + + GravitinoMockServer server; + + @Override + protected QueryRunner createQueryRunner() throws Exception { + server = closeAfterClass(new GravitinoMockServer(false)); + GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); + + Session session = testSessionBuilder().setCatalog("gravitino").build(); + QueryRunner queryRunner = null; + try { + queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); + + TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); + queryRunner.installPlugin(gravitinoPlugin); + queryRunner.installPlugin(new MemoryPlugin()); + + { + // create a gravitino connector named gravitino using metalake test + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "false"); + queryRunner.createCatalog("gravitino", "gravitino", properties); + } + + { + // create a gravitino connector named test1 using metalake test1 + HashMap properties = new HashMap<>(); + properties.put("gravitino.metalake", "test1"); + properties.put("gravitino.uri", "http://127.0.0.1:8090"); + properties.put("gravitino.simplify-catalog-names", "false"); + queryRunner.createCatalog("test1", "gravitino", properties); + } + + CatalogConnectorManager catalogConnectorManager = + gravitinoPlugin.getCatalogConnectorManager(); + server.setCatalogConnectorManager(catalogConnectorManager); + // Wait for the catalog to be created. Wait for at least 30 seconds. + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); + } catch (Exception e) { + throw new RuntimeException("Create query runner failed", e); + } + return queryRunner; + } + + @Test + public void testSystemTable() throws Exception { + MaterializedResult expectedResult = computeActual("select * from gravitino.system.catalog"); + assertEquals(expectedResult.getRowCount(), 1); + List expectedRows = expectedResult.getMaterializedRows(); + MaterializedRow row = expectedRows.get(0); + assertEquals(row.getField(0), "memory"); + assertEquals(row.getField(1), "memory"); + assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); + } + + @Test + public void testCreateCatalog() throws Exception { + // testing the catalogs + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory"); + + // testing the gravitino connector framework works. + assertThat(computeActual("select * from system.jdbc.tables")); + + // test metalake named test. the connector name is gravitino + assertUpdate("call gravitino.system.create_catalog('memory1', 'memory', Map())"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); + assertUpdate("call gravitino.system.drop_catalog('memory1')"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); + + assertUpdate( + "call gravitino.system.create_catalog(" + + "catalog=>'memory1', provider=>'memory', properties => Map(array['max_ttl'], array['10']), ignore_exist => true)"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); + + assertUpdate( + "call gravitino.system.drop_catalog(catalog => 'memory1', ignore_not_exist => true)"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); + + // test metalake named test1. the connnector name is test1 + GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); + gravitinoClient.createMetalake(NameIdentifier.ofMetalake("test1"), "", Collections.emptyMap()); + + assertUpdate("call test1.system.create_catalog('memory1', 'memory', Map())"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1.memory1"); + assertUpdate("call test1.system.drop_catalog('memory1')"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test1.memory1"); + } + + @Test + public void testCreateTable() { + String fullSchemaName = "\"test.memory\".db_01"; + String tableName = "tb_01"; + String fullTableName = fullSchemaName + "." + tableName; + + assertUpdate("create schema " + fullSchemaName); + + // try to get table + assertThat(computeActual("show tables from " + fullSchemaName).getOnlyColumnAsSet()) + .doesNotContain(tableName); + + // try to create table + assertUpdate("create table " + fullTableName + " (a varchar, b int)"); + assertThat(computeActual("show tables from " + fullSchemaName).getOnlyColumnAsSet()) + .contains(tableName); + + assertThat((String) computeScalar("show create table " + fullTableName)) + .startsWith(format("CREATE TABLE %s", fullTableName)); + + // cleanup + assertUpdate("drop table " + fullTableName); + assertUpdate("drop schema " + fullSchemaName); + } +} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java deleted file mode 100644 index 573816701de..00000000000 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithSimpleCatalogName.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.assertj.core.api.Assertions.assertThat; -import static org.testng.Assert.assertEquals; - -import com.datastrato.gravitino.client.GravitinoAdminClient; -import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; -import io.trino.Session; -import io.trino.plugin.memory.MemoryPlugin; -import io.trino.testing.AbstractTestQueryFramework; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import io.trino.testing.QueryRunner; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.testcontainers.shaded.org.awaitility.Awaitility; -import org.testng.annotations.Test; - -public class TestGravitinoConnectorWithSimpleCatalogName extends AbstractTestQueryFramework { - - GravitinoMockServer server; - - @Override - protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer(true)); - GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); - - Session session = testSessionBuilder().setCatalog("gravitino").build(); - QueryRunner queryRunner = null; - try { - queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); - - TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); - queryRunner.installPlugin(gravitinoPlugin); - queryRunner.installPlugin(new MemoryPlugin()); - - // create a gravitino connector named gravitino using metalake test - HashMap properties = new HashMap<>(); - properties.put("gravitino.metalake", "test"); - properties.put("gravitino.uri", "http://127.0.0.1:8090"); - properties.put("gravitino.simplify-catalog-names", "true"); - queryRunner.createCatalog("gravitino", "gravitino", properties); - - CatalogConnectorManager catalogConnectorManager = - gravitinoPlugin.getCatalogConnectorManager(); - server.setCatalogConnectorManager(catalogConnectorManager); - // Wait for the catalog to be created. Wait for at least 30 seconds. - Awaitility.await() - .atMost(30, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); - } catch (Exception e) { - throw new RuntimeException("Create query runner failed", e); - } - return queryRunner; - } - - @Test - public void testCatalogName() { - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory"); - assertUpdate("call gravitino.system.create_catalog('memory1', 'memory', Map())"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("memory1"); - - String schemaName = "db1"; - String fullSchemaName = String.format("\"%s\".%s", "memory", schemaName); - assertUpdate("create schema " + fullSchemaName); - assertThat(computeActual("show schemas from \"memory\"").getOnlyColumnAsSet()) - .contains(schemaName); - - assertUpdate("drop schema " + fullSchemaName); - assertUpdate("call gravitino.system.drop_catalog('memory1')"); - } - - @Test - public void testSystemTable() throws Exception { - MaterializedResult expectedResult = computeActual("select * from gravitino.system.catalog"); - assertEquals(expectedResult.getRowCount(), 1); - List expectedRows = expectedResult.getMaterializedRows(); - MaterializedRow row = expectedRows.get(0); - assertEquals(row.getField(0), "memory"); - assertEquals(row.getField(1), "memory"); - assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); - } -} From 2c0f79f981f8eb16c2ffc5599ece5785740e29d3 Mon Sep 17 00:00:00 2001 From: mchades Date: Fri, 19 Apr 2024 17:49:41 +0800 Subject: [PATCH 067/106] [#2954] feat(all): add case-sensitive capability (#2975) ### What changes were proposed in this pull request? - support case-sensitive capability for name - Hive catalog use case-insensitive capability ### Why are the changes needed? Fix: #2954 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? tests added --- .../gravitino/rel/indexes/Indexes.java | 2 +- .../catalog/hive/HiveCatalogCapability.java | 14 + .../hive/integration/test/CatalogHiveIT.java | 4 +- .../mysql/operation/MysqlTableOperations.java | 2 +- .../operation/PostgreSqlTableOperations.java | 2 +- .../iceberg/IcebergCatalogOperations.java | 2 +- .../lakehouse/iceberg/TestIcebergSchema.java | 12 + .../datastrato/gravitino/GravitinoEnv.java | 23 +- .../gravitino/catalog/CapabilityHelpers.java | 279 +++++++++++++++++- .../catalog/FilesetNormalizeDispatcher.java | 76 +++++ .../catalog/OperationDispatcher.java | 15 + .../catalog/SchemaNormalizeDispatcher.java | 68 +++++ .../catalog/TableNormalizeDispatcher.java | 97 ++++++ .../catalog/TableOperationDispatcher.java | 2 +- .../catalog/TopicNormalizeDispatcher.java | 69 +++++ .../connector/capability/Capability.java | 1 - .../com/datastrato/gravitino/TestCatalog.java | 6 + .../gravitino/TestCatalogCapabilities.java | 16 + .../gravitino/TestCatalogOperations.java | 38 ++- .../com/datastrato/gravitino/TestTable.java | 1 + .../gravitino/catalog/TestCatalogManager.java | 24 -- .../TestFilesetNormalizeDispatcher.java | 75 +++++ .../TestFilesetOperationDispatcher.java | 4 +- .../TestSchemaNormalizeDispatcher.java | 57 ++++ .../TestSchemaOperationDispatcher.java | 14 +- .../catalog/TestTableNormalizeDispatcher.java | 117 ++++++++ .../catalog/TestTableOperationDispatcher.java | 4 +- .../catalog/TestTopicNormalizeDispatcher.java | 61 ++++ .../catalog/TestTopicOperationDispatcher.java | 4 +- 29 files changed, 1016 insertions(+), 73 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java create mode 100644 core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java create mode 100644 core/src/test/java/com/datastrato/gravitino/TestCatalogCapabilities.java create mode 100644 core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java create mode 100644 core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java create mode 100644 core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java create mode 100644 core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java diff --git a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java index 242ab20a90e..531c0290d05 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java @@ -72,7 +72,7 @@ public static final class IndexImpl implements Index { * @param name The name of the index * @param fieldNames The field names under the table contained in the index. */ - public IndexImpl(IndexType indexType, String name, String[][] fieldNames) { + private IndexImpl(IndexType indexType, String name, String[][] fieldNames) { this.indexType = indexType; this.name = name; this.fieldNames = fieldNames; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java index d98f6e12d7b..8e1eb188b6b 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java @@ -25,4 +25,18 @@ public CapabilityResult columnDefaultValue() { "The DEFAULT constraint for column is only supported since Hive 3.0, " + "but the current Gravitino Hive catalog only supports Hive 2.x."); } + + @Override + public CapabilityResult caseSensitiveOnName(Scope scope) { + switch (scope) { + case SCHEMA: + case TABLE: + case COLUMN: + // Hive is case insensitive, see + // https://cwiki.apache.org/confluence/display/Hive/User+FAQ#UserFAQ-AreHiveSQLidentifiers(e.g.tablenames,columnnames,etc)casesensitive? + return CapabilityResult.unsupported("Hive is case insensitive."); + default: + return CapabilityResult.SUPPORTED; + } + } } diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index c660d8185d3..46c3a105643 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -584,7 +584,7 @@ public void testHiveTableProperties() throws TException, InterruptedException { Assertions.assertEquals(TEXT_INPUT_FORMAT_CLASS, actualTable2.getSd().getInputFormat()); Assertions.assertEquals(IGNORE_KEY_OUTPUT_FORMAT_CLASS, actualTable2.getSd().getOutputFormat()); Assertions.assertEquals(EXTERNAL_TABLE.name(), actualTable2.getTableType()); - Assertions.assertEquals(table2, actualTable2.getSd().getSerdeInfo().getName()); + Assertions.assertEquals(table2.toLowerCase(), actualTable2.getSd().getSerdeInfo().getName()); Assertions.assertEquals(TABLE_COMMENT, actualTable2.getParameters().get(COMMENT)); Assertions.assertEquals( ((Boolean) tablePropertiesMetadata.getDefaultValue(EXTERNAL)).toString().toUpperCase(), @@ -1224,7 +1224,7 @@ private void assertDefaultTableProperties( Assertions.assertEquals( ((TableType) tablePropertiesMetadata.getDefaultValue(TABLE_TYPE)).name(), actualTable.getTableType()); - Assertions.assertEquals(tableName, actualTable.getSd().getSerdeInfo().getName()); + Assertions.assertEquals(tableName.toLowerCase(), actualTable.getSd().getSerdeInfo().getName()); Assertions.assertEquals( ((Boolean) tablePropertiesMetadata.getDefaultValue(EXTERNAL)).toString().toUpperCase(), actualTable.getParameters().get(EXTERNAL)); diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 1538b31c551..0446713c77b 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -85,7 +85,7 @@ protected String generateCreateTableSql( } Preconditions.checkArgument( - distribution == Distributions.NONE, "MySQL does not support distribution"); + Distributions.NONE.equals(distribution), "MySQL does not support distribution"); validateIncrementCol(columns, indexes); StringBuilder sqlBuilder = new StringBuilder(); diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index 44b09f588e3..e7d8c058fde 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -84,7 +84,7 @@ protected String generateCreateTableSql( "Currently we do not support Partitioning in PostgreSQL"); } Preconditions.checkArgument( - distribution == Distributions.NONE, "PostgreSQL does not support distribution"); + Distributions.NONE.equals(distribution), "PostgreSQL does not support distribution"); StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index b3fc85e5883..3c28fbd6527 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -128,7 +128,7 @@ public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogExc icebergTableOps.listNamespace(IcebergTableOpsHelper.getIcebergNamespace()).namespaces(); return namespaces.stream() - .map(icebergNamespace -> NameIdentifier.of(icebergNamespace.levels())) + .map(icebergNamespace -> NameIdentifier.of(namespace, icebergNamespace.toString())) .toArray(NameIdentifier[]::new); } catch (NoSuchNamespaceException e) { throw new NoSuchSchemaException( diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java index 2017e899f38..a510c92598c 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java @@ -65,6 +65,18 @@ public void testCreateIcebergSchema() { Assertions.assertTrue(exception.getMessage().contains("already exists")); } + @Test + public void testListSchema() { + IcebergCatalog icebergCatalog = initIcebergCatalog("testListIcebergSchema"); + NameIdentifier ident = NameIdentifier.of("metalake", icebergCatalog.name(), "test"); + icebergCatalog.asSchemas().createSchema(ident, COMMENT_VALUE, Maps.newHashMap()); + + NameIdentifier[] schemas = icebergCatalog.asSchemas().listSchemas(ident.namespace()); + Assertions.assertEquals(1, schemas.length); + Assertions.assertEquals(ident.name(), schemas[0].name()); + Assertions.assertEquals(ident.namespace(), schemas[0].namespace()); + } + @Test public void testAlterSchema() { IcebergCatalog icebergCatalog = initIcebergCatalog("testAlterSchema"); diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index cbc11ca53e0..9f801b054a3 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -11,15 +11,19 @@ import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.catalog.FilesetEventDispatcher; +import com.datastrato.gravitino.catalog.FilesetNormalizeDispatcher; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.catalog.SchemaDispatcher; import com.datastrato.gravitino.catalog.SchemaEventDispatcher; +import com.datastrato.gravitino.catalog.SchemaNormalizeDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; import com.datastrato.gravitino.catalog.TableEventDispatcher; +import com.datastrato.gravitino.catalog.TableNormalizeDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.catalog.TopicDispatcher; import com.datastrato.gravitino.catalog.TopicEventDispatcher; +import com.datastrato.gravitino.catalog.TopicNormalizeDispatcher; import com.datastrato.gravitino.catalog.TopicOperationDispatcher; import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.EventListenerManager; @@ -155,16 +159,27 @@ public void initialize(Config config) { SchemaOperationDispatcher schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); - this.schemaDispatcher = new SchemaEventDispatcher(eventBus, schemaOperationDispatcher); + SchemaNormalizeDispatcher schemaNormalizeDispatcher = + new SchemaNormalizeDispatcher(schemaOperationDispatcher); + this.schemaDispatcher = new SchemaEventDispatcher(eventBus, schemaNormalizeDispatcher); + TableOperationDispatcher tableOperationDispatcher = new TableOperationDispatcher(catalogManager, entityStore, idGenerator); - this.tableDispatcher = new TableEventDispatcher(eventBus, tableOperationDispatcher); + TableNormalizeDispatcher tableNormalizeDispatcher = + new TableNormalizeDispatcher(tableOperationDispatcher); + this.tableDispatcher = new TableEventDispatcher(eventBus, tableNormalizeDispatcher); + FilesetOperationDispatcher filesetOperationDispatcher = new FilesetOperationDispatcher(catalogManager, entityStore, idGenerator); - this.filesetDispatcher = new FilesetEventDispatcher(eventBus, filesetOperationDispatcher); + FilesetNormalizeDispatcher filesetNormalizeDispatcher = + new FilesetNormalizeDispatcher(filesetOperationDispatcher); + this.filesetDispatcher = new FilesetEventDispatcher(eventBus, filesetNormalizeDispatcher); + TopicOperationDispatcher topicOperationDispatcher = new TopicOperationDispatcher(catalogManager, entityStore, idGenerator); - this.topicDispatcher = new TopicEventDispatcher(eventBus, topicOperationDispatcher); + TopicNormalizeDispatcher topicNormalizeDispatcher = + new TopicNormalizeDispatcher(topicOperationDispatcher); + this.topicDispatcher = new TopicEventDispatcher(eventBus, topicNormalizeDispatcher); // Create and initialize access control related modules boolean enableAuthorization = config.get(Configs.ENABLE_AUTHORIZATION); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java index d08587987ba..2ed2aca6e41 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java @@ -5,11 +5,25 @@ package com.datastrato.gravitino.catalog; import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; +import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.NAME_OF_IDENTITY; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.Expression; +import com.datastrato.gravitino.rel.expressions.FunctionExpression; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; import com.google.common.base.Preconditions; import java.util.Arrays; @@ -25,22 +39,244 @@ public static TableChange[] applyCapabilities(Capability capabilities, TableChan return Arrays.stream(changes) .map( change -> { - if (change instanceof TableChange.AddColumn) { - return applyCapabilities((TableChange.AddColumn) change, capabilities); + if (change instanceof TableChange.ColumnChange) { + return applyCapabilities((TableChange.ColumnChange) change, capabilities); - } else if (change instanceof TableChange.UpdateColumnNullability) { - return applyCapabilities( - (TableChange.UpdateColumnNullability) change, capabilities); - - } else if (change instanceof TableChange.UpdateColumnDefaultValue) { - return applyCapabilities( - ((TableChange.UpdateColumnDefaultValue) change), capabilities); + } else if (change instanceof TableChange.RenameTable) { + return applyCapabilities((TableChange.RenameTable) change, capabilities); } return change; }) .toArray(TableChange[]::new); } + public static FilesetChange[] applyCapabilities( + Capability capabilities, FilesetChange... changes) { + return Arrays.stream(changes) + .map( + change -> { + if (change instanceof FilesetChange.RenameFileset) { + return applyCapabilities((FilesetChange.RenameFileset) change, capabilities); + } + return change; + }) + .toArray(FilesetChange[]::new); + } + + public static NameIdentifier[] applyCapabilities( + NameIdentifier[] idents, Capability.Scope scope, Capability capabilities) { + return Arrays.stream(idents) + .map(ident -> applyCapabilities(ident, scope, capabilities)) + .toArray(NameIdentifier[]::new); + } + + public static NameIdentifier applyCapabilities( + NameIdentifier ident, Capability.Scope scope, Capability capabilities) { + Namespace namespace = ident.namespace(); + namespace = applyCapabilities(namespace, scope, capabilities); + + String name = applyCapabilitiesOnName(scope, ident.name(), capabilities); + return NameIdentifier.of(namespace, name); + } + + public static Transform[] applyCapabilities(Transform[] transforms, Capability capabilities) { + return Arrays.stream(transforms) + .map(t -> applyCapabilities(t, capabilities)) + .toArray(Transform[]::new); + } + + public static Distribution applyCapabilities(Distribution distribution, Capability capabilities) { + Expression[] expressions = applyCapabilities(distribution.expressions(), capabilities); + return Distributions.of(distribution.strategy(), distribution.number(), expressions); + } + + public static SortOrder[] applyCapabilities(SortOrder[] sortOrders, Capability capabilities) { + return Arrays.stream(sortOrders) + .map(s -> applyCapabilities(s, capabilities)) + .toArray(SortOrder[]::new); + } + + public static Index[] applyCapabilities(Index[] indexes, Capability capabilities) { + return Arrays.stream(indexes) + .map(i -> applyCapabilities(i, capabilities)) + .toArray(Index[]::new); + } + + public static Namespace applyCapabilities( + Namespace namespace, Capability.Scope identScope, Capability capabilities) { + String metalake = namespace.level(0); + String catalog = namespace.level(1); + if (identScope == Capability.Scope.TABLE + || identScope == Capability.Scope.FILESET + || identScope == Capability.Scope.TOPIC) { + String schema = namespace.level(namespace.length() - 1); + schema = applyCapabilitiesOnName(Capability.Scope.SCHEMA, schema, capabilities); + return Namespace.of(metalake, catalog, schema); + } + return namespace; + } + + private static Index applyCapabilities(Index index, Capability capabilities) { + return Indexes.of( + index.type(), index.name(), applyCapabilities(index.fieldNames(), capabilities)); + } + + private static String[][] applyCapabilities(String[][] fieldNames, Capability capabilities) { + String[][] standardizeFieldNames = new String[fieldNames.length][]; + for (int i = 0; i < standardizeFieldNames.length; i++) { + standardizeFieldNames[i] = applyCapabilities(fieldNames[i], capabilities); + } + return standardizeFieldNames; + } + + private static String[] applyCapabilities(String[] fieldName, Capability capabilities) { + String[] sensitiveOnColumnName = applyCaseSensitiveOnColumnName(fieldName, capabilities); + applyNameSpecification(Capability.Scope.COLUMN, sensitiveOnColumnName[0], capabilities); + return sensitiveOnColumnName; + } + + private static Transform applyCapabilities(Transform transform, Capability capabilities) { + if (transform instanceof Transform.SingleFieldTransform) { + String[] standardizeFieldName = + applyCapabilities(((Transform.SingleFieldTransform) transform).fieldName(), capabilities); + switch (transform.name()) { + case NAME_OF_IDENTITY: + return Transforms.identity(standardizeFieldName); + case Transforms.NAME_OF_YEAR: + return Transforms.year(standardizeFieldName); + case Transforms.NAME_OF_MONTH: + return Transforms.month(standardizeFieldName); + case Transforms.NAME_OF_DAY: + return Transforms.day(standardizeFieldName); + case Transforms.NAME_OF_HOUR: + return Transforms.hour(standardizeFieldName); + default: + throw new IllegalArgumentException("Unsupported transform: " + transform.name()); + } + + } else if (transform instanceof Transforms.BucketTransform) { + Transforms.BucketTransform bucketTransform = (Transforms.BucketTransform) transform; + return Transforms.bucket( + bucketTransform.numBuckets(), + applyCapabilities(bucketTransform.fieldNames(), capabilities)); + + } else if (transform instanceof Transforms.TruncateTransform) { + Transforms.TruncateTransform truncateTransform = (Transforms.TruncateTransform) transform; + return Transforms.truncate( + truncateTransform.width(), + applyCapabilities(truncateTransform.fieldName(), capabilities)); + + } else if (transform instanceof Transforms.ListTransform) { + return Transforms.list( + applyCapabilities(((Transforms.ListTransform) transform).fieldNames(), capabilities)); + + } else if (transform instanceof Transforms.RangeTransform) { + return Transforms.range( + applyCapabilities(((Transforms.RangeTransform) transform).fieldName(), capabilities)); + + } else if (transform instanceof Transforms.ApplyTransform) { + return Transforms.apply( + transform.name(), applyCapabilities(transform.arguments(), capabilities)); + + } else { + throw new IllegalArgumentException("Unsupported transform: " + transform.name()); + } + } + + private static SortOrder applyCapabilities(SortOrder sortOrder, Capability capabilities) { + Expression expression = applyCapabilities(sortOrder.expression(), capabilities); + return SortOrders.of(expression, sortOrder.direction(), sortOrder.nullOrdering()); + } + + private static Expression[] applyCapabilities(Expression[] expressions, Capability capabilities) { + return Arrays.stream(expressions) + .map(e -> applyCapabilities(e, capabilities)) + .toArray(Expression[]::new); + } + + private static Expression applyCapabilities(Expression expression, Capability capabilities) { + if (expression instanceof NamedReference.FieldReference) { + NamedReference.FieldReference ref = (NamedReference.FieldReference) expression; + String[] fieldName = applyCapabilities(ref.fieldName(), capabilities); + return NamedReference.field(fieldName); + + } else if (expression instanceof FunctionExpression) { + FunctionExpression functionExpression = (FunctionExpression) expression; + return FunctionExpression.of( + functionExpression.functionName(), + applyCapabilities(functionExpression.arguments(), capabilities)); + } + return expression; + } + + private static FilesetChange applyCapabilities( + FilesetChange.RenameFileset renameFileset, Capability capabilities) { + String newName = + applyCaseSensitiveOnName( + Capability.Scope.FILESET, renameFileset.getNewName(), capabilities); + applyNameSpecification(Capability.Scope.FILESET, newName, capabilities); + return FilesetChange.rename(newName); + } + + private static TableChange applyCapabilities( + TableChange.RenameTable renameTable, Capability capabilities) { + String newName = + applyCaseSensitiveOnName(Capability.Scope.TABLE, renameTable.getNewName(), capabilities); + applyNameSpecification(Capability.Scope.TABLE, newName, capabilities); + return TableChange.rename(newName); + } + + private static TableChange applyCapabilities( + TableChange.ColumnChange change, Capability capabilities) { + String[] fieldName = applyCaseSensitiveOnColumnName(change.fieldName(), capabilities); + applyNameSpecification(Capability.Scope.COLUMN, fieldName[0], capabilities); + + if (change instanceof TableChange.AddColumn) { + return applyCapabilities((TableChange.AddColumn) change, capabilities); + + } else if (change instanceof TableChange.UpdateColumnNullability) { + return applyCapabilities((TableChange.UpdateColumnNullability) change, capabilities); + + } else if (change instanceof TableChange.UpdateColumnDefaultValue) { + return applyCapabilities(((TableChange.UpdateColumnDefaultValue) change), capabilities); + + } else if (change instanceof TableChange.RenameColumn) { + return applyCapabilities((TableChange.RenameColumn) change, capabilities); + + } else if (change instanceof TableChange.DeleteColumn) { + return TableChange.deleteColumn(fieldName, ((TableChange.DeleteColumn) change).getIfExists()); + + } else if (change instanceof TableChange.UpdateColumnAutoIncrement) { + return TableChange.updateColumnAutoIncrement( + fieldName, ((TableChange.UpdateColumnAutoIncrement) change).isAutoIncrement()); + + } else if (change instanceof TableChange.UpdateColumnComment) { + return TableChange.updateColumnComment( + fieldName, ((TableChange.UpdateColumnComment) change).getNewComment()); + + } else if (change instanceof TableChange.UpdateColumnPosition) { + TableChange.UpdateColumnPosition updateColumnPosition = + (TableChange.UpdateColumnPosition) change; + if (updateColumnPosition.getPosition() instanceof TableChange.After) { + TableChange.After afterPosition = (TableChange.After) updateColumnPosition.getPosition(); + String afterFieldName = + applyCaseSensitiveOnName( + Capability.Scope.COLUMN, afterPosition.getColumn(), capabilities); + applyNameSpecification(Capability.Scope.COLUMN, afterFieldName, capabilities); + return TableChange.updateColumnPosition( + fieldName, TableChange.ColumnPosition.after(afterFieldName)); + } + return TableChange.updateColumnPosition(fieldName, updateColumnPosition.getPosition()); + + } else if (change instanceof TableChange.UpdateColumnType) { + return TableChange.updateColumnType( + fieldName, ((TableChange.UpdateColumnType) change).getNewDataType()); + + } else { + throw new IllegalArgumentException("Unsupported column change: " + change); + } + } + private static TableChange applyCapabilities( TableChange.AddColumn addColumn, Capability capabilities) { Column appliedColumn = @@ -54,8 +290,11 @@ private static TableChange applyCapabilities( addColumn.getDefaultValue()), capabilities); + String[] standardizeFieldName = + Arrays.copyOf(addColumn.fieldName(), addColumn.fieldName().length); + standardizeFieldName[0] = appliedColumn.name(); return TableChange.addColumn( - applyCaseSensitiveOnColumnName(addColumn.fieldName(), capabilities), + standardizeFieldName, appliedColumn.dataType(), appliedColumn.comment(), addColumn.getPosition(), @@ -89,13 +328,22 @@ private static TableChange applyCapabilities( updateColumnDefaultValue.getNewDefaultValue()); } + private static TableChange applyCapabilities( + TableChange.RenameColumn renameColumn, Capability capabilities) { + String[] fieldName = applyCapabilities(renameColumn.fieldName(), capabilities); + String newName = renameColumn.getNewName(); + if (fieldName.length == 1) { + newName = applyCapabilitiesOnName(Capability.Scope.COLUMN, newName, capabilities); + } + return TableChange.renameColumn(fieldName, newName); + } + private static Column applyCapabilities(Column column, Capability capabilities) { applyColumnNotNull(column, capabilities); applyColumnDefaultValue(column, capabilities); - applyNameSpecification(Capability.Scope.COLUMN, column.name(), capabilities); return Column.of( - applyCaseSensitiveOnName(Capability.Scope.COLUMN, column.name(), capabilities), + applyCapabilitiesOnName(Capability.Scope.COLUMN, column.name(), capabilities), column.dataType(), column.comment(), column.nullable(), @@ -103,6 +351,13 @@ private static Column applyCapabilities(Column column, Capability capabilities) column.defaultValue()); } + private static String applyCapabilitiesOnName( + Capability.Scope scope, String name, Capability capabilities) { + String standardizeName = applyCaseSensitiveOnName(scope, name, capabilities); + applyNameSpecification(scope, standardizeName, capabilities); + return standardizeName; + } + private static String applyCaseSensitiveOnName( Capability.Scope scope, String name, Capability capabilities) { return capabilities.caseSensitiveOnName(scope).supported() ? name : name.toLowerCase(); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java new file mode 100644 index 00000000000..d89871cbaad --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCapabilities; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchFilesetException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetChange; +import java.util.Map; + +public class FilesetNormalizeDispatcher implements FilesetDispatcher { + + private final FilesetOperationDispatcher dispatcher; + + public FilesetNormalizeDispatcher(FilesetOperationDispatcher dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { + Capability capability = dispatcher.getCatalogCapability(namespace); + Namespace standardizedNamespace = + applyCapabilities(namespace, Capability.Scope.FILESET, capability); + NameIdentifier[] identifiers = dispatcher.listFilesets(standardizedNamespace); + return applyCapabilities(identifiers, Capability.Scope.FILESET, capability); + } + + @Override + public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { + return dispatcher.loadFileset(normalizeNameIdentifier(ident)); + } + + @Override + public boolean filesetExists(NameIdentifier ident) { + return dispatcher.filesetExists(normalizeNameIdentifier(ident)); + } + + @Override + public Fileset createFileset( + NameIdentifier ident, + String comment, + Fileset.Type type, + String storageLocation, + Map properties) + throws NoSuchSchemaException, FilesetAlreadyExistsException { + return dispatcher.createFileset( + normalizeNameIdentifier(ident), comment, type, storageLocation, properties); + } + + @Override + public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) + throws NoSuchFilesetException, IllegalArgumentException { + Capability capability = dispatcher.getCatalogCapability(ident); + return dispatcher.alterFileset( + applyCapabilities(ident, Capability.Scope.FILESET, capability), + applyCapabilities(capability, changes)); + } + + @Override + public boolean dropFileset(NameIdentifier ident) { + return dispatcher.dropFileset(normalizeNameIdentifier(ident)); + } + + private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { + Capability capability = dispatcher.getCatalogCapability(ident); + return applyCapabilities(ident, Capability.Scope.FILESET, capability); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java index 9c503185e92..45d69ec0e3f 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java @@ -14,6 +14,7 @@ import com.datastrato.gravitino.connector.BasePropertiesMetadata; import com.datastrato.gravitino.connector.HasPropertyMetadata; import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.file.FilesetChange; @@ -101,6 +102,20 @@ R doWithCatalog( } } + Capability getCatalogCapability(NameIdentifier ident) { + return doWithCatalog( + getCatalogIdentifier(ident), + CatalogManager.CatalogWrapper::capabilities, + IllegalArgumentException.class); + } + + Capability getCatalogCapability(Namespace namespace) { + return doWithCatalog( + getCatalogIdentifier(NameIdentifier.of(namespace.levels())), + CatalogManager.CatalogWrapper::capabilities, + IllegalArgumentException.class); + } + Set getHiddenPropertyNames( NameIdentifier catalogIdent, ThrowableFunction provider, diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java new file mode 100644 index 00000000000..22da2a01606 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCapabilities; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NonEmptySchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.SchemaChange; +import java.util.Map; + +public class SchemaNormalizeDispatcher implements SchemaDispatcher { + + private final SchemaOperationDispatcher dispatcher; + + public SchemaNormalizeDispatcher(SchemaOperationDispatcher dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogException { + Capability capability = dispatcher.getCatalogCapability(namespace); + Namespace standardizedNamespace = + applyCapabilities(namespace, Capability.Scope.SCHEMA, capability); + NameIdentifier[] identifiers = dispatcher.listSchemas(standardizedNamespace); + return applyCapabilities(identifiers, Capability.Scope.SCHEMA, capability); + } + + @Override + public boolean schemaExists(NameIdentifier ident) { + return dispatcher.schemaExists(normalizeNameIdentifier(ident)); + } + + @Override + public Schema createSchema(NameIdentifier ident, String comment, Map properties) + throws NoSuchCatalogException, SchemaAlreadyExistsException { + return dispatcher.createSchema(normalizeNameIdentifier(ident), comment, properties); + } + + @Override + public Schema loadSchema(NameIdentifier ident) throws NoSuchSchemaException { + return dispatcher.loadSchema(normalizeNameIdentifier(ident)); + } + + @Override + public Schema alterSchema(NameIdentifier ident, SchemaChange... changes) + throws NoSuchSchemaException { + return dispatcher.alterSchema(normalizeNameIdentifier(ident), changes); + } + + @Override + public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmptySchemaException { + return dispatcher.dropSchema(normalizeNameIdentifier(ident), cascade); + } + + private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { + Capability capability = dispatcher.getCatalogCapability(ident); + return applyCapabilities(ident, Capability.Scope.SCHEMA, capability); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java new file mode 100644 index 00000000000..68784272e79 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java @@ -0,0 +1,97 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCapabilities; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import java.util.Map; + +public class TableNormalizeDispatcher implements TableDispatcher { + + private final TableOperationDispatcher dispatcher; + + public TableNormalizeDispatcher(TableOperationDispatcher dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { + Capability capability = dispatcher.getCatalogCapability(namespace); + Namespace standardizedNamespace = + applyCapabilities(namespace, Capability.Scope.TABLE, capability); + NameIdentifier[] identifiers = dispatcher.listTables(standardizedNamespace); + return applyCapabilities(identifiers, Capability.Scope.TABLE, capability); + } + + @Override + public Table loadTable(NameIdentifier ident) throws NoSuchTableException { + return dispatcher.loadTable(normalizeNameIdentifier(ident)); + } + + @Override + public Table createTable( + NameIdentifier ident, + Column[] columns, + String comment, + Map properties, + Transform[] partitions, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes) + throws NoSuchSchemaException, TableAlreadyExistsException { + Capability capability = dispatcher.getCatalogCapability(ident); + return dispatcher.createTable( + applyCapabilities(ident, Capability.Scope.TABLE, capability), + applyCapabilities(columns, capability), + comment, + properties, + applyCapabilities(partitions, capability), + applyCapabilities(distribution, capability), + applyCapabilities(sortOrders, capability), + applyCapabilities(indexes, capability)); + } + + @Override + public Table alterTable(NameIdentifier ident, TableChange... changes) + throws NoSuchTableException, IllegalArgumentException { + Capability capability = dispatcher.getCatalogCapability(ident); + return dispatcher.alterTable( + applyCapabilities(ident, Capability.Scope.TABLE, capability), + applyCapabilities(capability, changes)); + } + + @Override + public boolean dropTable(NameIdentifier ident) { + return dispatcher.dropTable(normalizeNameIdentifier(ident)); + } + + @Override + public boolean purgeTable(NameIdentifier ident) throws UnsupportedOperationException { + return dispatcher.purgeTable(normalizeNameIdentifier(ident)); + } + + @Override + public boolean tableExists(NameIdentifier ident) { + return dispatcher.tableExists(normalizeNameIdentifier(ident)); + } + + private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { + Capability capability = dispatcher.getCatalogCapability(ident); + return applyCapabilities(ident, Capability.Scope.TABLE, capability); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index d388979582f..0fc7e187dc3 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -164,7 +164,7 @@ public Table createTable( t -> t.createTable( ident, - applyCapabilities(columns, c.capabilities()), + columns, comment, updatedProperties, partitions == null ? EMPTY_TRANSFORM : partitions, diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java new file mode 100644 index 00000000000..753a2f01852 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCapabilities; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTopicException; +import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; +import com.datastrato.gravitino.messaging.DataLayout; +import com.datastrato.gravitino.messaging.Topic; +import com.datastrato.gravitino.messaging.TopicChange; +import java.util.Map; + +public class TopicNormalizeDispatcher implements TopicDispatcher { + + private final TopicOperationDispatcher dispatcher; + + public TopicNormalizeDispatcher(TopicOperationDispatcher dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public NameIdentifier[] listTopics(Namespace namespace) throws NoSuchSchemaException { + Capability capability = dispatcher.getCatalogCapability(namespace); + Namespace standardizedNamespace = + applyCapabilities(namespace, Capability.Scope.TOPIC, capability); + NameIdentifier[] identifiers = dispatcher.listTopics(standardizedNamespace); + return applyCapabilities(identifiers, Capability.Scope.TOPIC, capability); + } + + @Override + public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { + return dispatcher.loadTopic(normalizeNameIdentifier(ident)); + } + + @Override + public boolean topicExists(NameIdentifier ident) { + return dispatcher.topicExists(normalizeNameIdentifier(ident)); + } + + @Override + public Topic createTopic( + NameIdentifier ident, String comment, DataLayout dataLayout, Map properties) + throws NoSuchSchemaException, TopicAlreadyExistsException { + return dispatcher.createTopic(normalizeNameIdentifier(ident), comment, dataLayout, properties); + } + + @Override + public Topic alterTopic(NameIdentifier ident, TopicChange... changes) + throws NoSuchTopicException, IllegalArgumentException { + return dispatcher.alterTopic(normalizeNameIdentifier(ident), changes); + } + + @Override + public boolean dropTopic(NameIdentifier ident) { + return dispatcher.dropTopic(normalizeNameIdentifier(ident)); + } + + private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { + Capability capability = dispatcher.getCatalogCapability(ident); + return applyCapabilities(ident, Capability.Scope.TOPIC, capability); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java index 9c6dde58dc3..3303ea7b69c 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java @@ -17,7 +17,6 @@ public interface Capability { /** The scope of the capability. */ enum Scope { - CATALOG, SCHEMA, TABLE, COLUMN, diff --git a/core/src/test/java/com/datastrato/gravitino/TestCatalog.java b/core/src/test/java/com/datastrato/gravitino/TestCatalog.java index 0eb79c0b882..bbe4b449a68 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestCatalog.java +++ b/core/src/test/java/com/datastrato/gravitino/TestCatalog.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.rel.TableCatalog; import java.util.Map; import java.util.Objects; @@ -29,6 +30,11 @@ protected CatalogOperations newOps(Map config) { return new TestCatalogOperations(config); } + @Override + protected Capability newCapability() { + return new TestCatalogCapabilities(); + } + @Override public TableCatalog asTableCatalog() { return (TableCatalog) ops(); diff --git a/core/src/test/java/com/datastrato/gravitino/TestCatalogCapabilities.java b/core/src/test/java/com/datastrato/gravitino/TestCatalogCapabilities.java new file mode 100644 index 00000000000..1874ebf18c2 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/TestCatalogCapabilities.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class TestCatalogCapabilities implements Capability { + + @Override + public CapabilityResult caseSensitiveOnName(Scope scope) { + return CapabilityResult.unsupported("The case sensitive on name is not supported."); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java b/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java index dba2b03d307..47b3dccc9fd 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java +++ b/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java @@ -167,6 +167,7 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) Map newProps = table.properties() != null ? Maps.newHashMap(table.properties()) : Maps.newHashMap(); + NameIdentifier newIdent = ident; for (TableChange change : changes) { if (change instanceof TableChange.SetProperty) { newProps.put( @@ -174,6 +175,12 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) ((TableChange.SetProperty) change).getValue()); } else if (change instanceof TableChange.RemoveProperty) { newProps.remove(((TableChange.RemoveProperty) change).getProperty()); + } else if (change instanceof TableChange.RenameTable) { + String newName = ((TableChange.RenameTable) change).getNewName(); + newIdent = NameIdentifier.of(ident.namespace(), newName); + if (tables.containsKey(newIdent)) { + throw new TableAlreadyExistsException("Table %s already exists", ident); + } } else { throw new IllegalArgumentException("Unsupported table change: " + change); } @@ -181,23 +188,19 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) TestTable updatedTable = TestTable.builder() - .withName(ident.name()) + .withName(newIdent.name()) .withComment(table.comment()) .withProperties(new HashMap<>(newProps)) .withAuditInfo(updatedAuditInfo) .withColumns(table.columns()) .withPartitioning(table.partitioning()) + .withDistribution(table.distribution()) + .withSortOrders(table.sortOrder()) + .withIndexes(table.index()) .build(); tables.put(ident, updatedTable); - return TestTable.builder() - .withName(ident.name()) - .withComment(table.comment()) - .withProperties(new HashMap<>(newProps)) - .withAuditInfo(updatedAuditInfo) - .withColumns(table.columns()) - .withPartitioning(table.partitioning()) - .build(); + return updatedTable; } @Override @@ -440,8 +443,11 @@ public Fileset createFileset( .withStorageLocation(storageLocation) .build(); - if (tables.containsKey(ident)) { + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + if (filesets.containsKey(ident)) { throw new FilesetAlreadyExistsException("Fileset %s already exists", ident); + } else if (!schemas.containsKey(schemaIdent)) { + throw new NoSuchSchemaException("Schema %s does not exist", schemaIdent); } else { filesets.put(ident, fileset); } @@ -467,6 +473,7 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) TestFileset fileset = filesets.get(ident); Map newProps = fileset.properties() != null ? Maps.newHashMap(fileset.properties()) : Maps.newHashMap(); + NameIdentifier newIdent = ident; for (FilesetChange change : changes) { if (change instanceof FilesetChange.SetProperty) { @@ -475,6 +482,13 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) ((FilesetChange.SetProperty) change).getValue()); } else if (change instanceof FilesetChange.RemoveProperty) { newProps.remove(((FilesetChange.RemoveProperty) change).getProperty()); + } else if (change instanceof FilesetChange.RenameFileset) { + String newName = ((FilesetChange.RenameFileset) change).getNewName(); + newIdent = NameIdentifier.of(ident.namespace(), newName); + if (filesets.containsKey(newIdent)) { + throw new FilesetAlreadyExistsException("Fileset %s already exists", ident); + } + filesets.remove(ident); } else { throw new IllegalArgumentException("Unsupported fileset change: " + change); } @@ -482,14 +496,14 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) TestFileset updatedFileset = TestFileset.builder() - .withName(ident.name()) + .withName(newIdent.name()) .withComment(fileset.comment()) .withProperties(newProps) .withAuditInfo(updatedAuditInfo) .withType(fileset.type()) .withStorageLocation(fileset.storageLocation()) .build(); - filesets.put(ident, updatedFileset); + filesets.put(newIdent, updatedFileset); return updatedFileset; } diff --git a/core/src/test/java/com/datastrato/gravitino/TestTable.java b/core/src/test/java/com/datastrato/gravitino/TestTable.java index 768a50a7209..7b01c050591 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestTable.java +++ b/core/src/test/java/com/datastrato/gravitino/TestTable.java @@ -32,6 +32,7 @@ protected TestTable internalBuild() { table.distribution = distribution; table.sortOrders = sortOrders; table.partitioning = partitioning; + table.indexes = indexes; return table; } } diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java index bd7d66ad282..db043feef1f 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java @@ -132,30 +132,6 @@ void testCreateWithHiveProperty() throws IOException { ident, Catalog.Type.RELATIONAL, provider, "comment", props3)); } - @Test - void testLoadTable() throws IOException { - NameIdentifier ident = NameIdentifier.of("metalake", "test444"); - // key1 is required; - Map props1 = - ImmutableMap.builder() - .put("key2", "value2") - .put("key1", "value1") - .put("hidden_key", "hidden_value") - .put("mock", "mock") - .build(); - Assertions.assertDoesNotThrow( - () -> - catalogManager.createCatalog( - ident, Catalog.Type.RELATIONAL, provider, "comment", props1)); - - Map properties = catalogManager.loadCatalog(ident).properties(); - Assertions.assertTrue(properties.containsKey("key2")); - Assertions.assertTrue(properties.containsKey("key1")); - Assertions.assertFalse(properties.containsKey("hidden_key")); - Assertions.assertFalse(properties.containsKey(ID_KEY)); - reset(); - } - @Test void testPropertyValidationInAlter() throws IOException { // key1 is required and immutable and do not have default value, is not hidden and not reserved diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java new file mode 100644 index 00000000000..b0ea02cc4e6 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java @@ -0,0 +1,75 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetChange; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestFilesetNormalizeDispatcher extends TestFilesetOperationDispatcher { + private static FilesetNormalizeDispatcher filesetNormalizeDispatcher; + private static SchemaNormalizeDispatcher schemaNormalizeDispatcher; + + @BeforeAll + public static void initialize() throws IOException { + TestFilesetOperationDispatcher.initialize(); + filesetNormalizeDispatcher = new FilesetNormalizeDispatcher(filesetOperationDispatcher); + schemaNormalizeDispatcher = new SchemaNormalizeDispatcher(schemaOperationDispatcher); + } + + @Test + public void testNameCaseInsensitive() { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema112"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + // test case-insensitive in creation + NameIdentifier filesetIdent = NameIdentifier.of(filesetNs, "filesetNAME"); + Fileset createdFileset = + filesetNormalizeDispatcher.createFileset( + filesetIdent, "comment", Fileset.Type.MANAGED, "fileset41", props); + Assertions.assertEquals(filesetIdent.name().toLowerCase(), createdFileset.name()); + + // test case-insensitive in loading + Fileset loadedFileset = filesetNormalizeDispatcher.loadFileset(filesetIdent); + Assertions.assertEquals(filesetIdent.name().toLowerCase(), loadedFileset.name()); + + // test case-insensitive in listing + NameIdentifier[] filesets = filesetNormalizeDispatcher.listFilesets(filesetNs); + Arrays.stream(filesets).forEach(s -> Assertions.assertEquals(s.name().toLowerCase(), s.name())); + + // test case-insensitive in altering + Fileset alteredFileset = + filesetNormalizeDispatcher.alterFileset( + NameIdentifier.of(filesetNs, filesetIdent.name().toLowerCase()), + FilesetChange.setProperty("k2", "v2")); + Assertions.assertEquals(filesetIdent.name().toLowerCase(), alteredFileset.name()); + + Exception exception = + Assertions.assertThrows( + FilesetAlreadyExistsException.class, + () -> + filesetNormalizeDispatcher.alterFileset( + NameIdentifier.of(filesetNs, filesetIdent.name().toUpperCase()), + FilesetChange.rename(filesetIdent.name().toUpperCase()))); + Assertions.assertEquals( + "Fileset metalake.catalog.schema112.filesetname already exists", exception.getMessage()); + + // test case-insensitive in dropping + Assertions.assertTrue( + filesetNormalizeDispatcher.dropFileset( + NameIdentifier.of(filesetNs, filesetIdent.name().toUpperCase()))); + Assertions.assertFalse(filesetNormalizeDispatcher.filesetExists(filesetIdent)); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java index 3bcd906f330..a2f97136399 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -19,8 +19,8 @@ import org.junit.jupiter.api.Test; public class TestFilesetOperationDispatcher extends TestOperationDispatcher { - private static FilesetOperationDispatcher filesetOperationDispatcher; - private static SchemaOperationDispatcher schemaOperationDispatcher; + static FilesetOperationDispatcher filesetOperationDispatcher; + static SchemaOperationDispatcher schemaOperationDispatcher; @BeforeAll public static void initialize() throws IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java new file mode 100644 index 00000000000..206ce794e5f --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.SchemaChange; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Arrays; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestSchemaNormalizeDispatcher extends TestSchemaOperationDispatcher { + private static SchemaNormalizeDispatcher schemaNormalizeDispatcher; + + @BeforeAll + public static void initialize() throws IOException { + TestSchemaOperationDispatcher.initialize(); + schemaNormalizeDispatcher = new SchemaNormalizeDispatcher(dispatcher); + } + + @Test + public void testNameCaseInsensitive() { + // test case-insensitive in creation + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, "schemaNAME"); + Schema createdSchema = + schemaNormalizeDispatcher.createSchema( + schemaIdent, null, ImmutableMap.of("k1", "v1", "k2", "v2")); + Assertions.assertEquals(schemaIdent.name().toLowerCase(), createdSchema.name()); + + // test case-insensitive in loading + Schema loadSchema = schemaNormalizeDispatcher.loadSchema(schemaIdent); + Assertions.assertEquals(schemaIdent.name().toLowerCase(), loadSchema.name()); + + // test case-insensitive in listing + NameIdentifier[] schemas = + schemaNormalizeDispatcher.listSchemas(Namespace.of(metalake, catalog)); + Arrays.stream(schemas).forEach(s -> Assertions.assertEquals(s.name().toLowerCase(), s.name())); + + // test case-insensitive in altering + Schema alteredSchema = + schemaNormalizeDispatcher.alterSchema( + schemaIdent, SchemaChange.setProperty("k2", "v2"), SchemaChange.removeProperty("k1")); + Assertions.assertEquals(schemaIdent.name().toLowerCase(), alteredSchema.name()); + + // test case-insensitive in dropping + Assertions.assertTrue( + schemaNormalizeDispatcher.dropSchema( + NameIdentifier.of(schemaIdent.namespace(), schemaIdent.name().toLowerCase()), false)); + Assertions.assertFalse(schemaNormalizeDispatcher.schemaExists(schemaIdent)); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaOperationDispatcher.java index d44295364a3..6b830e8a269 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaOperationDispatcher.java @@ -34,7 +34,7 @@ public class TestSchemaOperationDispatcher extends TestOperationDispatcher { - private static SchemaOperationDispatcher dispatcher; + static SchemaOperationDispatcher dispatcher; @BeforeAll public static void initialize() throws IOException { @@ -54,8 +54,8 @@ public void testCreateAndListSchemas() throws IOException { Assertions.assertEquals("comment", schema.comment()); testProperties(props, schema.properties()); - // Test required table properties exception - Map illegalTableProperties = + // Test required schema properties exception + Map illegalSchemaProperties = new HashMap() { { put("k2", "v2"); @@ -63,14 +63,14 @@ public void testCreateAndListSchemas() throws IOException { }; testPropertyException( - () -> dispatcher.createSchema(schemaIdent, "comment", illegalTableProperties), + () -> dispatcher.createSchema(schemaIdent, "comment", illegalSchemaProperties), "Properties are required and must be set"); // Test reserved table properties exception - illegalTableProperties.put(COMMENT_KEY, "table comment"); - illegalTableProperties.put(ID_KEY, "gravitino.v1.uidfdsafdsa"); + illegalSchemaProperties.put(COMMENT_KEY, "table comment"); + illegalSchemaProperties.put(ID_KEY, "gravitino.v1.uidfdsafdsa"); testPropertyException( - () -> dispatcher.createSchema(schemaIdent, "comment", illegalTableProperties), + () -> dispatcher.createSchema(schemaIdent, "comment", illegalSchemaProperties), "Properties are reserved and cannot be set", "comment", "gravitino.identifier"); diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java new file mode 100644 index 00000000000..be3132050f7 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java @@ -0,0 +1,117 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.TestColumn; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.distributions.Strategy; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestTableNormalizeDispatcher extends TestTableOperationDispatcher { + private static TableNormalizeDispatcher tableNormalizeDispatcher; + private static SchemaNormalizeDispatcher schemaNormalizeDispatcher; + + @BeforeAll + public static void initialize() throws IOException { + TestTableOperationDispatcher.initialize(); + tableNormalizeDispatcher = new TableNormalizeDispatcher(tableOperationDispatcher); + schemaNormalizeDispatcher = new SchemaNormalizeDispatcher(schemaOperationDispatcher); + } + + @Test + public void testNameCaseInsensitive() { + Namespace tableNs = Namespace.of(metalake, catalog, "schema81"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(tableNs.levels()), "comment", props); + + // test case-insensitive in creation + NameIdentifier tableIdent = NameIdentifier.of(tableNs, "tableNAME"); + Column[] columns = + new Column[] { + TestColumn.builder().withName("colNAME1").withType(Types.StringType.get()).build(), + TestColumn.builder().withName("colNAME2").withType(Types.StringType.get()).build() + }; + Transform[] transforms = new Transform[] {Transforms.identity(columns[0].name())}; + Distribution distribution = + Distributions.fields(Strategy.HASH, 5, new String[] {columns[0].name()}); + SortOrder[] sortOrders = + new SortOrder[] {SortOrders.ascending(NamedReference.field(columns[0].name()))}; + Index[] indexes = new Index[] {Indexes.primary("index1", new String[][] {{columns[0].name()}})}; + Table createdTable = + tableNormalizeDispatcher.createTable( + tableIdent, columns, "comment", props, transforms, distribution, sortOrders, indexes); + assertTableCaseInsensitive(tableIdent, columns, createdTable); + + // test case-insensitive in loading + Table loadedTable = tableNormalizeDispatcher.loadTable(tableIdent); + assertTableCaseInsensitive(tableIdent, columns, loadedTable); + + // test case-insensitive in listing + NameIdentifier[] tableIdents = tableNormalizeDispatcher.listTables(tableNs); + Arrays.stream(tableIdents) + .forEach(s -> Assertions.assertEquals(s.name().toLowerCase(), s.name())); + + // test case-insensitive in altering + Table alteredTable = + tableNormalizeDispatcher.alterTable( + NameIdentifier.of(tableNs, tableIdent.name().toLowerCase()), + TableChange.setProperty("k2", "v2")); + assertTableCaseInsensitive(tableIdent, columns, alteredTable); + + Exception exception = + Assertions.assertThrows( + TableAlreadyExistsException.class, + () -> + tableNormalizeDispatcher.alterTable( + NameIdentifier.of(tableNs, tableIdent.name().toUpperCase()), + TableChange.rename(tableIdent.name().toUpperCase()))); + Assertions.assertEquals( + "Table metalake.catalog.schema81.tablename already exists", exception.getMessage()); + + // test case-insensitive in dropping + Assertions.assertTrue( + tableNormalizeDispatcher.dropTable( + NameIdentifier.of(tableNs, tableIdent.name().toUpperCase()))); + } + + private void assertTableCaseInsensitive( + NameIdentifier tableIdent, Column[] expectedColumns, Table table) { + Assertions.assertEquals(tableIdent.name().toLowerCase(), table.name()); + Assertions.assertEquals(expectedColumns[0].name().toLowerCase(), table.columns()[0].name()); + Assertions.assertEquals(expectedColumns[1].name().toLowerCase(), table.columns()[1].name()); + Assertions.assertEquals( + expectedColumns[0].name().toLowerCase(), + table.partitioning()[0].references()[0].fieldName()[0]); + Assertions.assertEquals( + expectedColumns[0].name().toLowerCase(), + table.distribution().references()[0].fieldName()[0]); + Assertions.assertEquals( + expectedColumns[0].name().toLowerCase(), + table.sortOrder()[0].expression().references()[0].fieldName()[0]); + Assertions.assertEquals( + expectedColumns[0].name().toLowerCase(), table.index()[0].fieldNames()[0][0].toLowerCase()); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java index e87af83d485..d7cc7236e5c 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableOperationDispatcher.java @@ -37,8 +37,8 @@ import org.junit.jupiter.api.Test; public class TestTableOperationDispatcher extends TestOperationDispatcher { - private static TableOperationDispatcher tableOperationDispatcher; - private static SchemaOperationDispatcher schemaOperationDispatcher; + static TableOperationDispatcher tableOperationDispatcher; + static SchemaOperationDispatcher schemaOperationDispatcher; @BeforeAll public static void initialize() throws IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java new file mode 100644 index 00000000000..8f8cd8e41f0 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java @@ -0,0 +1,61 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.messaging.Topic; +import com.datastrato.gravitino.messaging.TopicChange; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestTopicNormalizeDispatcher extends TestTopicOperationDispatcher { + private static TopicNormalizeDispatcher topicNormalizeDispatcher; + private static SchemaNormalizeDispatcher schemaNormalizeDispatcher; + + @BeforeAll + public static void initialize() throws IOException { + TestTopicOperationDispatcher.initialize(); + schemaNormalizeDispatcher = new SchemaNormalizeDispatcher(schemaOperationDispatcher); + topicNormalizeDispatcher = new TopicNormalizeDispatcher(topicOperationDispatcher); + } + + @Test + public void testNameCaseInsensitive() { + Namespace topicNs = Namespace.of(metalake, catalog, "schema161"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(topicNs.levels()), "comment", props); + + // test case-insensitive in creation + NameIdentifier topicIdent = NameIdentifier.of(topicNs, "topicNAME"); + Topic createdTopic = topicNormalizeDispatcher.createTopic(topicIdent, "comment", null, props); + Assertions.assertEquals(topicIdent.name().toLowerCase(), createdTopic.name()); + + // test case-insensitive in loading + Topic loadedTopic = topicNormalizeDispatcher.loadTopic(topicIdent); + Assertions.assertEquals(topicIdent.name().toLowerCase(), loadedTopic.name()); + + // test case-insensitive in listing + NameIdentifier[] idents = topicNormalizeDispatcher.listTopics(topicNs); + Assertions.assertEquals(1, idents.length); + + // test case-insensitive in altering + Topic alteredTopic = + topicNormalizeDispatcher.alterTopic( + NameIdentifier.of(topicNs, topicIdent.name().toLowerCase()), + TopicChange.setProperty("k2", "v2")); + Assertions.assertEquals(topicIdent.name().toLowerCase(), alteredTopic.name()); + + // test case-insensitive in dropping + Assertions.assertTrue( + topicNormalizeDispatcher.dropTopic( + NameIdentifier.of(topicNs, topicIdent.name().toUpperCase()))); + Assertions.assertFalse(topicNormalizeDispatcher.topicExists(topicIdent)); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicOperationDispatcher.java index 22dee4baa4a..7aeb39ec1e0 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicOperationDispatcher.java @@ -30,8 +30,8 @@ public class TestTopicOperationDispatcher extends TestOperationDispatcher { - private static SchemaOperationDispatcher schemaOperationDispatcher; - private static TopicOperationDispatcher topicOperationDispatcher; + static SchemaOperationDispatcher schemaOperationDispatcher; + static TopicOperationDispatcher topicOperationDispatcher; @BeforeAll public static void initialize() throws IOException { From 6c081d88e3ce7f99516018026ac647d82e140b8b Mon Sep 17 00:00:00 2001 From: XiaoZ <57973980+xiaozcy@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:03:32 +0800 Subject: [PATCH 068/106] [#3038] fix(catalog-doris): use a string builder rather than concatenation (#3044) ### What changes were proposed in this pull request? Use a string builder rather than concatenation for comment += resultSet.getString("TABLE_COMMENT") ### Why are the changes needed? Fix: #3038 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? existing unit tests Co-authored-by: zhanghan18 --- .../catalog/doris/operation/DorisTableOperations.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 058e1c2f9fd..afa168c33e5 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -255,7 +255,7 @@ protected void correctJdbcTableFields( } // Doris Cannot get comment from JDBC 8.x, so we need to get comment from sql - String comment = ""; + StringBuilder comment = new StringBuilder(); String sql = "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"; try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { @@ -264,10 +264,10 @@ protected void correctJdbcTableFields( try (ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { - comment += resultSet.getString("TABLE_COMMENT"); + comment.append(resultSet.getString("TABLE_COMMENT")); } } - tableBuilder.withComment(comment); + tableBuilder.withComment(comment.toString()); } catch (SQLException e) { throw exceptionMapper.toGravitinoException(e); } From 8c145ed05d964cfc1fe46ca36a4cca3cbd5572cf Mon Sep 17 00:00:00 2001 From: XiaoZ <57973980+xiaozcy@users.noreply.github.com> Date: Fri, 19 Apr 2024 19:23:49 +0800 Subject: [PATCH 069/106] [#3033]fix(catalog-kafka, trino-connector): Remove calls to toString() (#3045) ### What changes were proposed in this pull request? Remove calls to toString() in KafkaCatalogOperations.java and CatalogInjector.java ### Why are the changes needed? Fix: #3033 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? existing unit tests Co-authored-by: zhanghan18 Co-authored-by: Qi Yu --- .../gravitino/catalog/kafka/KafkaCatalogOperations.java | 2 +- .../gravitino/trino/connector/catalog/CatalogInjector.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java index 938c73e5813..3c5c423aed3 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java @@ -216,7 +216,7 @@ public Topic createTopic( LOG.info( "Created topic {}[id: {}] with {} partitions and replication factor {}", ident, - topicId.toString(), + topicId, createTopicsResult.numPartitions(ident.name()).get(), createTopicsResult.replicationFactor(ident.name()).get()); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java index 02cac9f4be4..a33a9b85984 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java @@ -336,7 +336,7 @@ Connector createConnector(String connectorName, Map properties) LOG.error( "Create internal catalog connector {} failed. Connector properties: {} ", connectorName, - properties.toString(), + properties, e); throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, e); } From 354a9c2c5db582d40ef53e6166f67c951dff0e6b Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 19 Apr 2024 19:59:06 +0800 Subject: [PATCH 070/106] [#2989] improvement(docs): Simplify catalog name in Trino server (#2990) ### What changes were proposed in this pull request? Simplify the catalog name in the Trino server from the format "{metalake_name}.{catalog_name}" to "{catalog_name}". ### Why are the changes needed? Make it match the general naming specifications. Fix: #2989 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- docs/trino-connector/catalog-hive.md | 34 +++++++++++----------- docs/trino-connector/catalog-iceberg.md | 10 +++---- docs/trino-connector/catalog-mysql.md | 24 +++++++-------- docs/trino-connector/catalog-postgresql.md | 24 +++++++-------- docs/trino-connector/configuration.md | 6 ++-- docs/trino-connector/development.md | 1 + docs/trino-connector/installation.md | 6 ++-- docs/trino-connector/supported-catalog.md | 2 +- docs/trino-connector/trino-connector.md | 9 ++---- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/docs/trino-connector/catalog-hive.md b/docs/trino-connector/catalog-hive.md index 70a9ef74e06..f86c9f8e13e 100644 --- a/docs/trino-connector/catalog-hive.md +++ b/docs/trino-connector/catalog-hive.md @@ -44,7 +44,7 @@ per catalog: Users can create a schema with properties through Gravitino Trino connector as follows: ```SQL -CREATE SCHEMA "metalake.catalog".schema_name +CREATE SCHEMA catalog.schema_name ``` ## Table operations @@ -57,7 +57,7 @@ allowing null values, and adding comments. The Gravitino connector does not supp The following example shows how to create a table in the Hive catalog: ```shell -CREATE TABLE "metalake.catalog".schema_name.table_name +CREATE TABLE catalog.schema_name.table_name ( name varchar, salary int @@ -112,7 +112,7 @@ Reserved properties: A reserved property is one can't be set by users but can be Users can use the following example to create a table with properties: ```sql -CREATE TABLE "metalake.catalog".dbname.tabname +CREATE TABLE catalog.dbname.tabname ( name varchar, salary int @@ -190,7 +190,7 @@ The results are similar to: gravitino jmx system - test.hive_test + hive_test (4 rows) Query 20231017_082503_00018_6nt3n, FINISHED, 1 node @@ -202,24 +202,24 @@ Other catalogs are regular user-configured Trino catalogs. ### Creating tables and schemas -Create a new schema named `database_01` in `test.hive_test` catalog. +Create a new schema named `database_01` in `hive_test` catalog. ```sql -CREATE SCHEMA "test.hive_test".database_01; +CREATE SCHEMA hive_test.database_01; ``` Create a new schema using HDFS location: ```sql -CREATE SCHEMA "test.hive_test".database_01 WITH ( +CREATE SCHEMA hive_test.database_01 WITH ( location = 'hdfs://hdfs-host:9000/user/hive/warehouse/database_01' ); ``` -Create a new table named `table_01` in schema `"test.hive_test".database_01` and stored in a TEXTFILE format, partitioning by `salary`, bucket by `name` and sorted by `salary`. +Create a new table named `table_01` in schema `hive_test.database_01` and stored in a TEXTFILE format, partitioning by `salary`, bucket by `name` and sorted by `salary`. ```sql -CREATE TABLE "test.hive_test".database_01.table_01 +CREATE TABLE hive_test.database_01.table_01 ( name varchar, salary int @@ -238,13 +238,13 @@ WITH ( Insert data into the table `table_01`: ```sql -INSERT INTO "test.hive_test".database_01.table_01 (name, salary) VALUES ('ice', 12); +INSERT INTO hive_test.database_01.table_01 (name, salary) VALUES ('ice', 12); ``` Insert data into the table `table_01` from select: ```sql -INSERT INTO "test.hive_test".database_01.table_01 (name, salary) SELECT * FROM "test.hive_test".database_01.table_01; +INSERT INTO hive_test.database_01.table_01 (name, salary) SELECT * FROM hive_test.database_01.table_01; ``` ### Querying data @@ -252,7 +252,7 @@ INSERT INTO "test.hive_test".database_01.table_01 (name, salary) SELECT * FROM " Query the `table_01` table: ```sql -SELECT * FROM "test.hive_test".database_01.table_01; +SELECT * FROM hive_test.database_01.table_01; ``` ### Modify a table @@ -260,19 +260,19 @@ SELECT * FROM "test.hive_test".database_01.table_01; Add a new column `age` to the `table_01` table: ```sql -ALTER TABLE "test.hive_test".database_01.table_01 ADD COLUMN age int; +ALTER TABLE hive_test.database_01.table_01 ADD COLUMN age int; ``` Drop a column `age` from the `table_01` table: ```sql -ALTER TABLE "test.hive_test".database_01.table_01 DROP COLUMN age; +ALTER TABLE hive_test.database_01.table_01 DROP COLUMN age; ``` Rename the `table_01` table to `table_02`: ```sql -ALTER TABLE "test.hive_test".database_01.table_01 RENAME TO "test.hive_test".database_01.table_02; +ALTER TABLE hive_test.database_01.table_01 RENAME TO hive_test.database_01.table_02; ``` ### DROP @@ -280,13 +280,13 @@ ALTER TABLE "test.hive_test".database_01.table_01 RENAME TO "test.hive_test".dat Drop a schema: ```sql -DROP SCHEMA "test.hive_test".database_01; +DROP SCHEMA hive_test.database_01; ``` Drop a table: ```sql -DROP TABLE "test.hive_test".database_01.table_01; +DROP TABLE hive_test.database_01.table_01; ``` ## HDFS config and permissions diff --git a/docs/trino-connector/catalog-iceberg.md b/docs/trino-connector/catalog-iceberg.md index 03903f9bd4e..033e3d8938d 100644 --- a/docs/trino-connector/catalog-iceberg.md +++ b/docs/trino-connector/catalog-iceberg.md @@ -141,7 +141,7 @@ The results are similar to: gravitino jmx system - test.iceberg_test + iceberg_test (4 rows) Query 20231017_082503_00018_6nt3n, FINISHED, 1 node @@ -156,13 +156,13 @@ Other catalogs are regular user-configured Trino catalogs. Create a new schema named `database_01` in `test.iceberg_test` catalog. ```sql -CREATE SCHEMA "test.iceberg_test".database_01; +CREATE SCHEMA iceberg_test.database_01; ``` Create a new table named `table_01` in schema `"test.iceberg_test".database_01`. ```sql -CREATE TABLE "test.iceberg_test".database_01.table_01 +CREATE TABLE iceberg_test.database_01.table_01 ( name varchar, salary int @@ -177,13 +177,13 @@ salary int Insert data into the table `table_01`: ```sql -INSERT INTO "test.iceberg_test".database_01.table_01 (name, salary) VALUES ('ice', 12); +INSERT INTO iceberg_test.database_01.table_01 (name, salary) VALUES ('ice', 12); ``` Insert data into the table `table_01` from select: ```sql -INSERT INTO "test.iceberg_test".database_01.table_01 (name, salary) SELECT * FROM "test.iceberg_test".database_01.table_01; +INSERT INTO iceberg_test.database_01.table_01 (name, salary) SELECT * FROM "test.iceberg_test".database_01.table_01; ``` ### Querying data diff --git a/docs/trino-connector/catalog-mysql.md b/docs/trino-connector/catalog-mysql.md index ce0b1298180..0219b3e04cd 100644 --- a/docs/trino-connector/catalog-mysql.md +++ b/docs/trino-connector/catalog-mysql.md @@ -90,7 +90,7 @@ The results are similar to: gravitino jmx system - test.mysql_test + mysql_test (4 rows) Query 20231017_082503_00018_6nt3n, FINISHED, 1 node @@ -105,13 +105,13 @@ Other catalogs are regular user-configured Trino catalogs. Create a new schema named `database_01` in `test.mysql_test` catalog. ```sql -CREATE SCHEMA "test.mysql_test".database_01; +CREATE SCHEMA mysql_test.database_01; ``` -Create a new table named `table_01` in schema `"test.mysql_test".database_01`. +Create a new table named `table_01` in schema `mysql_test.database_01`. ```sql -CREATE TABLE "test.mysql_test".database_01.table_01 +CREATE TABLE mysql_test.database_01.table_01 ( name varchar, salary int @@ -123,13 +123,13 @@ salary int Insert data into the table `table_01`: ```sql -INSERT INTO "test.mysql_test".database_01.table_01 (name, salary) VALUES ('ice', 12); +INSERT INTO mysql_test.database_01.table_01 (name, salary) VALUES ('ice', 12); ``` Insert data into the table `table_01` from select: ```sql -INSERT INTO "test.mysql_test".database_01.table_01 (name, salary) SELECT * FROM "test.mysql_test".database_01.table_01; +INSERT INTO mysql_test.database_01.table_01 (name, salary) SELECT * FROM "test.mysql_test".database_01.table_01; ``` ### Querying data @@ -137,7 +137,7 @@ INSERT INTO "test.mysql_test".database_01.table_01 (name, salary) SELECT * FROM Query the `table_01` table: ```sql -SELECT * FROM "test.mysql_test".database_01.table_01; +SELECT * FROM mysql_test.database_01.table_01; ``` ### Modify a table @@ -145,19 +145,19 @@ SELECT * FROM "test.mysql_test".database_01.table_01; Add a new column `age` to the `table_01` table: ```sql -ALTER TABLE "test.mysql_test".database_01.table_01 ADD COLUMN age int; +ALTER TABLE mysql_test.database_01.table_01 ADD COLUMN age int; ``` Drop a column `age` from the `table_01` table: ```sql -ALTER TABLE "test.mysql_test".database_01.table_01 DROP COLUMN age; +ALTER TABLE mysql_test.database_01.table_01 DROP COLUMN age; ``` Rename the `table_01` table to `table_02`: ```sql -ALTER TABLE "test.mysql_test".database_01.table_01 RENAME TO "test.mysql_test".database_01.table_02; +ALTER TABLE mysql_test.database_01.table_01 RENAME TO "test.mysql_test".database_01.table_02; ``` ### DROP @@ -165,11 +165,11 @@ ALTER TABLE "test.mysql_test".database_01.table_01 RENAME TO "test.mysql_test".d Drop a schema: ```sql -DROP SCHEMA "test.mysql_test".database_01; +DROP SCHEMA mysql_test.database_01; ``` Drop a table: ```sql -DROP TABLE "test.mysql_test".database_01.table_01; +DROP TABLE mysql_test.database_01.table_01; ``` \ No newline at end of file diff --git a/docs/trino-connector/catalog-postgresql.md b/docs/trino-connector/catalog-postgresql.md index ac595b07afe..66b03a92b3b 100644 --- a/docs/trino-connector/catalog-postgresql.md +++ b/docs/trino-connector/catalog-postgresql.md @@ -90,7 +90,7 @@ The results are similar to: gravitino jmx system - test.postgresql_test + postgresql_test (4 rows) Query 20231017_082503_00018_6nt3n, FINISHED, 1 node @@ -102,16 +102,16 @@ Other catalogs are regular user-configured Trino catalogs. ### Creating tables and schemas -Create a new schema named `database_01` in `test.postgresql_test` catalog. +Create a new schema named `database_01` in `postgresql_test` catalog. ```sql -CREATE SCHEMA "test.postgresql_test".database_01; +CREATE SCHEMA postgresql_test.database_01; ``` Create a new table named `table_01` in schema `"test.postgresql_test".database_01`. ```sql -CREATE TABLE "test.postgresql_test".database_01.table_01 +CREATE TABLE postgresql_test.database_01.table_01 ( name varchar, salary int @@ -123,13 +123,13 @@ salary int Insert data into the table `table_01`: ```sql -INSERT INTO "test.postgresql_test".database_01.table_01 (name, salary) VALUES ('ice', 12); +INSERT INTO postgresql_test.database_01.table_01 (name, salary) VALUES ('ice', 12); ``` Insert data into the table `table_01` from select: ```sql -INSERT INTO "test.postgresql_test".database_01.table_01 (name, salary) SELECT * FROM "test.postgresql_test".database_01.table_01; +INSERT INTO postgresql_test.database_01.table_01 (name, salary) SELECT * FROM "test.postgresql_test".database_01.table_01; ``` ### Querying data @@ -137,7 +137,7 @@ INSERT INTO "test.postgresql_test".database_01.table_01 (name, salary) SELECT * Query the `table_01` table: ```sql -SELECT * FROM "test.postgresql_test".database_01.table_01; +SELECT * FROM postgresql_test.database_01.table_01; ``` ### Modify a table @@ -145,19 +145,19 @@ SELECT * FROM "test.postgresql_test".database_01.table_01; Add a new column `age` to the `table_01` table: ```sql -ALTER TABLE "test.postgresql_test".database_01.table_01 ADD COLUMN age int; +ALTER TABLE postgresql_test.database_01.table_01 ADD COLUMN age int; ``` Drop a column `age` from the `table_01` table: ```sql -ALTER TABLE "test.postgresql_test".database_01.table_01 DROP COLUMN age; +ALTER TABLE postgresql_test.database_01.table_01 DROP COLUMN age; ``` Rename the `table_01` table to `table_02`: ```sql -ALTER TABLE "test.postgresql_test".database_01.table_01 RENAME TO "test.postgresql_test".database_01.table_02; +ALTER TABLE postgresql_test.database_01.table_01 RENAME TO "test.postgresql_test".database_01.table_02; ``` ### Drop @@ -165,11 +165,11 @@ ALTER TABLE "test.postgresql_test".database_01.table_01 RENAME TO "test.postgres Drop a schema: ```sql -DROP SCHEMA "test.postgresql_test".database_01; +DROP SCHEMA postgresql_test.database_01; ``` Drop a table: ```sql -DROP TABLE "test.postgresql_test".database_01.table_01; +DROP TABLE postgresql_test.database_01.table_01; ``` \ No newline at end of file diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index cd1e4c6684f..1b6bf02d8e9 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -6,9 +6,9 @@ license: "Copyright 2023 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2." --- -| Property | Type | Default Value | Description | Required | Since Version | +| Property | Type | Default Value | Description | Required | Since Version | |-----------------------------------|---------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| -| connector.name | string | (none) | The `connector.name` defines the name of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | +| connector.name | string | (none) | The `connector.name` defines the type of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | | gravitino.metalake | string | (none) | The `gravitino.metalake` defines which metalake in Gravitino server the Trino connector uses. Trino connector should set it at start, the value of `gravitino.metalake` needs to be a valid name, Trino connector can detect and load the metalake with catalogs, schemas and tables once created and keep in sync. | Yes | 0.2.0 | | gravitino.uri | string | http://localhost:8090 | The `gravitino.uri` defines the connection URL of the Gravitino server, the default value is `http://localhost:8090`. Trino connector can detect and connect to Gravitino server once it is ready, no need to start Gravitino server beforehand. | Yes | 0.2.0 | -| gravitino.simplify-catalog-names | boolean | true | The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. If you set it to true, Trino will configure only one Graviton catalog. | NO | 0.5.0 | +| gravitino.simplify-catalog-names | boolean | true | The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. | NO | 0.5.0 | diff --git a/docs/trino-connector/development.md b/docs/trino-connector/development.md index 1c7a7b167af..c1393e14c87 100644 --- a/docs/trino-connector/development.md +++ b/docs/trino-connector/development.md @@ -249,6 +249,7 @@ gravitino.uri=http://localhost:8090 # The name of the metalake to which the connector is connected, you need to change it according to your environment gravitino.metalake=test + ``` - Trino configuration file: `config.properties` ```properties diff --git a/docs/trino-connector/installation.md b/docs/trino-connector/installation.md index 707d1d3802b..2fb44c53c35 100644 --- a/docs/trino-connector/installation.md +++ b/docs/trino-connector/installation.md @@ -74,11 +74,13 @@ To configure Gravitino connector correctly, you need to put the following config connector.name=gravitino gravitino.uri=http://gravitino-server-host:8090 gravitino.metalake=test +gravitino.simplify-catalog-names=true ``` - The `gravitino.name` defines which Gravitino connector is used. It must be `gravitino`. - The `gravitino.metalake` defines which metalake are used. It should exist in the Gravitino server. - The `gravitino.uri` defines the connection information about Gravitino server. Make sure your container can access the Gravitino server. +- The `gravitino.simplify-catalog-names` setting omits the metalake prefix from catalog names when set to true. Full configurations for Gravitino connector can be seen [here](configuration.md) @@ -126,7 +128,7 @@ memory tpcds tpch system -test.jdbc-mysql +jdbc-mysql ``` -The catalog named 'test.jdbc-mysql' is the catalog that you created by gravitino server, and you can use it to access the mysql database like other Trino catalogs. +The catalog named 'jdbc-mysql' is the catalog that you created by gravitino server, and you can use it to access the mysql database like other Trino catalogs. diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index b578e668a51..158002c469f 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -75,7 +75,7 @@ The result is like: ```test name | provider | properties --------------+----------+------------------------------------------------------------------------------------------------------------- - test.gt_hive | hive | {gravitino.bypass.hive.metastore.client.capability.check=false, metastore.uris=thrift://trino-ci-hive:9083} + gt_hive | hive | {gravitino.bypass.hive.metastore.client.capability.check=false, metastore.uris=thrift://trino-ci-hive:9083} ``` Example: diff --git a/docs/trino-connector/trino-connector.md b/docs/trino-connector/trino-connector.md index 1858d695963..648fc579034 100644 --- a/docs/trino-connector/trino-connector.md +++ b/docs/trino-connector/trino-connector.md @@ -15,17 +15,14 @@ Once metadata such as catalogs, schemas, or tables are changed in Gravitino, Tri about 3~10 seconds. ::: -The loading of Gravitino's catalogs into Trino follows the naming convention: +By default, the loading of Gravitino's catalogs into Trino follows the naming convention: ```text -{metalake}.{catalog} +{catalog} ``` -Regarding `metalake` and `catalog`, -you can refer to [Create a Metalake](../manage-relational-metadata-using-gravitino.md#create-a-metalake), [Create a Catalog](../manage-relational-metadata-using-gravitino.md#create-a-catalog). - Usage in queries is as follows: ```text -SELECT * from "metalake.catalog".dbname.tabname +SELECT * from catalog.dbname.tabname ``` From 5bc83ccad294400a8e9b82fe881198acae182c18 Mon Sep 17 00:00:00 2001 From: Peidian li <38486782+coolderli@users.noreply.github.com> Date: Fri, 19 Apr 2024 20:00:00 +0800 Subject: [PATCH 071/106] [#2985] refactor: use disable error-prone instead of enable error-prone (#2986) ### What changes were proposed in this pull request? - use disable error-prone instead of enable error-prone to make it more clear. ### Why are the changes needed? Fix: #2985 ### Does this PR introduce _any_ user-facing change? - no ### How was this patch tested? - Github CI Pass --------- Co-authored-by: Jerry Shao --- build.gradle.kts | 127 +++++------------- .../gravitino/client/GravitinoClientBase.java | 4 +- .../client/TestGravitinoVersion.java | 2 +- core/build.gradle.kts | 2 + gradle/libs.versions.toml | 2 + .../test/web/rest/KerberosOperationsIT.java | 5 +- .../server/authentication/KerberosConfig.java | 3 +- .../server/authentication/OAuthConfig.java | 4 +- 8 files changed, 47 insertions(+), 102 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 20d08c02d96..6004c3a60df 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -282,100 +282,41 @@ subprojects { tasks.withType().configureEach { options.errorprone.isEnabled.set(true) - options.errorprone.disableAllChecks.set(true) options.errorprone.disableWarningsInGeneratedCode.set(true) - options.errorprone.enable( - "AnnotateFormatMethod", - "AlwaysThrows", - "ArrayEquals", - "ArrayToString", - "ArraysAsListPrimitiveArray", - "ArrayFillIncompatibleType", - "BadImport", - "BoxedPrimitiveEquality", - "ChainingConstructorIgnoresParameter", - "CheckNotNullMultipleTimes", - "ClassCanBeStatic", - "CollectionIncompatibleType", - "CollectionToArraySafeParameter", - "ComparingThisWithNull", - "ComparisonOutOfRange", - "CompatibleWithAnnotationMisuse", - "CompileTimeConstant", - "ConditionalExpressionNumericPromotion", - "DangerousLiteralNull", - "DeadException", - "DeadThread", - "DefaultCharset", - "DoNotCall", - "DoNotMock", - "DuplicateMapKeys", - "EqualsGetClass", - "EqualsNaN", - "EqualsNull", - "EqualsReference", - "EqualsWrongThing", - "ForOverride", - "FormatString", - "FormatStringAnnotation", - "GetClassOnAnnotation", - "GetClassOnClass", - "HashtableContains", - "IdentityBinaryExpression", - "IdentityHashMapBoxing", - "Immutable", - "ImmutableEnumChecker", - "Incomparable", - "IncompatibleArgumentType", - "IndexOfChar", - "InfiniteRecursion", - "InlineFormatString", - "InvalidJavaTimeConstant", - "InvalidPatternSyntax", - "IsInstanceIncompatibleType", - "JavaUtilDate", - "JUnit4ClassAnnotationNonStatic", - "JUnit4SetUpNotRun", - "JUnit4TearDownNotRun", - "JUnit4TestNotRun", - "JUnitAssertSameCheck", - "LockOnBoxedPrimitive", - "LoopConditionChecker", - "LossyPrimitiveCompare", - "MathRoundIntLong", - "MissingSuperCall", - "ModifyingCollectionWithItself", - "MutablePublicArray", - "NonCanonicalStaticImport", - "NonFinalCompileTimeConstant", - "NonRuntimeAnnotation", - "NullTernary", - "OptionalEquality", - "PackageInfo", - "ParametersButNotParameterized", - "RandomCast", - "RandomModInteger", - "ReferenceEquality", - "SelfAssignment", - "SelfComparison", - "SelfEquals", - "SizeGreaterThanOrEqualsZero", - "StaticGuardedByInstance", - "StreamToString", - "StringBuilderInitWithChar", - "SubstringOfZero", - "ThrowNull", - "TruthSelfEquals", - "TryFailThrowable", - "TypeParameterQualifier", - "UnnecessaryCheckNotNull", - "UnnecessaryTypeArgument", - "UnusedAnonymousClass", - "UnusedCollectionModifiedInPlace", - "UnusedVariable", - "UseCorrectAssertInTests", - "VarTypeName", - "XorPower" + options.errorprone.disable( + "AlmostJavadoc", + "CanonicalDuration", + "CheckReturnValue", + "ComparableType", + "ConstantOverflow", + "DoubleBraceInitialization", + "EqualsUnsafeCast", + "EmptyBlockTag", + "FutureReturnValueIgnored", + "InconsistentCapitalization", + "InconsistentHashCode", + "JavaTimeDefaultTimeZone", + "JdkObsolete", + "LockNotBeforeTry", + "MissingSummary", + "MissingOverride", + "MutableConstantField", + "NonOverridingEquals", + "ObjectEqualsForPrimitives", + "OperatorPrecedence", + "ReturnValueIgnored", + "SameNameButDifferent", + "StaticAssignmentInConstructor", + "StringSplitter", + "ThreadPriorityCheck", + "ThrowIfUncheckedKnownChecked", + "TypeParameterUnusedInFormals", + "UnicodeEscape", + "UnnecessaryParentheses", + "UnsafeReflectiveConstructionCast", + "UnusedMethod", + "VariableNameSameAsType", + "WaitNotInLoop" ) } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java index 6b83af03623..04490659816 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.InlineMe; import java.io.Closeable; import java.net.URI; import java.net.URISyntaxException; @@ -128,7 +129,8 @@ public GravitinoMetalake loadMetalake(NameIdentifier ident) throws NoSuchMetalak * @return A GravitinoVersion instance representing the version of the Gravitino API. */ @Deprecated - public GravitinoVersion getVersion() { + @InlineMe(replacement = "this.serverVersion()") + public final GravitinoVersion getVersion() { return serverVersion(); } diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java index d9ca564359f..c94b613af67 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java @@ -58,7 +58,7 @@ void testVersionCompare() { // test less than version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); version2 = new GravitinoVersion("2.5.4", "2023-01-01", "1234567"); - assertTrue(version1.compareTo(version2) < 1); + assertTrue(version1.compareTo(version2) < 0); // test greater than version1 = new GravitinoVersion("2.5.3", "2023-01-01", "1234567"); diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 983bbb59c02..a07b29ae5b7 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -28,7 +28,9 @@ dependencies { implementation(libs.rocksdbjni) annotationProcessor(libs.lombok) + compileOnly(libs.lombok) + compileOnly(libs.servlet) // fix error-prone compile error testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e5dbf1a46fe..c581d27a1ca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -53,6 +53,7 @@ kyuubi = "1.8.0" kafka = "3.4.0" curator = "2.12.0" awaitility = "4.2.1" +servlet = "3.1.0" protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' @@ -164,6 +165,7 @@ rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rausch mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} h2db = { group = "com.h2database", name = "h2", version.ref = "h2db"} awaitility = { group = "org.awaitility", name = "awaitility", version.ref = "awaitility" } +servlet = { group = "javax.servlet", name = "javax.servlet-api", version.ref = "servlet" } [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"] diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java index 0c28d1b5d2b..2a48024f353 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/rest/KerberosOperationsIT.java @@ -9,13 +9,13 @@ import static com.datastrato.gravitino.server.authentication.KerberosConfig.PRINCIPAL; import static org.apache.hadoop.minikdc.MiniKdc.MAX_TICKET_LIFETIME; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.client.GravitinoVersion; import com.datastrato.gravitino.client.KerberosTokenProvider; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.ITUtils; import com.datastrato.gravitino.integration.test.util.KerberosProviderHelper; -import com.datastrato.gravitino.server.authentication.KerberosConfig; import com.google.common.collect.Maps; import java.io.File; import java.io.IOException; @@ -59,8 +59,7 @@ public static void startIntegrationTest() throws Exception { .withClientPrincipal(clientPrincipal) .withKeyTabFile(new File(keytabFile)) .build()); - configs.put( - KerberosConfig.AUTHENTICATOR.getKey(), AuthenticatorType.KERBEROS.name().toLowerCase()); + configs.put(Configs.AUTHENTICATOR.getKey(), AuthenticatorType.KERBEROS.name().toLowerCase()); configs.put(PRINCIPAL.getKey(), serverPrincipal); configs.put(KEYTAB.getKey(), keytabFile); registerCustomConfigs(configs); diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/authentication/KerberosConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/authentication/KerberosConfig.java index 53573cda7e1..bdb4019e16b 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/authentication/KerberosConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/authentication/KerberosConfig.java @@ -4,13 +4,12 @@ */ package com.datastrato.gravitino.server.authentication; -import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import org.apache.commons.lang3.StringUtils; -public interface KerberosConfig extends Configs { +public interface KerberosConfig { String KERBEROS_CONFIG_PREFIX = "gravitino.authenticator.kerberos."; diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/authentication/OAuthConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/authentication/OAuthConfig.java index 820dec0578e..70f44c9600f 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/authentication/OAuthConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/authentication/OAuthConfig.java @@ -5,14 +5,14 @@ package com.datastrato.gravitino.server.authentication; -import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import io.jsonwebtoken.SignatureAlgorithm; import org.apache.commons.lang3.StringUtils; -public interface OAuthConfig extends Configs { +public interface OAuthConfig { + String OAUTH_CONFIG_PREFIX = "gravitino.authenticator.oauth."; ConfigEntry SERVICE_AUDIENCE = From 589ac449af126ff54fa32c114323b360951670f5 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Fri, 19 Apr 2024 20:49:22 +0800 Subject: [PATCH 072/106] [#2241] feat(client,server): Support the permission operations for both server and client side. (#3006) ### What changes were proposed in this pull request? Support the permission operations for both server and client side. ### Why are the changes needed? Fix: #2241 ### Does this PR introduce _any_ user-facing change? Yes, I will add the document later. ### How was this patch tested? Add new uts. --------- Co-authored-by: Heng Qin --- .../gravitino/client/ErrorHandlers.java | 45 +++ .../client/GravitinoAdminClient.java | 127 +++++- .../gravitino/client/TestPermission.java | 145 +++++++ .../dto/requests/RoleGrantRequest.java | 52 +++ .../dto/requests/RoleRevokeRequest.java | 52 +++ .../authorization/AccessControlManager.java | 90 +++-- .../authorization/PermissionManager.java | 123 +++--- ...estAccessControlManagerForPermissions.java | 100 +++-- server/build.gradle.kts | 1 + .../server/web/rest/ExceptionHandlers.java | 60 +++ .../server/web/rest/OperationType.java | 4 +- .../server/web/rest/PermissionOperations.java | 136 +++++++ .../web/rest/TestPermissionOperations.java | 382 ++++++++++++++++++ 13 files changed, 1174 insertions(+), 143 deletions(-) create mode 100644 clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java create mode 100644 server/src/main/java/com/datastrato/gravitino/server/web/rest/PermissionOperations.java create mode 100644 server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPermissionOperations.java diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java index d140b0c2a70..9e322fbaf8a 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ErrorHandlers.java @@ -155,6 +155,14 @@ public static Consumer groupErrorHandler() { public static Consumer roleErrorHandler() { return RoleErrorHandler.INSTANCE; } + /** + * Creates an error handler specific to permission operations. + * + * @return A Consumer representing the permission error handler. + */ + public static Consumer permissionOperationErrorHandler() { + return PermissionOperationErrorHandler.INSTANCE; + } private ErrorHandlers() {} @@ -597,6 +605,43 @@ public void accept(ErrorResponse errorResponse) { } } + /** Error handler specific to Permission operations. */ + @SuppressWarnings("FormatStringAnnotation") + private static class PermissionOperationErrorHandler extends RestErrorHandler { + + private static final PermissionOperationErrorHandler INSTANCE = + new PermissionOperationErrorHandler(); + + @Override + public void accept(ErrorResponse errorResponse) { + String errorMessage = formatErrorMessage(errorResponse); + + switch (errorResponse.getCode()) { + case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: + throw new IllegalArgumentException(errorMessage); + + case ErrorConstants.NOT_FOUND_CODE: + if (errorResponse.getType().equals(NoSuchMetalakeException.class.getSimpleName())) { + throw new NoSuchMetalakeException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchUserException.class.getSimpleName())) { + throw new NoSuchUserException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchGroupException.class.getSimpleName())) { + throw new NoSuchGroupException(errorMessage); + } else if (errorResponse.getType().equals(NoSuchRoleException.class.getSimpleName())) { + throw new NoSuchRoleException(errorMessage); + } else { + throw new NotFoundException(errorMessage); + } + + case ErrorConstants.INTERNAL_ERROR_CODE: + throw new RuntimeException(errorMessage); + + default: + super.accept(errorResponse); + } + } + } + /** Generic error handler for REST requests. */ private static class RestErrorHandler extends ErrorHandler { private static final ErrorHandler INSTANCE = new RestErrorHandler(); diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index 07675037f2f..4cea0f062d8 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -18,6 +18,8 @@ import com.datastrato.gravitino.dto.requests.MetalakeUpdateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdatesRequest; import com.datastrato.gravitino.dto.requests.RoleCreateRequest; +import com.datastrato.gravitino.dto.requests.RoleGrantRequest; +import com.datastrato.gravitino.dto.requests.RoleRevokeRequest; import com.datastrato.gravitino.dto.requests.UserAddRequest; import com.datastrato.gravitino.dto.responses.DeleteResponse; import com.datastrato.gravitino.dto.responses.DropResponse; @@ -58,6 +60,8 @@ public class GravitinoAdminClient extends GravitinoClientBase implements Support private static final String API_METALAKES_GROUPS_PATH = "api/metalakes/%s/groups/%s"; private static final String API_METALAKES_ROLES_PATH = "api/metalakes/%s/roles/%s"; private static final String API_ADMIN_PATH = "api/admins/%s"; + private static final String API_PERMISSION_PATH = "api/metalakes/%s/permissions/%s"; + private static final String BLANK_PLACE_HOLDER = ""; /** * Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. @@ -203,7 +207,7 @@ public User addUser(String metalake, String user) UserResponse resp = restClient.post( - String.format(API_METALAKES_USERS_PATH, metalake, ""), + String.format(API_METALAKES_USERS_PATH, metalake, BLANK_PLACE_HOLDER), req, UserResponse.class, Collections.emptyMap(), @@ -218,7 +222,7 @@ public User addUser(String metalake, String user) * * @param metalake The Metalake of the User. * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` only when there's no such user, + * @return True if the User was successfully removed, false only when there's no such user, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the User encounters storage issues. @@ -275,7 +279,7 @@ public Group addGroup(String metalake, String group) GroupResponse resp = restClient.post( - String.format(API_METALAKES_GROUPS_PATH, metalake, ""), + String.format(API_METALAKES_GROUPS_PATH, metalake, BLANK_PLACE_HOLDER), req, GroupResponse.class, Collections.emptyMap(), @@ -290,7 +294,7 @@ public Group addGroup(String metalake, String group) * * @param metalake The Metalake of the Group. * @param group THe name of the Group. - * @return `true` if the Group was successfully removed, `false` only when there's no such group, + * @return True if the Group was successfully removed, false only when there's no such group, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the Group encounters storage issues. @@ -344,7 +348,7 @@ public User addMetalakeAdmin(String user) throws UserAlreadyExistsException { UserResponse resp = restClient.post( - String.format(API_ADMIN_PATH, ""), + String.format(API_ADMIN_PATH, BLANK_PLACE_HOLDER), req, UserResponse.class, Collections.emptyMap(), @@ -358,7 +362,7 @@ public User addMetalakeAdmin(String user) throws UserAlreadyExistsException { * Removes a metalake admin. * * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` only when there's no such metalake + * @return True if the User was successfully removed, false only when there's no such metalake * admin, otherwise it will throw an exception. * @throws RuntimeException If removing the User encounters storage issues. */ @@ -402,7 +406,7 @@ public Role getRole(String metalake, String role) * * @param metalake The Metalake of the Role. * @param role The name of the Role. - * @return `true` if the Role was successfully deleted, `false` only when there's no such role, + * @return True if the Role was successfully deleted, false only when there's no such role, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If deleting the Role encounters storage issues. @@ -452,7 +456,7 @@ public Role createRole( RoleResponse resp = restClient.post( - String.format(API_METALAKES_ROLES_PATH, metalake, ""), + String.format(API_METALAKES_ROLES_PATH, metalake, BLANK_PLACE_HOLDER), req, RoleResponse.class, Collections.emptyMap(), @@ -461,6 +465,113 @@ public Role createRole( return resp.getRole(); } + /** + * Grant roles to a user. + * + * @param metalake The metalake of the User. + * @param user The name of the User. + * @param roles The names of the Role. + * @return The Group after granted. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If granting roles to a user encounters storage issues. + */ + public User grantRolesToUser(String metalake, List roles, String user) + throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + RoleGrantRequest request = new RoleGrantRequest(roles); + UserResponse resp = + restClient.put( + String.format(API_PERMISSION_PATH, metalake, String.format("users/%s/grant", user)), + request, + UserResponse.class, + Collections.emptyMap(), + ErrorHandlers.permissionOperationErrorHandler()); + resp.validate(); + + return resp.getUser(); + } + + /** + * Grant roles to a group. + * + * @param metalake The metalake of the Group. + * @param group The name of the Group. + * @param roles The names of the Role. + * @return The Group after granted. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If granting roles to a group encounters storage issues. + */ + public Group grantRolesToGroup(String metalake, List roles, String group) + throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + RoleGrantRequest request = new RoleGrantRequest(roles); + GroupResponse resp = + restClient.put( + String.format(API_PERMISSION_PATH, metalake, String.format("groups/%s/grant", group)), + request, + GroupResponse.class, + Collections.emptyMap(), + ErrorHandlers.permissionOperationErrorHandler()); + resp.validate(); + + return resp.getGroup(); + } + + /** + * Revoke roles from a user. + * + * @param metalake The metalake of the User. + * @param user The name of the User. + * @param roles The names of the Role. + * @return The User after revoked. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If revoking roles from a user encounters storage issues. + */ + public User revokeRolesFromUser(String metalake, List roles, String user) + throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + RoleRevokeRequest request = new RoleRevokeRequest(roles); + UserResponse resp = + restClient.put( + String.format(API_PERMISSION_PATH, metalake, String.format("users/%s/revoke", user)), + request, + UserResponse.class, + Collections.emptyMap(), + ErrorHandlers.permissionOperationErrorHandler()); + resp.validate(); + + return resp.getUser(); + } + + /** + * Revoke roles from a group. + * + * @param metalake The metalake of the Group. + * @param group The name of the Group. + * @param roles The names of the Role. + * @return The Group after revoked. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If revoking roles from a group encounters storage issues. + */ + public Group revokeRolesFromGroup(String metalake, List roles, String group) + throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + RoleRevokeRequest request = new RoleRevokeRequest(roles); + GroupResponse resp = + restClient.put( + String.format(API_PERMISSION_PATH, metalake, String.format("groups/%s/revoke", group)), + request, + GroupResponse.class, + Collections.emptyMap(), + ErrorHandlers.permissionOperationErrorHandler()); + resp.validate(); + + return resp.getGroup(); + } /** * Creates a new builder for constructing a GravitinoClient. diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java new file mode 100644 index 00000000000..69ac63c1ccf --- /dev/null +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java @@ -0,0 +1,145 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.client; + +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; + +import com.datastrato.gravitino.authorization.Group; +import com.datastrato.gravitino.authorization.User; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.authorization.GroupDTO; +import com.datastrato.gravitino.dto.authorization.UserDTO; +import com.datastrato.gravitino.dto.requests.RoleGrantRequest; +import com.datastrato.gravitino.dto.requests.RoleRevokeRequest; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.GroupResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.google.common.collect.Lists; +import java.time.Instant; +import java.util.List; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestPermission extends TestBase { + + private static final String metalakeName = "testMetalake"; + private static final String API_PERMISSION_PATH = "/api/metalakes/%s/permissions/%s"; + + @BeforeAll + public static void setUp() throws Exception { + TestBase.setUp(); + } + + @Test + public void testGrantRolesToUser() throws Exception { + List roles = Lists.newArrayList("role"); + String user = "user"; + String userPath = + String.format(API_PERMISSION_PATH, metalakeName, String.format("users/%s/grant", user)); + RoleGrantRequest request = new RoleGrantRequest(roles); + UserDTO userDTO = + UserDTO.builder() + .withName("user") + .withRoles(Lists.newArrayList("roles")) + .withAudit(AuditDTO.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + UserResponse response = new UserResponse(userDTO); + + buildMockResource(Method.PUT, userPath, request, response, SC_OK); + User grantedUser = client.grantRolesToUser(metalakeName, roles, user); + Assertions.assertEquals(grantedUser.roles(), userDTO.roles()); + Assertions.assertEquals(grantedUser.name(), userDTO.name()); + + // test Exception + ErrorResponse errResp2 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.PUT, userPath, request, errResp2, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.grantRolesToUser(metalakeName, roles, user)); + } + + @Test + public void testRevokeRolesFromUser() throws Exception { + List roles = Lists.newArrayList("role"); + String user = "user"; + String userPath = + String.format(API_PERMISSION_PATH, metalakeName, String.format("users/%s/revoke", user)); + UserDTO userDTO = + UserDTO.builder() + .withName("user") + .withRoles(Lists.newArrayList()) + .withAudit(AuditDTO.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + UserResponse response = new UserResponse(userDTO); + RoleRevokeRequest request = new RoleRevokeRequest(roles); + + buildMockResource(Method.PUT, userPath, request, response, SC_OK); + User revokedUser = client.revokeRolesFromUser(metalakeName, roles, user); + Assertions.assertEquals(revokedUser.roles(), userDTO.roles()); + Assertions.assertEquals(revokedUser.name(), userDTO.name()); + + // test Exception + ErrorResponse errResp2 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.PUT, userPath, null, errResp2, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.revokeRolesFromUser(metalakeName, roles, user)); + } + + @Test + public void testGrantRolesToGroup() throws Exception { + List roles = Lists.newArrayList("role"); + String group = "group"; + String groupPath = + String.format(API_PERMISSION_PATH, metalakeName, String.format("groups/%s/grant", group)); + RoleGrantRequest request = new RoleGrantRequest(roles); + GroupDTO groupDTO = + GroupDTO.builder() + .withName("group") + .withRoles(Lists.newArrayList("roles")) + .withAudit(AuditDTO.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + GroupResponse response = new GroupResponse(groupDTO); + + buildMockResource(Method.PUT, groupPath, request, response, SC_OK); + Group grantedGroup = client.grantRolesToGroup(metalakeName, roles, group); + Assertions.assertEquals(grantedGroup.roles(), groupDTO.roles()); + Assertions.assertEquals(grantedGroup.name(), groupDTO.name()); + + // test Exception + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.POST, groupPath, request, errResp, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.grantRolesToGroup(metalakeName, roles, group)); + } + + @Test + public void testRevokeRoleFromGroup() throws Exception { + List roles = Lists.newArrayList("role"); + String group = "group"; + String groupPath = + String.format(API_PERMISSION_PATH, metalakeName, String.format("groups/%s/revoke", group)); + GroupDTO groupDTO = + GroupDTO.builder() + .withName("group") + .withRoles(Lists.newArrayList()) + .withAudit(AuditDTO.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + GroupResponse response = new GroupResponse(groupDTO); + RoleRevokeRequest request = new RoleRevokeRequest(roles); + + buildMockResource(Method.PUT, groupPath, request, response, SC_OK); + Group revokedGroup = client.revokeRolesFromGroup(metalakeName, roles, group); + Assertions.assertEquals(revokedGroup.roles(), groupDTO.roles()); + Assertions.assertEquals(revokedGroup.name(), groupDTO.name()); + + // test Exception + ErrorResponse errResp = ErrorResponse.internalError("internal error"); + buildMockResource(Method.DELETE, groupPath, null, errResp, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, () -> client.revokeRolesFromGroup(metalakeName, roles, group)); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java new file mode 100644 index 00000000000..2bd6880ee21 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.List; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.jackson.Jacksonized; + +/** Represents a request to grant roles to the user or the group. */ +@Getter +@EqualsAndHashCode +@ToString +@Builder +@Jacksonized +public class RoleGrantRequest implements RESTRequest { + @JsonProperty("roleNames") + private final List roleNames; + + /** + * Constructor for RoleGrantRequest. + * + * @param roleNames The roleName for the RoleGrantRequest. + */ + public RoleGrantRequest(List roleNames) { + this.roleNames = roleNames; + } + + /** Default constructor for RoleGrantRequest. */ + public RoleGrantRequest() { + this(null); + } + + /** + * Validates the fields of the request. + * + * @throws IllegalArgumentException if the role names is not set or empty. + */ + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + roleNames != null && !roleNames.isEmpty(), + "\"roleName\" field is required and cannot be empty"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java new file mode 100644 index 00000000000..02f3a3ac31b --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.List; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.jackson.Jacksonized; + +/** Represents a request to revoke roles from the user or the group. */ +@Getter +@EqualsAndHashCode +@ToString +@Builder +@Jacksonized +public class RoleRevokeRequest implements RESTRequest { + @JsonProperty("roleNames") + private final List roleNames; + + /** + * Constructor for RoleRevokeRequest. + * + * @param roleNames The roleName for the RoleRevokeRequest. + */ + public RoleRevokeRequest(List roleNames) { + this.roleNames = roleNames; + } + + /** Default constructor for RoleRevokeRequest. */ + public RoleRevokeRequest() { + this(null); + } + + /** + * Validates the fields of the request. + * + * @throws IllegalArgumentException if the role names is not set or empty. + */ + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + roleNames != null && !roleNames.isEmpty(), + "\"roleName\" field is required and cannot be empty"); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index 35c3ea99f47..5f1843614f6 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -63,7 +63,7 @@ public User addUser(String metalake, String user) * * @param metalake The Metalake of the User. * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` only when there's no such user, + * @return True if the User was successfully removed, false only when there's no such user, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the User encounters storage issues. @@ -107,7 +107,7 @@ public Group addGroup(String metalake, String group) * * @param metalake The Metalake of the Group. * @param group THe name of the Group. - * @return `true` if the Group was successfully removed, `false` only when there's no such group, + * @return True if the Group was successfully removed, false only when there's no such group, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If removing the Group encounters storage issues. @@ -132,67 +132,71 @@ public Group getGroup(String metalake, String group) } /** - * Grant a role to a user. + * Grant roles to a user. * * @param metalake The metalake of the User. * @param user The name of the User. - * @return true` if the User was successfully granted, `false` otherwise. - * @throws NoSuchUserException If the User with the given identifier does not exist. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the - * User. - * @throws RuntimeException If granting a role to a user encounters storage issues. + * @param roles The names of the Role. + * @return The User after granted. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If granting roles to a user encounters storage issues. */ - public boolean grantRoleToUser(String metalake, String role, String user) { - return doWithNonAdminLock(() -> permissionManager.grantRoleToUser(metalake, role, user)); + public User grantRolesToUser(String metalake, List roles, String user) + throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> permissionManager.grantRolesToUser(metalake, roles, user)); } /** - * Grant a role to a group. + * Grant roles to a group. * * @param metalake The metalake of the Group. - * @param group THe name of the Group. - * @return true` if the Group was successfully granted, `false` otherwise. - * @throws NoSuchGroupException If the Group with the given identifier does not exist. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the - * Group. - * @throws RuntimeException If granting a role to a group encounters storage issues. + * @param group The name of the Group. + * @param roles The names of the Role. + * @return The Group after granted. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If granting roles to a group encounters storage issues. */ - public boolean grantRoleToGroup(String metalake, String role, String group) { - return doWithNonAdminLock(() -> permissionManager.grantRoleToGroup(metalake, role, group)); + public Group grantRolesToGroup(String metalake, List roles, String group) + throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> permissionManager.grantRolesToGroup(metalake, roles, group)); } /** - * Revoke a role from a group. + * Revoke roles from a group. * * @param metalake The metalake of the Group. * @param group The name of the Group. - * @return true` if the Group was successfully revoked, `false` otherwise. - * @throws NoSuchGroupException If the Group with the given identifier does not exist. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the - * Group. - * @throws RuntimeException If revoking a role from a group encounters storage issues. + * @param roles The name of the Role. + * @return The Group after revoked. + * @throws NoSuchGroupException If the Group with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If revoking roles from a group encounters storage issues. */ - public boolean revokeRoleFromGroup(String metalake, String role, String group) { - return doWithNonAdminLock(() -> permissionManager.revokeRoleFromGroup(metalake, role, group)); + public Group revokeRolesFromGroup(String metalake, List roles, String group) + throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> permissionManager.revokeRolesFromGroup(metalake, roles, group)); } /** - * Revoke a role from a user. + * Revoke roles from a user. * * @param metalake The metalake of the User. * @param user The name of the User. - * @return true` if the User was successfully revoked, `false` otherwise. - * @throws NoSuchUserException If the User with the given identifier does not exist. - * @throws NoSuchRoleException If the Role with the given identifier does not exist. - * @throws RoleAlreadyExistsException If the Role with the given identifier already exists in the - * User. - * @throws RuntimeException If revoking a role from a user encounters storage issues. + * @param roles The name of the Role. + * @return The User after revoked. + * @throws NoSuchUserException If the User with the given name does not exist. + * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws RuntimeException If revoking roles from a user encounters storage issues. */ - public boolean revokeRoleFromUser(String metalake, String role, String user) { - return doWithNonAdminLock(() -> permissionManager.revokeRoleFromUser(metalake, role, user)); + public User revokeRolesFromUser(String metalake, List roles, String user) + throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + return doWithNonAdminLock(() -> permissionManager.revokeRolesFromUser(metalake, roles, user)); } /** @@ -211,7 +215,7 @@ public User addMetalakeAdmin(String user) throws UserAlreadyExistsException { * Removes a metalake admin. * * @param user The name of the User. - * @return `true` if the User was successfully removed, `false` only when there's no such metalake + * @return True if the User was successfully removed, false only when there's no such metalake * admin, otherwise it will throw an exception. * @throws RuntimeException If removing the User encounters storage issues. */ @@ -223,7 +227,7 @@ public boolean removeMetalakeAdmin(String user) { * Judges whether the user is the service admin. * * @param user the name of the user - * @return true, if the user is service admin, otherwise false. + * @return True if the user is service admin, otherwise false. */ public boolean isServiceAdmin(String user) { return adminManager.isServiceAdmin(user); @@ -233,7 +237,7 @@ public boolean isServiceAdmin(String user) { * Judges whether the user is the metalake admin. * * @param user the name of the user - * @return true, if the user is metalake admin, otherwise false. + * @return True if the user is metalake admin, otherwise false. */ public boolean isMetalakeAdmin(String user) { return doWithAdminLock(() -> adminManager.isMetalakeAdmin(user)); @@ -283,7 +287,7 @@ public Role getRole(String metalake, String role) * * @param metalake The Metalake of the Role. * @param role The name of the Role. - * @return `true` if the Role was successfully deleted, `false` only when there's no such role, + * @return True if the Role was successfully deleted, false only when there's no such role, * otherwise it will throw an exception. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If deleting the Role encounters storage issues. diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java index a77eb191336..c8e6060ab52 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchGroupException; import com.datastrato.gravitino.exceptions.NoSuchUserException; -import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; @@ -22,8 +21,8 @@ import java.io.IOException; import java.time.Instant; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,11 +41,14 @@ class PermissionManager { this.roleManager = roleManager; } - boolean grantRoleToUser(String metalake, String role, String user) { + User grantRolesToUser(String metalake, List roles, String user) { try { - RoleEntity roleEntity = roleManager.getRole(metalake, role); + List roleEntitiesToGrant = Lists.newArrayList(); + for (String role : roles) { + roleEntitiesToGrant.add(roleManager.getRole(metalake, role)); + } - store.update( + return store.update( AuthorizationUtils.ofUser(metalake, user), UserEntity.class, Entity.EntityType.USER, @@ -57,13 +59,19 @@ boolean grantRoleToUser(String metalake, String role, String user) { List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); - if (roleNames.contains(roleEntity.name())) { - throw new RoleAlreadyExistsException( - "Role %s already exists in the user %s of the metalake %s", role, user, metalake); + for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { + if (roleNames.contains(roleEntityToGrant.name())) { + LOG.warn( + "Failed to grant, role {} already exists in the user {} of metalake {}", + roleEntityToGrant.name(), + user, + metalake); + } else { + roleNames.add(roleEntityToGrant.name()); + roleIds.add(roleEntityToGrant.id()); + } } - roleNames.add(roleEntity.name()); - roleIds.add(roleEntity.id()); AuditInfo auditInfo = AuditInfo.builder() .withCreator(userEntity.auditInfo().creator()) @@ -81,14 +89,13 @@ boolean grantRoleToUser(String metalake, String role, String user) { .withAuditInfo(auditInfo) .build(); }); - return true; } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, user {} does not exist in the metalake {}", user, metalake, nse); throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); } catch (IOException ioe) { LOG.error( "Failed to grant role {} to user {} in the metalake {} due to storage issues", - role, + StringUtils.join(roles, ","), user, metalake, ioe); @@ -96,11 +103,14 @@ boolean grantRoleToUser(String metalake, String role, String user) { } } - boolean grantRoleToGroup(String metalake, String role, String group) { + Group grantRolesToGroup(String metalake, List roles, String group) { try { - RoleEntity roleEntity = roleManager.getRole(metalake, role); + List roleEntitiesToGrant = Lists.newArrayList(); + for (String role : roles) { + roleEntitiesToGrant.add(roleManager.getRole(metalake, role)); + } - store.update( + return store.update( AuthorizationUtils.ofGroup(metalake, group), GroupEntity.class, Entity.EntityType.GROUP, @@ -110,10 +120,17 @@ boolean grantRoleToGroup(String metalake, String role, String group) { List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); - if (roleNames.contains(roleEntity.name())) { - throw new RoleAlreadyExistsException( - "Role %s already exists in the group %s of the metalake %s", - role, group, metalake); + for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { + if (roleNames.contains(roleEntityToGrant.name())) { + LOG.warn( + "Failed to grant, role {} already exists in the group {} of metalake {}", + roleEntityToGrant.name(), + group, + metalake); + } else { + roleNames.add(roleEntityToGrant.name()); + roleIds.add(roleEntityToGrant.id()); + } } AuditInfo auditInfo = @@ -124,8 +141,6 @@ boolean grantRoleToGroup(String metalake, String role, String group) { .withLastModifiedTime(Instant.now()) .build(); - roleNames.add(roleEntity.name()); - roleIds.add(roleEntity.id()); return GroupEntity.builder() .withId(groupEntity.id()) .withNamespace(groupEntity.namespace()) @@ -135,14 +150,13 @@ boolean grantRoleToGroup(String metalake, String role, String group) { .withAuditInfo(auditInfo) .build(); }); - return true; } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, group {} does not exist in the metalake {}", group, metalake, nse); throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); } catch (IOException ioe) { LOG.error( "Failed to grant role {} to group {} in the metalake {} due to storage issues", - role, + StringUtils.join(roles, ","), group, metalake, ioe); @@ -150,13 +164,14 @@ boolean grantRoleToGroup(String metalake, String role, String group) { } } - boolean revokeRoleFromGroup(String metalake, String role, String group) { + Group revokeRolesFromGroup(String metalake, List roles, String group) { try { - RoleEntity roleEntity = roleManager.getRole(metalake, role); + List roleEntitiesToRevoke = Lists.newArrayList(); + for (String role : roles) { + roleEntitiesToRevoke.add(roleManager.getRole(metalake, role)); + } - AtomicBoolean removed = new AtomicBoolean(true); - - store.update( + return store.update( AuthorizationUtils.ofGroup(metalake, group), GroupEntity.class, Entity.EntityType.GROUP, @@ -165,15 +180,17 @@ boolean revokeRoleFromGroup(String metalake, String role, String group) { roleManager.getValidRoles(metalake, groupEntity.roleNames(), groupEntity.roleIds()); List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); - roleNames.remove(roleEntity.name()); - removed.set(roleIds.remove(roleEntity.id())); - if (!removed.get()) { - LOG.warn( - "Failed to revoke, role {} does not exist in the group {} of metalake {}", - role, - group, - metalake); + for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { + roleNames.remove(roleEntityToRevoke.name()); + boolean removed = roleIds.remove(roleEntityToRevoke.id()); + if (!removed) { + LOG.warn( + "Failed to revoke, role {} does not exist in the group {} of metalake {}", + roleEntityToRevoke.name(), + group, + metalake); + } } AuditInfo auditInfo = @@ -194,7 +211,6 @@ boolean revokeRoleFromGroup(String metalake, String role, String group) { .build(); }); - return removed.get(); } catch (NoSuchEntityException nse) { LOG.warn( "Failed to revoke, group {} does not exist in the metalake {}", group, metalake, nse); @@ -202,7 +218,7 @@ boolean revokeRoleFromGroup(String metalake, String role, String group) { } catch (IOException ioe) { LOG.error( "Failed to revoke role {} from group {} in the metalake {} due to storage issues", - role, + StringUtils.join(roles, ","), group, metalake, ioe); @@ -210,12 +226,14 @@ boolean revokeRoleFromGroup(String metalake, String role, String group) { } } - boolean revokeRoleFromUser(String metalake, String role, String user) { + User revokeRolesFromUser(String metalake, List roles, String user) { try { - RoleEntity roleEntity = roleManager.getRole(metalake, role); - AtomicBoolean removed = new AtomicBoolean(true); + List roleEntitiesToRevoke = Lists.newArrayList(); + for (String role : roles) { + roleEntitiesToRevoke.add(roleManager.getRole(metalake, role)); + } - store.update( + return store.update( AuthorizationUtils.ofUser(metalake, user), UserEntity.class, Entity.EntityType.USER, @@ -226,14 +244,16 @@ boolean revokeRoleFromUser(String metalake, String role, String user) { List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); - roleNames.remove(roleEntity.name()); - removed.set(roleIds.remove(roleEntity.id())); - if (!removed.get()) { - LOG.warn( - "Failed to revoke, role {} doesn't exist in the user {} of metalake {}", - role, - user, - metalake); + for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { + roleNames.remove(roleEntityToRevoke.name()); + boolean removed = roleIds.remove(roleEntityToRevoke.id()); + if (!removed) { + LOG.warn( + "Failed to revoke, role {} doesn't exist in the user {} of metalake {}", + roleEntityToRevoke.name(), + user, + metalake); + } } AuditInfo auditInfo = @@ -252,14 +272,13 @@ boolean revokeRoleFromUser(String metalake, String role, String user) { .withAuditInfo(auditInfo) .build(); }); - return removed.get(); } catch (NoSuchEntityException nse) { LOG.warn("Failed to revoke, user {} does not exist in the metalake {}", user, metalake, nse); throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); } catch (IOException ioe) { LOG.error( "Failed to revoke role {} from user {} in the metalake {} due to storage issues", - role, + StringUtils.join(roles, ","), user, metalake, ioe); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java index cf38825fe3f..4d1080649a7 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -13,7 +13,6 @@ import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.NoSuchUserException; -import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.GroupEntity; @@ -26,6 +25,7 @@ import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -46,7 +46,7 @@ public class TestAccessControlManagerForPermissions { private static String GROUP = "group"; - private static String ROLE = "role"; + private static List ROLE = Lists.newArrayList("role"); private static AuditInfo auditInfo = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); @@ -82,7 +82,7 @@ public class TestAccessControlManagerForPermissions { .withNamespace( Namespace.of(METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) .withId(1L) - .withName(ROLE) + .withName("role") .withProperties(Maps.newHashMap()) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withSecurableObject(SecurableObjects.of(CATALOG)) @@ -123,59 +123,66 @@ public void testGrantRoleToUser() { User user = accessControlManager.getUser(METALAKE, USER); Assertions.assertTrue(user.roles().isEmpty()); - Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); + user = accessControlManager.grantRolesToUser(METALAKE, ROLE, USER); + Assertions.assertFalse(user.roles().isEmpty()); + user = accessControlManager.getUser(METALAKE, USER); Assertions.assertEquals(1, user.roles().size()); - Assertions.assertEquals(ROLE, user.roles().get(0)); + Assertions.assertEquals(ROLE, user.roles()); - // Throw RoleAlreadyExistsException - Assertions.assertThrows( - RoleAlreadyExistsException.class, - () -> accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); + // Test with a role which exists + user = accessControlManager.grantRolesToUser(METALAKE, ROLE, USER); + Assertions.assertEquals(1, user.roles().size()); // Throw NoSuchMetalakeException Assertions.assertThrows( NoSuchMetalakeException.class, - () -> accessControlManager.grantRoleToUser(notExist, ROLE, USER)); + () -> accessControlManager.grantRolesToUser(notExist, ROLE, USER)); // Throw NoSuchRoleException Assertions.assertThrows( NoSuchRoleException.class, - () -> accessControlManager.grantRoleToUser(METALAKE, notExist, USER)); + () -> accessControlManager.grantRolesToUser(METALAKE, Lists.newArrayList(notExist), USER)); // Throw NoSuchUserException Assertions.assertThrows( NoSuchUserException.class, - () -> accessControlManager.grantRoleToUser(METALAKE, ROLE, notExist)); + () -> accessControlManager.grantRolesToUser(METALAKE, Lists.newArrayList(ROLE), notExist)); // Clear Resource - Assertions.assertTrue(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + user = accessControlManager.revokeRolesFromUser(METALAKE, Lists.newArrayList(ROLE), USER); + Assertions.assertTrue(user.roles().isEmpty()); } @Test public void testRevokeRoleFromUser() { String notExist = "not-exist"; - Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, ROLE, USER)); - Assertions.assertTrue(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + User user = accessControlManager.grantRolesToUser(METALAKE, ROLE, USER); + Assertions.assertFalse(user.roles().isEmpty()); + + user = accessControlManager.revokeRolesFromUser(METALAKE, ROLE, USER); + Assertions.assertTrue(user.roles().isEmpty()); // Throw NoSuchMetalakeException Assertions.assertThrows( NoSuchMetalakeException.class, - () -> accessControlManager.revokeRoleFromUser(notExist, ROLE, USER)); + () -> accessControlManager.revokeRolesFromUser(notExist, ROLE, USER)); // Throw NoSuchRoleException Assertions.assertThrows( NoSuchRoleException.class, - () -> accessControlManager.revokeRoleFromUser(METALAKE, notExist, USER)); + () -> + accessControlManager.revokeRolesFromUser(METALAKE, Lists.newArrayList(notExist), USER)); // Remove role which doesn't exist. - Assertions.assertFalse(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + user = accessControlManager.revokeRolesFromUser(METALAKE, ROLE, USER); + Assertions.assertTrue(user.roles().isEmpty()); // Throw NoSuchUserException Assertions.assertThrows( NoSuchUserException.class, - () -> accessControlManager.revokeRoleFromUser(METALAKE, ROLE, notExist)); + () -> accessControlManager.revokeRolesFromUser(METALAKE, ROLE, notExist)); } @Test @@ -185,60 +192,68 @@ public void testGrantRoleToGroup() { Group group = accessControlManager.getGroup(METALAKE, GROUP); Assertions.assertTrue(group.roles().isEmpty()); - Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); + group = accessControlManager.grantRolesToGroup(METALAKE, ROLE, GROUP); + Assertions.assertFalse(group.roles().isEmpty()); group = accessControlManager.getGroup(METALAKE, GROUP); Assertions.assertEquals(1, group.roles().size()); - Assertions.assertEquals(ROLE, group.roles().get(0)); + Assertions.assertEquals(ROLE, group.roles()); - // Throw RoleAlreadyExistsException - Assertions.assertThrows( - RoleAlreadyExistsException.class, - () -> accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); + // Test with a role which exists + group = accessControlManager.grantRolesToGroup(METALAKE, ROLE, GROUP); + Assertions.assertEquals(1, group.roles().size()); // Throw NoSuchMetalakeException Assertions.assertThrows( NoSuchMetalakeException.class, - () -> accessControlManager.grantRoleToGroup(notExist, ROLE, GROUP)); + () -> accessControlManager.grantRolesToGroup(notExist, ROLE, GROUP)); // Throw NoSuchRoleException Assertions.assertThrows( NoSuchRoleException.class, - () -> accessControlManager.grantRoleToGroup(METALAKE, notExist, GROUP)); + () -> + accessControlManager.grantRolesToGroup(METALAKE, Lists.newArrayList(notExist), GROUP)); // Throw NoSuchGroupException Assertions.assertThrows( NoSuchGroupException.class, - () -> accessControlManager.grantRoleToGroup(METALAKE, ROLE, notExist)); + () -> accessControlManager.grantRolesToGroup(METALAKE, ROLE, notExist)); // Clear Resource - Assertions.assertTrue(accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, GROUP)); + group = accessControlManager.revokeRolesFromGroup(METALAKE, ROLE, GROUP); + Assertions.assertTrue(group.roles().isEmpty()); } @Test public void testRevokeRoleFormGroup() { String notExist = "not-exist"; - Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, ROLE, GROUP)); - Assertions.assertTrue(accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, GROUP)); + Group group = accessControlManager.grantRolesToGroup(METALAKE, ROLE, GROUP); + Assertions.assertFalse(group.roles().isEmpty()); + + group = accessControlManager.revokeRolesFromGroup(METALAKE, ROLE, GROUP); + Assertions.assertTrue(group.roles().isEmpty()); // Throw NoSuchMetalakeException Assertions.assertThrows( NoSuchMetalakeException.class, - () -> accessControlManager.revokeRoleFromGroup(notExist, ROLE, GROUP)); + () -> accessControlManager.revokeRolesFromGroup(notExist, ROLE, GROUP)); // Throw NoSuchRoleException Assertions.assertThrows( NoSuchRoleException.class, - () -> accessControlManager.revokeRoleFromGroup(METALAKE, notExist, USER)); + () -> + accessControlManager.revokeRolesFromGroup( + METALAKE, Lists.newArrayList(notExist), GROUP)); // Remove not exist role - Assertions.assertFalse(accessControlManager.revokeRoleFromUser(METALAKE, ROLE, USER)); + group = accessControlManager.revokeRolesFromGroup(METALAKE, ROLE, GROUP); + Assertions.assertTrue(group.roles().isEmpty()); // Throw NoSuchGroupException Assertions.assertThrows( NoSuchGroupException.class, - () -> accessControlManager.revokeRoleFromGroup(METALAKE, ROLE, notExist)); + () -> accessControlManager.revokeRolesFromGroup(METALAKE, ROLE, notExist)); } @Test @@ -259,10 +274,17 @@ public void testDropRole() throws IOException { .build(); entityStore.put(roleEntity, true); - Assertions.assertTrue(accessControlManager.grantRoleToUser(METALAKE, anotherRole, USER)); - Assertions.assertTrue(accessControlManager.grantRoleToGroup(METALAKE, anotherRole, GROUP)); - accessControlManager.deleteRole(METALAKE, anotherRole); - Group group = accessControlManager.getGroup(METALAKE, GROUP); + + User user = + accessControlManager.grantRolesToUser(METALAKE, Lists.newArrayList(anotherRole), USER); + Assertions.assertFalse(user.roles().isEmpty()); + + Group group = + accessControlManager.grantRolesToGroup(METALAKE, Lists.newArrayList(anotherRole), GROUP); + Assertions.assertFalse(group.roles().isEmpty()); + + Assertions.assertTrue(accessControlManager.deleteRole(METALAKE, anotherRole)); + group = accessControlManager.getGroup(METALAKE, GROUP); Assertions.assertTrue(group.roles().isEmpty()); } } diff --git a/server/build.gradle.kts b/server/build.gradle.kts index bdd557ec5a7..8cfc757cab8 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { implementation(libs.bundles.jetty) implementation(libs.bundles.jersey) implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.jackson.annotations) implementation(libs.jackson.datatype.jdk8) diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java index 130c6f2fa82..c51c666b793 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java @@ -79,6 +79,16 @@ public static Response handleTopicException( return TopicExceptionHandler.INSTANCE.handle(op, topic, schema, e); } + public static Response handleUserPermissionOperationException( + OperationType op, String roles, String parent, Exception e) { + return UserPermissionOperationExceptionHandler.INSTANCE.handle(op, roles, parent, e); + } + + public static Response handleGroupPermissionOperationException( + OperationType op, String roles, String parent, Exception e) { + return GroupPermissionOperationExceptionHandler.INSTANCE.handle(op, roles, parent, e); + } + private static class PartitionExceptionHandler extends BaseExceptionHandler { private static final ExceptionHandler INSTANCE = new PartitionExceptionHandler(); @@ -414,6 +424,56 @@ public Response handle(OperationType op, String topic, String schema, Exception } } + private static class UserPermissionOperationExceptionHandler + extends BasePermissionExceptionHandler { + private static final ExceptionHandler INSTANCE = new UserPermissionOperationExceptionHandler(); + + @Override + protected String getPermissionErrorMsg( + String role, String operation, String parent, String reason) { + return String.format( + "Failed to operate role(s)%s operation [%s] under user [%s], reason [%s]", + role, operation, parent, reason); + } + } + + private static class GroupPermissionOperationExceptionHandler + extends BasePermissionExceptionHandler { + + private static final ExceptionHandler INSTANCE = new GroupPermissionOperationExceptionHandler(); + + @Override + protected String getPermissionErrorMsg( + String roles, String operation, String parent, String reason) { + return String.format( + "Failed to operate role(s)%s operation [%s] under group [%s], reason [%s]", + roles, operation, parent, reason); + } + } + + private abstract static class BasePermissionExceptionHandler extends BaseExceptionHandler { + + protected abstract String getPermissionErrorMsg( + String role, String operation, String parent, String reason); + + @Override + public Response handle(OperationType op, String roles, String parent, Exception e) { + String formatted = StringUtil.isBlank(roles) ? "" : " [" + roles + "]"; + String errorMsg = getPermissionErrorMsg(formatted, op.name(), parent, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else { + return super.handle(op, roles, parent, e); + } + } + } + @VisibleForTesting static class BaseExceptionHandler extends ExceptionHandler { diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java index 6daacfc822e..7649f43b33b 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java @@ -14,5 +14,7 @@ public enum OperationType { GET, ADD, REMOVE, - DELETE + DELETE, + GRANT, + REVOKE } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/PermissionOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/PermissionOperations.java new file mode 100644 index 00000000000..95b973cc474 --- /dev/null +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/PermissionOperations.java @@ -0,0 +1,136 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.authorization.AccessControlManager; +import com.datastrato.gravitino.dto.requests.RoleGrantRequest; +import com.datastrato.gravitino.dto.requests.RoleRevokeRequest; +import com.datastrato.gravitino.dto.responses.GroupResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.metrics.MetricNames; +import com.datastrato.gravitino.server.web.Utils; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; + +@Path("/metalakes/{metalake}/permissions") +public class PermissionOperations { + + private final AccessControlManager accessControlManager; + + @Context private HttpServletRequest httpRequest; + + public PermissionOperations() { + // Because accessManager may be null when Gravitino doesn't enable authorization, + // and Jersey injection doesn't support null value. So PermissionOperations chooses to retrieve + // accessControlManager from GravitinoEnv instead of injection here. + this.accessControlManager = GravitinoEnv.getInstance().accessControlManager(); + } + + @PUT + @Path("users/{user}/grant/") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "grant-roles-to-user." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "grant-roles-to-user", absolute = true) + public Response grantRolesToUser( + @PathParam("metalake") String metalake, + @PathParam("user") String user, + RoleGrantRequest request) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new UserResponse( + DTOConverters.toDTO( + accessControlManager.grantRolesToUser( + metalake, request.getRoleNames(), user))))); + } catch (Exception e) { + return ExceptionHandlers.handleUserPermissionOperationException( + OperationType.GRANT, StringUtils.join(request.getRoleNames(), ","), user, e); + } + } + + @PUT + @Path("groups/{group}/grant/") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "grant-roles-to-group." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "grant-roles-to-group", absolute = true) + public Response grantRolesToGroup( + @PathParam("metalake") String metalake, + @PathParam("group") String group, + RoleGrantRequest request) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new GroupResponse( + DTOConverters.toDTO( + accessControlManager.grantRolesToGroup( + metalake, request.getRoleNames(), group))))); + } catch (Exception e) { + return ExceptionHandlers.handleGroupPermissionOperationException( + OperationType.GRANT, StringUtils.join(request.getRoleNames(), ","), group, e); + } + } + + @PUT + @Path("users/{user}/revoke/") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "revoke-roles-from-user." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "revoke-roles-from-user", absolute = true) + public Response revokeRolesFromUser( + @PathParam("metalake") String metalake, + @PathParam("user") String user, + RoleRevokeRequest request) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new UserResponse( + DTOConverters.toDTO( + accessControlManager.revokeRolesFromUser( + metalake, request.getRoleNames(), user))))); + } catch (Exception e) { + return ExceptionHandlers.handleUserPermissionOperationException( + OperationType.REVOKE, StringUtils.join(request.getRoleNames(), ","), user, e); + } + } + + @PUT + @Path("groups/{group}/revoke") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "revoke-roles-from-group." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "revokes-role-from-group", absolute = true) + public Response revokeRolesFromGroup( + @PathParam("metalake") String metalake, + @PathParam("group") String group, + RoleRevokeRequest request) { + try { + return Utils.doAs( + httpRequest, + () -> + Utils.ok( + new GroupResponse( + DTOConverters.toDTO( + accessControlManager.revokeRolesFromGroup( + metalake, request.getRoleNames(), group))))); + } catch (Exception e) { + return ExceptionHandlers.handleGroupPermissionOperationException( + OperationType.REVOKE, StringUtils.join(request.getRoleNames()), group, e); + } + } +} diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPermissionOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPermissionOperations.java new file mode 100644 index 00000000000..9e8ee453b7c --- /dev/null +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestPermissionOperations.java @@ -0,0 +1,382 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import static com.datastrato.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL; +import static com.datastrato.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; +import static com.datastrato.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.authorization.AccessControlManager; +import com.datastrato.gravitino.authorization.Group; +import com.datastrato.gravitino.authorization.User; +import com.datastrato.gravitino.dto.requests.RoleGrantRequest; +import com.datastrato.gravitino.dto.requests.RoleRevokeRequest; +import com.datastrato.gravitino.dto.responses.ErrorConstants; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.GroupResponse; +import com.datastrato.gravitino.dto.responses.UserResponse; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchRoleException; +import com.datastrato.gravitino.exceptions.NoSuchUserException; +import com.datastrato.gravitino.lock.LockManager; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.UserEntity; +import com.datastrato.gravitino.rest.RESTUtils; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestPermissionOperations extends JerseyTest { + + private static final AccessControlManager manager = mock(AccessControlManager.class); + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + @BeforeAll + public static void setup() { + Config config = mock(Config.class); + Mockito.doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY); + Mockito.doReturn(1000L).when(config).get(TREE_LOCK_MIN_NODE_IN_MEMORY); + Mockito.doReturn(36000L).when(config).get(TREE_LOCK_CLEAN_INTERVAL); + GravitinoEnv.getInstance().setLockManager(new LockManager(config)); + GravitinoEnv.getInstance().setAccessControlManager(manager); + } + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(PermissionOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testGrantRolesToUser() { + UserEntity userEntity = + UserEntity.builder() + .withId(1L) + .withName("user") + .withRoleNames(Lists.newArrayList("roles")) + .withRoleIds(Lists.newArrayList(1L)) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + when(manager.grantRolesToUser(any(), any(), any())).thenReturn(userEntity); + + RoleGrantRequest request = new RoleGrantRequest(Lists.newArrayList("role1")); + + Response resp = + target("/metalakes/metalake1/permissions/users/user/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + UserResponse userResponse = resp.readEntity(UserResponse.class); + Assertions.assertEquals(0, userResponse.getCode()); + User user = userResponse.getUser(); + Assertions.assertEquals(userEntity.roles(), user.roles()); + Assertions.assertEquals(userEntity.name(), user.name()); + + // Test to throw NoSuchMetalakeException + doThrow(new NoSuchMetalakeException("mock error")) + .when(manager) + .grantRolesToUser(any(), any(), any()); + Response resp1 = + target("/metalakes/metalake1/permissions/users/user/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ErrorResponse errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw NoSuchUserException + doThrow(new NoSuchUserException("mock error")) + .when(manager) + .grantRolesToUser(any(), any(), any()); + resp1 = + target("/metalakes/metalake1/permissions/users/user/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchUserException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw NoSuchRoleException + doThrow(new NoSuchRoleException("mock error")) + .when(manager) + .grantRolesToUser(any(), any(), any()); + resp1 = + target("/metalakes/metalake1/permissions/users/user/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw internal RuntimeException + doThrow(new RuntimeException("mock error")).when(manager).grantRolesToUser(any(), any(), any()); + Response resp3 = + target("/metalakes/metalake1/permissions/users/user/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse2.getType()); + } + + @Test + public void testGrantRolesToGroup() { + GroupEntity groupEntity = + GroupEntity.builder() + .withId(1L) + .withName("group") + .withRoleNames(Lists.newArrayList("roles")) + .withRoleIds(Lists.newArrayList(1L)) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + when(manager.grantRolesToGroup(any(), any(), any())).thenReturn(groupEntity); + + RoleGrantRequest request = new RoleGrantRequest(Lists.newArrayList("role1")); + + Response resp = + target("/metalakes/metalake1/permissions/groups/group/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + GroupResponse grantResponse = resp.readEntity(GroupResponse.class); + Assertions.assertEquals(0, grantResponse.getCode()); + + Group group = grantResponse.getGroup(); + Assertions.assertEquals(groupEntity.roles(), group.roles()); + Assertions.assertEquals(groupEntity.name(), group.name()); + + // Test to throw NoSuchMetalakeException + doThrow(new NoSuchMetalakeException("mock error")) + .when(manager) + .grantRolesToGroup(any(), any(), any()); + Response resp1 = + target("/metalakes/metalake1/permissions/groups/group/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ErrorResponse errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw NoSuchUserException + doThrow(new NoSuchUserException("mock error")) + .when(manager) + .grantRolesToGroup(any(), any(), any()); + resp1 = + target("/metalakes/metalake1/permissions/groups/group/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchUserException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw NoSuchRoleException + doThrow(new NoSuchRoleException("mock error")) + .when(manager) + .grantRolesToGroup(any(), any(), any()); + resp1 = + target("/metalakes/metalake1/permissions/groups/group/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + errorResponse = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw internal RuntimeException + doThrow(new RuntimeException("mock error")) + .when(manager) + .grantRolesToGroup(any(), any(), any()); + Response resp3 = + target("/metalakes/metalake1/permissions/groups/group/grant") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse2.getType()); + } + + @Test + public void testRevokeRolesFromUser() { + UserEntity userEntity = + UserEntity.builder() + .withId(1L) + .withName("user") + .withRoleNames(Lists.newArrayList()) + .withRoleIds(Lists.newArrayList(1L)) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + when(manager.revokeRolesFromUser(any(), any(), any())).thenReturn(userEntity); + RoleRevokeRequest request = new RoleRevokeRequest(Lists.newArrayList("role1")); + + Response resp = + target("/metalakes/metalake1/permissions/users/user1/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + UserResponse revokeResponse = resp.readEntity(UserResponse.class); + Assertions.assertEquals(0, revokeResponse.getCode()); + + User user = revokeResponse.getUser(); + Assertions.assertEquals(userEntity.roles(), user.roles()); + Assertions.assertEquals(userEntity.name(), user.name()); + + doThrow(new RuntimeException("mock error")) + .when(manager) + .revokeRolesFromUser(any(), any(), any()); + Response resp3 = + target("/metalakes/metalake1/permissions/users/user1/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + } + + @Test + public void testRevokeRolesFromGroup() { + GroupEntity groupEntity = + GroupEntity.builder() + .withId(1L) + .withName("group") + .withRoleNames(Lists.newArrayList()) + .withRoleIds(Lists.newArrayList(1L)) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + when(manager.revokeRolesFromGroup(any(), any(), any())).thenReturn(groupEntity); + RoleRevokeRequest request = new RoleRevokeRequest(Lists.newArrayList("role1")); + + Response resp = + target("/metalakes/metalake1/permissions/groups/group1/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + GroupResponse revokeResponse = resp.readEntity(GroupResponse.class); + Assertions.assertEquals(0, revokeResponse.getCode()); + + Group group = revokeResponse.getGroup(); + Assertions.assertEquals(groupEntity.roles(), group.roles()); + Assertions.assertEquals(groupEntity.name(), group.name()); + + doThrow(new RuntimeException("mock error")) + .when(manager) + .revokeRolesFromGroup(any(), any(), any()); + Response resp3 = + target("/metalakes/metalake1/permissions/groups/group1/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResponse = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + } +} From 6aa16679105843f28df0bdec5a64b6d257e9959e Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Fri, 19 Apr 2024 21:53:39 +0800 Subject: [PATCH 073/106] [#2896] improve(IT): Add e2e test case for verifying fileset schema level and kafka topic level (#2937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …a topic level ### What changes were proposed in this pull request? Improvement of fileset and kafka test case ### Why are the changes needed? Fix: #2896 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? image --- .../test/container/ContainerSuite.java | 3 +- integration-test/build.gradle.kts | 1 + .../test/web/ui/CatalogsPageKafkaTest.java | 222 ++++++++++++ .../test/web/ui/CatalogsPageTest.java | 318 +++++++++++++----- .../test/web/ui/pages/CatalogsPage.java | 48 ++- integration-test/src/test/resources/run | 45 +++ .../tabsContent/detailsView/DetailsView.js | 22 +- 7 files changed, 569 insertions(+), 90 deletions(-) create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java create mode 100755 integration-test/src/test/resources/run diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index c2bd66a9ede..0eeb4962a29 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -154,7 +154,8 @@ public void startKafkaContainer() { if (kafkaContainer == null) { synchronized (ContainerSuite.class) { if (kafkaContainer == null) { - KafkaContainer container = closer.register(KafkaContainer.builder().build()); + KafkaContainer.Builder builder = KafkaContainer.builder().withNetwork(network); + KafkaContainer container = closer.register(builder.build()); try { container.start(); } catch (Exception e) { diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 0bcaf0cbf0c..c9b68e75a8d 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -155,6 +155,7 @@ tasks.test { // Gravitino CI Docker image environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.5") + environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0") copy { from("${project.rootDir}/dev/docker/trino/conf") diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java new file mode 100644 index 00000000000..59badd669d2 --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.web.ui; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.web.ui.pages.CatalogsPage; +import com.datastrato.gravitino.integration.test.web.ui.pages.MetalakePage; +import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +@Tag("gravitino-docker-it") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class CatalogsPageKafkaTest extends AbstractWebIT { + MetalakePage metalakePage = new MetalakePage(); + CatalogsPage catalogsPage = new CatalogsPage(); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static GravitinoAdminClient gravitinoClient; + private static GravitinoMetalake metalake; + + protected static String gravitinoUri = "http://127.0.0.1:8090"; + protected static String kafkaUri = "http://127.0.0.1:9092"; + + private static final String CATALOG_TABLE_TITLE = "Schemas"; + private static final String SCHEMA_TOPIC_TITLE = "Topics"; + private static final String METALAKE_NAME = "test"; + private static final String CATALOG_TYPE_MESSAGING = "messaging"; + private static final String HIVE_CATALOG_NAME = "catalog_hive"; + private static final String MODIFIED_HIVE_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; + private static final String ICEBERG_CATALOG_NAME = "catalog_iceberg"; + private static final String FILESET_CATALOG_NAME = "catalog_fileset"; + private static final String KAFKA_CATALOG_NAME = "catalog_kafka"; + private static final String SCHEMA_NAME = "default"; + private static final String TOPIC_NAME = "topic1"; + + private static final String MYSQL_CATALOG_NAME = "catalog_mysql"; + + private static final String PG_CATALOG_NAME = "catalog_pg"; + + @BeforeAll + public static void before() throws Exception { + gravitinoClient = AbstractIT.getGravitinoClient(); + + gravitinoUri = String.format("http://127.0.0.1:%d", AbstractIT.getGravitinoServerPort()); + + containerSuite.startKafkaContainer(); + + String address = containerSuite.getKafkaContainer().getContainerIpAddress(); + kafkaUri = String.format("%s:%s", address, "9092"); + } + + /** + * Creates a Kafka topic within the specified Metalake, Catalog, Schema, and Topic names. + * + * @param metalakeName The name of the Metalake. + * @param catalogName The name of the Catalog. + * @param schemaName The name of the Schema. + * @param topicName The name of the Kafka topic. + */ + void createTopic(String metalakeName, String catalogName, String schemaName, String topicName) { + Catalog catalog_kafka = + metalake.loadCatalog(NameIdentifier.ofCatalog(metalakeName, catalogName)); + catalog_kafka + .asTopicCatalog() + .createTopic( + NameIdentifier.of(metalakeName, catalogName, schemaName, topicName), + "comment", + null, + Collections.emptyMap()); + } + + /** + * Drops a Kafka topic from the specified Metalake, Catalog, and Schema. + * + * @param metalakeName The name of the Metalake where the topic resides. + * @param catalogName The name of the Catalog that contains the topic. + * @param schemaName The name of the Schema under which the topic exists. + * @param topicName The name of the Kafka topic to be dropped. + */ + void dropTopic(String metalakeName, String catalogName, String schemaName, String topicName) { + Catalog catalog_kafka = + metalake.loadCatalog(NameIdentifier.ofCatalog(metalakeName, catalogName)); + catalog_kafka + .asTopicCatalog() + .dropTopic(NameIdentifier.of(metalakeName, catalogName, schemaName, topicName)); + } + + @Test + @Order(0) + public void testCreateKafkaCatalog() throws InterruptedException { + // create metalake + clickAndWait(metalakePage.createMetalakeBtn); + metalakePage.setMetalakeNameField(METALAKE_NAME); + clickAndWait(metalakePage.submitHandleMetalakeBtn); + // load metalake + metalake = gravitinoClient.loadMetalake(NameIdentifier.of(METALAKE_NAME)); + metalakePage.clickMetalakeLink(METALAKE_NAME); + // create kafka catalog actions + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(KAFKA_CATALOG_NAME); + clickAndWait(catalogsPage.catalogTypeSelector); + catalogsPage.clickSelectType("messaging"); + catalogsPage.setCatalogCommentField("kafka catalog comment"); + // set kafka catalog props + catalogsPage.setCatalogFixedProp("bootstrap.servers", kafkaUri); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(KAFKA_CATALOG_NAME)); + } + + @Test + @Order(1) + public void testKafkaSchemaTreeNode() throws InterruptedException { + // click kafka catalog tree node + String kafkaCatalogNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, KAFKA_CATALOG_NAME, CATALOG_TYPE_MESSAGING); + catalogsPage.clickTreeNode(kafkaCatalogNode); + // verify show table title、 schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); + List treeNodes = + Arrays.asList( + MODIFIED_HIVE_CATALOG_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME, + SCHEMA_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + } + + @Test + @Order(2) + public void testKafkaTopicTreeNode() throws InterruptedException { + // 1. create topic of kafka catalog + createTopic(METALAKE_NAME, KAFKA_CATALOG_NAME, SCHEMA_NAME, TOPIC_NAME); + // 2. click schema tree node + String kafkaSchemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, KAFKA_CATALOG_NAME, CATALOG_TYPE_MESSAGING, SCHEMA_NAME); + catalogsPage.clickTreeNode(kafkaSchemaNode); + // 3. verify show table title、 default schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TOPIC_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TOPIC_NAME, false)); + List treeNodes = + Arrays.asList( + MODIFIED_HIVE_CATALOG_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME, + KAFKA_CATALOG_NAME, + SCHEMA_NAME, + TOPIC_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + } + + @Test + @Order(3) + public void testKafkaTopicDetail() throws InterruptedException { + // 1. click topic tree node + String topicNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, KAFKA_CATALOG_NAME, CATALOG_TYPE_MESSAGING, SCHEMA_NAME, TOPIC_NAME); + catalogsPage.clickTreeNode(topicNode); + // 2. verify show tab details + Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); + // 3. verify show highlight properties + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList( + "key", "partition-count", "partition-count", true)); + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList("value", "partition-count", "1", true)); + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList( + "key", "replication-factor", "replication-factor", true)); + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList("value", "replication-factor", "1", true)); + } + + @Test + @Order(4) + public void testDropKafkaTopic() throws InterruptedException { + // delete topic of kafka catalog + dropTopic(METALAKE_NAME, KAFKA_CATALOG_NAME, SCHEMA_NAME, TOPIC_NAME); + // click schema tree node + String kafkaSchemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, KAFKA_CATALOG_NAME, CATALOG_TYPE_MESSAGING, SCHEMA_NAME); + catalogsPage.clickTreeNode(kafkaSchemaNode); + // verify empty topic list + Assertions.assertTrue(catalogsPage.verifyEmptyTableData()); + } + + @Test + @Order(5) + public void testBackHomePage() throws InterruptedException { + clickAndWait(catalogsPage.backHomeBtn); + Assertions.assertTrue(catalogsPage.verifyBackHomePage()); + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java index b31631030ba..752d85d7074 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.integration.test.container.ContainerSuite; import com.datastrato.gravitino.integration.test.container.TrinoITContainers; import com.datastrato.gravitino.integration.test.util.AbstractIT; @@ -47,26 +48,30 @@ public class CatalogsPageTest extends AbstractWebIT { protected static String hdfsUri = "hdfs://127.0.0.1:9000"; protected static String mysqlUri = "jdbc:mysql://127.0.0.1"; protected static String postgresqlUri = "jdbc:postgresql://127.0.0.1"; - protected static String kafkaUri = "http://127.0.0.1:9092"; private static final String WEB_TITLE = "Gravitino"; private static final String CATALOG_TABLE_TITLE = "Schemas"; private static final String SCHEMA_TABLE_TITLE = "Tables"; + private static final String SCHEMA_FILESET_TITLE = "Filesets"; private static final String TABLE_TABLE_TITLE = "Columns"; private static final String METALAKE_NAME = "test"; private static final String METALAKE_SELECT_NAME = "metalake_select_name"; - private static final String CATALOG_TYPE = "relational"; + private static final String CATALOG_TYPE_RELATIONAL = "relational"; + private static final String CATALOG_TYPE_FILESET = "fileset"; private static final String DEFAULT_CATALOG_NAME = "default_catalog"; private static final String HIVE_CATALOG_NAME = "catalog_hive"; - private static final String MODIFIED_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; + private static final String MODIFIED_HIVE_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; private static final String ICEBERG_CATALOG_NAME = "catalog_iceberg"; private static final String FILESET_CATALOG_NAME = "catalog_fileset"; - private static final String KAFKA_CATALOG_NAME = "catalog_kafka"; private static final String SCHEMA_NAME = "default"; + private static final String SCHEMA_NAME_FILESET = "schema_fileset"; + private static final String FILESET_NAME = "fileset1"; private static final String TABLE_NAME = "table1"; private static final String TABLE_NAME_2 = "table2"; private static final String COLUMN_NAME = "column"; private static final String COLUMN_NAME_2 = "column_2"; + private static final String PROPERTIES_KEY1 = "key1"; + private static final String PROPERTIES_VALUE1 = "val1"; private static final String MYSQL_CATALOG_NAME = "catalog_mysql"; private static final String MYSQL_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; @@ -78,6 +83,8 @@ public class CatalogsPageTest extends AbstractWebIT { private static final String COMMON_JDBC_USER = "trino"; private static final String COMMON_JDBC_PWD = "ds123"; + private static String defaultBaseLocation; + @BeforeAll public static void before() throws Exception { gravitinoClient = AbstractIT.getGravitinoClient(); @@ -94,6 +101,31 @@ public static void before() throws Exception { postgresqlUri = trinoITContainers.getPostgresqlUri(); } + /** + * Create the specified schema + * + * @param metalakeName The name of the Metalake where the schema will be created. + * @param catalogName The name of the Catalog where the schema will be created. + * @param schemaName The name of the Schema where the schema will be created. + */ + void createSchema(String metalakeName, String catalogName, String schemaName) { + Map properties = Maps.newHashMap(); + properties.put(PROPERTIES_KEY1, PROPERTIES_VALUE1); + catalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), "comment", properties); + } + + /** + * Creates a table with a single column in the specified Metalake, Catalog, Schema, and Table. + * + * @param metalakeName The name of the Metalake where the table will be created. + * @param catalogName The name of the Catalog where the table will be created. + * @param schemaName The name of the Schema where the table will be created. + * @param tableName The name of the Table to be created. + * @param colName The name of the Column to be created in the Table. + */ void createTableAndColumn( String metalakeName, String catalogName, @@ -111,6 +143,56 @@ void createTableAndColumn( properties); } + /** + * Retrieves the default base location for the given schema name. + * + * @param schemaName The name of the schema. + * @return The default HDFS storage location for the schema. + */ + private static String defaultBaseLocation(String schemaName) { + if (defaultBaseLocation == null) { + defaultBaseLocation = + String.format("%s/user/hadoop/%s.db", hdfsUri, schemaName.toLowerCase()); + } + return defaultBaseLocation; + } + + /** + * Retrieves the storage location for the given schema name and fileset name. + * + * @param schemaName The name of the schema. + * @param filesetName The name of the fileset. + * @return The storage path for the combination of schema and fileset. + */ + private static String storageLocation(String schemaName, String filesetName) { + return defaultBaseLocation(schemaName) + "/" + filesetName; + } + + /** + * Creates a fileset within the specified Metalake, Catalog, Schema, and Fileset names. + * + * @param metalakeName The name of the Metalake. + * @param catalogName The name of the Catalog. + * @param schemaName The name of the Schema. + * @param filesetName The name of the Fileset. + */ + void createFileset( + String metalakeName, String catalogName, String schemaName, String filesetName) { + Map properties = Maps.newHashMap(); + properties.put(PROPERTIES_KEY1, PROPERTIES_VALUE1); + String storageLocation = storageLocation(schemaName, filesetName); + Catalog catalog_fileset = + metalake.loadCatalog(NameIdentifier.ofCatalog(metalakeName, catalogName)); + catalog_fileset + .asFilesetCatalog() + .createFileset( + NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), + "comment", + Fileset.Type.MANAGED, + storageLocation, + properties); + } + @AfterAll public static void after() { try { @@ -142,7 +224,7 @@ public void testDeleteCatalog() throws InterruptedException { // delete catalog catalogsPage.clickDeleteCatalogBtn(DEFAULT_CATALOG_NAME); clickAndWait(catalogsPage.confirmDeleteBtn); - Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); + Assertions.assertTrue(catalogsPage.verifyEmptyTableData()); } @Test @@ -235,20 +317,6 @@ public void testCreateFilesetCatalog() throws InterruptedException { @Test @Order(6) - public void testCreateKafkaCatalog() throws InterruptedException { - clickAndWait(catalogsPage.createCatalogBtn); - catalogsPage.setCatalogNameField(KAFKA_CATALOG_NAME); - clickAndWait(catalogsPage.catalogTypeSelector); - catalogsPage.clickSelectType("messaging"); - catalogsPage.setCatalogCommentField("kafka catalog comment"); - // set kafka catalog props - catalogsPage.setCatalogFixedProp("bootstrap.servers", kafkaUri); - clickAndWait(catalogsPage.handleSubmitCatalogBtn); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(KAFKA_CATALOG_NAME)); - } - - @Test - @Order(7) public void testRefreshPage() { driver.navigate().refresh(); Assertions.assertEquals(WEB_TITLE, driver.getTitle()); @@ -259,13 +327,12 @@ public void testRefreshPage() { ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME); + FILESET_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyCreatedCatalogs(catalogsNames)); } @Test - @Order(8) + @Order(7) public void testViewTabMetalakeDetails() throws InterruptedException { clickAndWait(catalogsPage.tabDetailsBtn); Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); @@ -274,7 +341,7 @@ public void testViewTabMetalakeDetails() throws InterruptedException { } @Test - @Order(9) + @Order(8) public void testViewCatalogDetails() throws InterruptedException { catalogsPage.clickViewCatalogBtn(HIVE_CATALOG_NAME); Assertions.assertTrue( @@ -282,26 +349,27 @@ public void testViewCatalogDetails() throws InterruptedException { } @Test - @Order(10) - public void testEditCatalog() throws InterruptedException { + @Order(9) + public void testEditHiveCatalog() throws InterruptedException { catalogsPage.clickEditCatalogBtn(HIVE_CATALOG_NAME); - catalogsPage.setCatalogNameField(MODIFIED_CATALOG_NAME); + catalogsPage.setCatalogNameField(MODIFIED_HIVE_CATALOG_NAME); clickAndWait(catalogsPage.handleSubmitCatalogBtn); - Assertions.assertTrue(catalogsPage.verifyEditedCatalog(MODIFIED_CATALOG_NAME)); + Assertions.assertTrue(catalogsPage.verifyEditedCatalog(MODIFIED_HIVE_CATALOG_NAME)); } // test catalog show schema list @Test - @Order(11) + @Order(10) public void testClickCatalogLink() { - catalogsPage.clickCatalogLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickCatalogLink( + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); - Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_HIVE_CATALOG_NAME)); } @Test - @Order(12) + @Order(11) public void testRefreshCatalogPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -309,32 +377,32 @@ public void testRefreshCatalogPage() { Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME, false)); List treeNodes = Arrays.asList( - MODIFIED_CATALOG_NAME, + MODIFIED_HIVE_CATALOG_NAME, SCHEMA_NAME, ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME); + FILESET_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); - Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_HIVE_CATALOG_NAME)); } // test schema show table list @Test - @Order(13) + @Order(12) public void testClickSchemaLink() { // create table createTableAndColumn( - METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, COLUMN_NAME); - catalogsPage.clickSchemaLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, COLUMN_NAME); + catalogsPage.clickSchemaLink( + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); } @Test - @Order(14) + @Order(13) public void testRefreshSchemaPage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -342,24 +410,27 @@ public void testRefreshSchemaPage() { Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); List treeNodes = Arrays.asList( - MODIFIED_CATALOG_NAME, + MODIFIED_HIVE_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME); + FILESET_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); } // test table show column list @Test - @Order(15) + @Order(14) public void testClickTableLink() { catalogsPage.clickTableLink( - METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); + METALAKE_NAME, + MODIFIED_HIVE_CATALOG_NAME, + CATALOG_TYPE_RELATIONAL, + SCHEMA_NAME, + TABLE_NAME); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); @@ -367,7 +438,7 @@ public void testClickTableLink() { } @Test - @Order(16) + @Order(15) public void testRefreshTablePage() { driver.navigate().refresh(); Assertions.assertEquals(driver.getTitle(), WEB_TITLE); @@ -377,62 +448,41 @@ public void testRefreshTablePage() { Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); List treeNodes = Arrays.asList( - MODIFIED_CATALOG_NAME, + MODIFIED_HIVE_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, ICEBERG_CATALOG_NAME, MYSQL_CATALOG_NAME, PG_CATALOG_NAME, - FILESET_CATALOG_NAME, - KAFKA_CATALOG_NAME); + FILESET_CATALOG_NAME); Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); } @Test - @Order(17) - public void testSelectMetalake() throws InterruptedException { - catalogsPage.metalakeSelectChange(METALAKE_SELECT_NAME); - Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); - - catalogsPage.metalakeSelectChange(METALAKE_NAME); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(MODIFIED_CATALOG_NAME)); - } - - @Test - @Order(18) - public void testClickTreeList() throws InterruptedException { - String icebergNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, ICEBERG_CATALOG_NAME, CATALOG_TYPE); - catalogsPage.clickTreeNode(icebergNode); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(ICEBERG_CATALOG_NAME)); - String mysqlNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MYSQL_CATALOG_NAME, CATALOG_TYPE); - catalogsPage.clickTreeNode(mysqlNode); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(MYSQL_CATALOG_NAME)); - String pgNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, PG_CATALOG_NAME, CATALOG_TYPE); - catalogsPage.clickTreeNode(pgNode); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(PG_CATALOG_NAME)); - String filesetNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, FILESET_CATALOG_NAME, "fileset"); - catalogsPage.clickTreeNode(filesetNode); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(FILESET_CATALOG_NAME)); + @Order(16) + public void testRelationalHiveCatalogTreeNode() throws InterruptedException { String hiveNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + String.format( + "{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); catalogsPage.clickTreeNode(hiveNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); - Assertions.assertTrue(catalogsPage.verifyGetCatalog(MODIFIED_CATALOG_NAME)); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MODIFIED_HIVE_CATALOG_NAME)); String schemaNode = String.format( "{{%s}}{{%s}}{{%s}}{{%s}}", - METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME); catalogsPage.clickTreeNode(schemaNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME, false)); String tableNode = String.format( "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", - METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); + METALAKE_NAME, + MODIFIED_HIVE_CATALOG_NAME, + CATALOG_TYPE_RELATIONAL, + SCHEMA_NAME, + TABLE_NAME); catalogsPage.clickTreeNode(tableNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME, true)); @@ -440,30 +490,128 @@ public void testClickTreeList() throws InterruptedException { } @Test - @Order(19) + @Order(17) public void testTreeNodeRefresh() throws InterruptedException { createTableAndColumn( - METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME_2, COLUMN_NAME_2); + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME_2, COLUMN_NAME_2); String hiveNode = - String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + String.format( + "{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); catalogsPage.clickTreeNode(hiveNode); String schemaNode = String.format( "{{%s}}{{%s}}{{%s}}{{%s}}", - METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + METALAKE_NAME, MODIFIED_HIVE_CATALOG_NAME, CATALOG_TYPE_RELATIONAL, SCHEMA_NAME); catalogsPage.clickTreeNodeRefresh(schemaNode); String tableNode = String.format( "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", - METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME_2); + METALAKE_NAME, + MODIFIED_HIVE_CATALOG_NAME, + CATALOG_TYPE_RELATIONAL, + SCHEMA_NAME, + TABLE_NAME_2); catalogsPage.clickTreeNode(tableNode); Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME_2, true)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); } + @Test + @Order(18) + public void testOtherRelationaCatalogTreeNode() throws InterruptedException { + String icebergNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, ICEBERG_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); + catalogsPage.clickTreeNode(icebergNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(ICEBERG_CATALOG_NAME)); + String mysqlNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MYSQL_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); + catalogsPage.clickTreeNode(mysqlNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MYSQL_CATALOG_NAME)); + String pgNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, PG_CATALOG_NAME, CATALOG_TYPE_RELATIONAL); + catalogsPage.clickTreeNode(pgNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(PG_CATALOG_NAME)); + } + + @Test + @Order(19) + public void testSelectMetalake() throws InterruptedException { + catalogsPage.metalakeSelectChange(METALAKE_SELECT_NAME); + Assertions.assertTrue(catalogsPage.verifyEmptyTableData()); + + catalogsPage.metalakeSelectChange(METALAKE_NAME); + driver.navigate().refresh(); + } + @Test @Order(20) + public void testFilesetCatalogTreeNode() throws InterruptedException { + // 1. create schema and fileset of fileset catalog + createSchema(METALAKE_NAME, FILESET_CATALOG_NAME, SCHEMA_NAME_FILESET); + createFileset(METALAKE_NAME, FILESET_CATALOG_NAME, SCHEMA_NAME_FILESET, FILESET_NAME); + // 2. click fileset catalog tree node + String filesetCatalogNode = + String.format( + "{{%s}}{{%s}}{{%s}}", METALAKE_NAME, FILESET_CATALOG_NAME, CATALOG_TYPE_FILESET); + catalogsPage.clickTreeNode(filesetCatalogNode); + // 3. verify show table title、 schema name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME_FILESET, false)); + List treeNodes = + Arrays.asList( + MODIFIED_HIVE_CATALOG_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME, + SCHEMA_NAME_FILESET); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + // 4. click schema tree node + String filesetSchemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, FILESET_CATALOG_NAME, CATALOG_TYPE_FILESET, SCHEMA_NAME_FILESET); + catalogsPage.clickTreeNode(filesetSchemaNode); + // 5. verify show table title、 fileset name and tree node + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_FILESET_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(FILESET_NAME, false)); + treeNodes = + Arrays.asList( + MODIFIED_HIVE_CATALOG_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME, + SCHEMA_NAME_FILESET, + FILESET_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + // 6. click fileset tree node + String filesetNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, + FILESET_CATALOG_NAME, + CATALOG_TYPE_FILESET, + SCHEMA_NAME_FILESET, + FILESET_NAME); + catalogsPage.clickTreeNode(filesetNode); + // 7. verify show tab details + Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList( + "key", PROPERTIES_KEY1, PROPERTIES_KEY1, false)); + Assertions.assertTrue( + catalogsPage.verifyShowPropertiesItemInList( + "value", PROPERTIES_KEY1, PROPERTIES_VALUE1, false)); + } + + @Test + @Order(21) public void testBackHomePage() throws InterruptedException { clickAndWait(catalogsPage.backHomeBtn); Assertions.assertTrue(catalogsPage.verifyBackHomePage()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java index 8f8c1a73a20..a2508fb0a50 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java @@ -201,6 +201,18 @@ public void clickDeleteCatalogBtn(String name) { } } + public void clickMetalakeLink(String metalakeName) { + try { + String xpath = "//a[@href='?metalake=" + metalakeName + "']"; + WebElement link = tableGrid.findElement(By.xpath(xpath)); + WebDriverWait wait = new WebDriverWait(driver, MAX_TIMEOUT); + wait.until(ExpectedConditions.elementToBeClickable(By.xpath(xpath))); + clickAndWait(link); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + public void clickCatalogLink(String metalakeName, String catalogName, String catalogType) { try { String xpath = @@ -415,7 +427,7 @@ public boolean verifyEditedCatalog(String name) { } } - public boolean verifyEmptyCatalog() { + public boolean verifyEmptyTableData() { try { // Check is empty table boolean isNoRows = waitShowText("No rows", tableWrapper); @@ -450,6 +462,40 @@ public boolean verifyShowTableTitle(String title) { } } + /** + * Verifies if a given property item is present in a specified list. + * + * @param item The key or value item of the property. + * @param key The key of the property. + * @param value The value of key item of the property. + * @param isHighlight Whether to highlight the property item or not. + * @return True if the property item is found in the list, false otherwise. + */ + public boolean verifyShowPropertiesItemInList( + String item, String key, String value, Boolean isHighlight) { + try { + Thread.sleep(ACTION_SLEEP_MILLIS); + String xpath; + if (isHighlight) { + xpath = "//div[@data-refer='props-" + item + "-" + key + "-highlight']"; + } else { + xpath = "//div[@data-refer='props-" + item + "-" + key + "']"; + } + WebElement propertyElement = driver.findElement(By.xpath(xpath)); + boolean match = Objects.equals(propertyElement.getText(), value); + + if (!match) { + LOG.error("Prop: does not include itemName: {}", value); + return false; + } + + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return false; + } + } + public boolean verifyShowDataItemInList(String itemName, Boolean isColumnLevel) { try { Thread.sleep(ACTION_SLEEP_MILLIS); diff --git a/integration-test/src/test/resources/run b/integration-test/src/test/resources/run new file mode 100755 index 00000000000..eae50741495 --- /dev/null +++ b/integration-test/src/test/resources/run @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is originally from the official Kafka docker image container `/etc/kafka/docker/run` +# and has been modified to be used to modify the Kafka configuration `advertised.listeners` to use the container's IP address +IP=$(hostname -i) +export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$IP:$DEFAULT_BROKER_PORT" +echo "KAFKA_ADVERTISED_LISTENERS is set to $KAFKA_ADVERTISED_LISTENERS" + + +. /etc/kafka/docker/bash-config + +# Set environment values if they exist as arguments +if [ $# -ne 0 ]; then + echo "===> Overriding env params with args ..." + for var in "$@" + do + export "$var" + done +fi + +echo "===> User" +id + +echo "===> Setting default values of environment variables if not already set." +. /etc/kafka/docker/configureDefaults + +echo "===> Configuring ..." +. /etc/kafka/docker/configure + +echo "===> Launching ... " +. /etc/kafka/docker/launch \ No newline at end of file diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js index 84728a83cf5..eec3f7fcf18 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/detailsView/DetailsView.js @@ -130,7 +130,7 @@ const DetailsView = () => { Properties - + { : 400 }} > - {item.key} +
+ {item.key} +
`${theme.spacing(2.75)} !important` }}> @@ -167,7 +175,15 @@ const DetailsView = () => { : 400 }} > - {item.value} +
+ {item.value} +
From 24fcb1bc363a765567a07936a3164b2fcb5e047d Mon Sep 17 00:00:00 2001 From: mchades Date: Sat, 20 Apr 2024 10:34:23 +0800 Subject: [PATCH 074/106] [#2955] refactor(catalogs): use managedStorage capability (#3022) ### What changes were proposed in this pull request? - remove property `GRAVITINO_MANAGED_ENTITY` - add managedStorage capability for Hadoop and Kafka catalog ### Why are the changes needed? Fix: #2955 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests --- .../catalog/hadoop/HadoopCatalog.java | 6 ++++++ .../hadoop/HadoopCatalogCapability.java | 20 +++++++++++++++++++ .../hadoop/HadoopCatalogOperations.java | 13 ++---------- .../hadoop/TestHadoopCatalogOperations.java | 18 ----------------- .../hive/TestHiveCatalogOperations.java | 2 +- .../gravitino/catalog/kafka/KafkaCatalog.java | 6 ++++++ .../catalog/kafka/KafkaCatalogCapability.java | 20 +++++++++++++++++++ .../catalog/kafka/KafkaCatalogOperations.java | 2 -- .../kafka/TestKafkaCatalogOperations.java | 6 +----- .../catalog/OperationDispatcher.java | 15 +++++--------- .../catalog/SchemaOperationDispatcher.java | 7 ++++--- .../connector/BasePropertiesMetadata.java | 9 --------- .../TestFilesetOperationDispatcher.java | 6 ++---- .../catalog/TestOperationDispatcher.java | 2 -- 14 files changed, 67 insertions(+), 65 deletions(-) create mode 100644 catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java create mode 100644 catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java index 9ee9a3ce505..6864ee874e4 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.file.FilesetCatalog; import com.datastrato.gravitino.rel.SupportsSchemas; import java.util.Map; @@ -28,6 +29,11 @@ protected CatalogOperations newOps(Map config) { return ops; } + @Override + protected Capability newCapability() { + return new HadoopCatalogCapability(); + } + @Override public SupportsSchemas asSchemas() { return (HadoopCatalogOperations) ops(); diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java new file mode 100644 index 00000000000..d063a04fa4f --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java @@ -0,0 +1,20 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.hadoop; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; +import java.util.Objects; + +public class HadoopCatalogCapability implements Capability { + @Override + public CapabilityResult managedStorage(Scope scope) { + if (Objects.requireNonNull(scope) == Scope.SCHEMA) { + return CapabilityResult.SUPPORTED; + } + return CapabilityResult.unsupported( + String.format("Hadoop catalog does not support managed storage for %s.", scope)); + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java index cd87e2d2cc4..7de08bedf6e 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.connector.BasePropertiesMetadata; import com.datastrato.gravitino.connector.CatalogInfo; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.PropertiesMetadata; @@ -36,7 +35,6 @@ import com.datastrato.gravitino.utils.PrincipalUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; @@ -225,7 +223,7 @@ public Fileset createFileset( // managed fileset, Gravitino will get and store the location based on the // catalog/schema's location and store it to the store. .withStorageLocation(filesetPath.toString()) - .withProperties(addManagedFlagToProperties(properties)) + .withProperties(properties) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -372,7 +370,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map addManagedFlagToProperties(Map properties) { - return ImmutableMap.builder() - .putAll(properties) - .put(BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY, Boolean.TRUE.toString()) - .build(); - } - private SchemaEntity updateSchemaEntity( NameIdentifier ident, SchemaEntity schemaEntity, SchemaChange... changes) { Map props = diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 46a370c4c90..bb4b69b752e 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -20,7 +20,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata; import com.datastrato.gravitino.exceptions.NoSuchFilesetException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; @@ -126,11 +125,6 @@ public void testCreateSchemaWithNoLocation() throws IOException { Schema schema = createSchema(name, comment, null, null); Assertions.assertEquals(name, schema.name()); Assertions.assertEquals(comment, schema.comment()); - Map props = schema.properties(); - Assertions.assertTrue( - props.containsKey(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); - Assertions.assertEquals( - "true", props.get(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); Throwable exception = Assertions.assertThrows( @@ -205,10 +199,6 @@ public void testLoadSchema() throws IOException { Assertions.assertEquals(comment, schema1.comment()); Map props = schema1.properties(); - Assertions.assertTrue( - props.containsKey(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); - Assertions.assertEquals( - "true", props.get(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); Assertions.assertTrue(props.containsKey(StringIdentifier.ID_KEY)); Throwable exception = @@ -251,10 +241,6 @@ public void testAlterSchema() throws IOException { Assertions.assertEquals(comment, schema1.comment()); Map props = schema1.properties(); - Assertions.assertTrue( - props.containsKey(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); - Assertions.assertEquals( - "true", props.get(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); Assertions.assertTrue(props.containsKey(StringIdentifier.ID_KEY)); String newKey = "k1"; @@ -301,10 +287,6 @@ public void testDropSchema() throws IOException { Assertions.assertEquals(comment, schema1.comment()); Map props = schema1.properties(); - Assertions.assertTrue( - props.containsKey(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); - Assertions.assertEquals( - "true", props.get(BaseCatalogPropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); Assertions.assertTrue(props.containsKey(StringIdentifier.ID_KEY)); ops.dropSchema(id, false); diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java index fc46c6fd914..dff964a3f31 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java @@ -68,7 +68,7 @@ void testPropertyMeta() { Map> propertyEntryMap = hiveCatalogOperations.catalogPropertiesMetadata().propertyEntries(); - Assertions.assertEquals(12, propertyEntryMap.size()); + Assertions.assertEquals(11, propertyEntryMap.size()); Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS)); Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE)); Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL)); diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java index 5b5515e95c6..46fbc1bf1c7 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.messaging.TopicCatalog; import com.datastrato.gravitino.rel.SupportsSchemas; import java.util.Map; @@ -25,6 +26,11 @@ protected CatalogOperations newOps(Map config) { return ops; } + @Override + protected Capability newCapability() { + return new KafkaCatalogCapability(); + } + @Override public SupportsSchemas asSchemas() throws UnsupportedOperationException { return (KafkaCatalogOperations) ops(); diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java new file mode 100644 index 00000000000..be2141ce331 --- /dev/null +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java @@ -0,0 +1,20 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.kafka; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; +import java.util.Objects; + +public class KafkaCatalogCapability implements Capability { + @Override + public CapabilityResult managedStorage(Scope scope) { + if (Objects.requireNonNull(scope) == Scope.SCHEMA) { + return CapabilityResult.SUPPORTED; + } + return CapabilityResult.unsupported( + String.format("Kafka catalog does not support managed storage for %s.", scope)); + } +} diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java index 3c5c423aed3..e278c3bd463 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java @@ -18,7 +18,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.connector.BasePropertiesMetadata; import com.datastrato.gravitino.connector.CatalogInfo; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.PropertiesMetadata; @@ -552,7 +551,6 @@ private void createDefaultSchemaIfNecessary() { ImmutableMap properties = ImmutableMap.builder() .put(ID_KEY, StringIdentifier.fromId(uid).toString()) - .put(BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY, Boolean.TRUE.toString()) .build(); SchemaEntity defaultSchema = diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index 0dc1e0597d8..975bd851628 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -25,7 +25,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.kafka.embedded.KafkaClusterEmbedded; -import com.datastrato.gravitino.connector.BasePropertiesMetadata; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTopicException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; @@ -189,10 +188,7 @@ public void testLoadSchema() { Assertions.assertEquals(DEFAULT_SCHEMA_NAME, schema.name()); Assertions.assertEquals( "The default schema of Kafka catalog including all topics", schema.comment()); - Assertions.assertEquals(2, schema.properties().size()); - Assertions.assertTrue( - schema.properties().containsKey(BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY)); - Assertions.assertEquals("true", schema.properties().get("gravitino.managed.entity")); + Assertions.assertEquals(1, schema.properties().size()); } @Test diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java index 45d69ec0e3f..276b66dfa80 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java @@ -11,7 +11,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.connector.BasePropertiesMetadata; import com.datastrato.gravitino.connector.HasPropertyMetadata; import com.datastrato.gravitino.connector.PropertiesMetadata; import com.datastrato.gravitino.connector.capability.Capability; @@ -28,7 +27,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -250,14 +248,11 @@ NameIdentifier getCatalogIdentifier(NameIdentifier ident) { return NameIdentifier.of(allElems.get(0), allElems.get(1)); } - boolean isManagedEntity(Map properties) { - return Optional.ofNullable(properties) - .map( - p -> - p.getOrDefault( - BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY, Boolean.FALSE.toString()) - .equals(Boolean.TRUE.toString())) - .orElse(false); + boolean isManagedEntity(NameIdentifier catalogIdent, Capability.Scope scope) { + return doWithCatalog( + catalogIdent, + c -> c.capabilities().managedStorage(scope).supported(), + IllegalArgumentException.class); } static final class FormattedErrorMessages { diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index b13124c3af0..41bca38c374 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -13,6 +13,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; @@ -104,7 +105,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map> BASIC_PROPERTY_ENTRIES; private volatile Map> propertyEntries; @@ -42,11 +38,6 @@ public abstract class BasePropertiesMetadata implements PropertiesMetadata { PropertyEntry.stringReservedPropertyEntry( ID_KEY, "To differentiate the entities created directly by the underlying sources", - true), - PropertyEntry.booleanReservedPropertyEntry( - GRAVITINO_MANAGED_ENTITY, - "Whether the entity is managed by Gravitino's own store or not", - false, true)); BASIC_PROPERTY_ENTRIES = Maps.uniqueIndex(basicPropertyEntries, PropertyEntry::getName); diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java index a2f97136399..068bdeb9ba4 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -5,7 +5,6 @@ package com.datastrato.gravitino.catalog; import static com.datastrato.gravitino.StringIdentifier.ID_KEY; -import static com.datastrato.gravitino.connector.BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -118,11 +117,10 @@ public void testCreateAndAlterFileset() { testProperties(expectedProps, alteredFileset.properties()); // Test immutable fileset properties - FilesetChange[] illegalChange = - new FilesetChange[] {FilesetChange.setProperty(GRAVITINO_MANAGED_ENTITY, "test")}; + FilesetChange[] illegalChange = new FilesetChange[] {FilesetChange.setProperty(ID_KEY, "test")}; testPropertyException( () -> filesetOperationDispatcher.alterFileset(filesetIdent1, illegalChange), - "Property gravitino.managed.entity is immutable or reserved, cannot be set"); + "Property gravitino.identifier is immutable or reserved, cannot be set"); } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestOperationDispatcher.java index f2f57ac1418..a4d80e30fc7 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestOperationDispatcher.java @@ -5,7 +5,6 @@ package com.datastrato.gravitino.catalog; import static com.datastrato.gravitino.TestFilesetPropertiesMetadata.TEST_FILESET_HIDDEN_KEY; -import static com.datastrato.gravitino.connector.BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.reset; @@ -119,7 +118,6 @@ void testProperties(Map expectedProps, Map testP Assertions.assertEquals(v, testProps.get(k)); }); Assertions.assertFalse(testProps.containsKey(StringIdentifier.ID_KEY)); - Assertions.assertFalse(testProps.containsKey(GRAVITINO_MANAGED_ENTITY)); Assertions.assertFalse(testProps.containsKey(TEST_FILESET_HIDDEN_KEY)); } From a92f5aa2a6b45aa9f4268692e46b85d9d67d301c Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 20 Apr 2024 10:43:13 +0800 Subject: [PATCH 075/106] [#2877] test(catalog-jdbc-doris): add Doris catalog integration test (#2936) ### What changes were proposed in this pull request? add Doris catalog integration test ### Why are the changes needed? Fix: #2877 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? IT --- build.gradle.kts | 3 +- catalogs/catalog-jdbc-doris/build.gradle.kts | 3 + .../integration/test/CatalogDorisIT.java | 493 ++++++++++++++++++ 3 files changed, 497 insertions(+), 2 deletions(-) create mode 100644 catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java diff --git a/build.gradle.kts b/build.gradle.kts index 6004c3a60df..08b2dccf8bf 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -646,8 +646,7 @@ tasks { dependsOn( ":catalogs:catalog-hive:copyLibAndConfig", ":catalogs:catalog-lakehouse-iceberg:copyLibAndConfig", - // TODO. Enable packaging the catalog-jdbc-doris module when it is ready for shipping - // ":catalogs:catalog-jdbc-doris:copyLibAndConfig", + ":catalogs:catalog-jdbc-doris:copyLibAndConfig", ":catalogs:catalog-jdbc-mysql:copyLibAndConfig", ":catalogs:catalog-jdbc-postgresql:copyLibAndConfig", ":catalogs:catalog-hadoop:copyLibAndConfig", diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index bd78ec34822..204b5d5532a 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -24,7 +24,10 @@ dependencies { implementation(libs.slf4j.api) testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts")) + testImplementation(project(":clients:client-java")) testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) testImplementation(libs.commons.lang3) testImplementation(libs.guava) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java new file mode 100644 index 00000000000..faa1b990481 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -0,0 +1,493 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.integration.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; +import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.DorisContainer; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.SupportsSchemas; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableCatalog; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.NamedReference; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +@Tag("gravitino-docker-it") +@TestInstance(Lifecycle.PER_CLASS) +public class CatalogDorisIT extends AbstractIT { + + private static final String provider = "jdbc-doris"; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; + + private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; + + public String metalakeName = GravitinoITUtils.genRandomName("doris_it_metalake"); + public String catalogName = GravitinoITUtils.genRandomName("doris_it_catalog"); + public String schemaName = GravitinoITUtils.genRandomName("doris_it_schema"); + public String tableName = GravitinoITUtils.genRandomName("doris_it_table"); + + public String table_comment = "table_comment"; + + // Doris doesn't support schema comment + public String schema_comment = null; + public String DORIS_COL_NAME1 = "doris_col_name1"; + public String DORIS_COL_NAME2 = "doris_col_name2"; + public String DORIS_COL_NAME3 = "doris_col_name3"; + + // Because the creation of Schema Change is an asynchronous process, we need to wait for a while + // For more information, you can refer to the comment in + // DorisTableOperations.generateAlterTableSql(). + private static final long MAX_WAIT_IN_SECONDS = 30; + + private static final long WAIT_INTERVAL_IN_SECONDS = 1; + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private GravitinoMetalake metalake; + + protected Catalog catalog; + + @BeforeAll + public void startup() throws IOException { + + if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } + + containerSuite.startDorisContainer(); + + createMetalake(); + createCatalog(); + createSchema(); + } + + @AfterAll + public void stop() { + clearTableAndSchema(); + AbstractIT.client.dropMetalake(NameIdentifier.of(metalakeName)); + } + + @AfterEach + public void resetSchema() { + clearTableAndSchema(); + createSchema(); + } + + private void clearTableAndSchema() { + NameIdentifier[] nameIdentifiers = + catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName)); + for (NameIdentifier nameIdentifier : nameIdentifiers) { + catalog.asTableCatalog().dropTable(nameIdentifier); + } + catalog.asSchemas().dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), true); + } + + private void createMetalake() { + GravitinoMetalake[] gravitinoMetaLakes = AbstractIT.client.listMetalakes(); + Assertions.assertEquals(0, gravitinoMetaLakes.length); + + GravitinoMetalake createdMetalake = + AbstractIT.client.createMetalake( + NameIdentifier.of(metalakeName), "comment", Collections.emptyMap()); + GravitinoMetalake loadMetalake = + AbstractIT.client.loadMetalake(NameIdentifier.of(metalakeName)); + Assertions.assertEquals(createdMetalake, loadMetalake); + + metalake = loadMetalake; + } + + private void createCatalog() { + Map catalogProperties = Maps.newHashMap(); + + DorisContainer dorisContainer = containerSuite.getDorisContainer(); + + String jdbcUrl = + String.format( + "jdbc:mysql://%s:%d/", + dorisContainer.getContainerIpAddress(), DorisContainer.FE_MYSQL_PORT); + + catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); + catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), DRIVER_CLASS_NAME); + catalogProperties.put(JdbcConfig.USERNAME.getKey(), DorisContainer.USER_NAME); + catalogProperties.put(JdbcConfig.PASSWORD.getKey(), DorisContainer.PASSWORD); + + Catalog createdCatalog = + metalake.createCatalog( + NameIdentifier.of(metalakeName, catalogName), + Catalog.Type.RELATIONAL, + provider, + "doris catalog comment", + catalogProperties); + Catalog loadCatalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); + Assertions.assertEquals(createdCatalog, loadCatalog); + + catalog = loadCatalog; + } + + private void createSchema() { + NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); + String propKey = "key"; + String propValue = "value"; + Map prop = Maps.newHashMap(); + prop.put(propKey, propValue); + + Schema createdSchema = catalog.asSchemas().createSchema(ident, schema_comment, prop); + Schema loadSchema = catalog.asSchemas().loadSchema(ident); + Assertions.assertEquals(createdSchema.name(), loadSchema.name()); + + Assertions.assertEquals(createdSchema.properties().get(propKey), propValue); + } + + private Column[] createColumns() { + Column col1 = Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10), "col_2_comment"); + Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10), "col_3_comment"); + + return new Column[] {col1, col2, col3}; + } + + private Map createTableProperties() { + Map properties = Maps.newHashMap(); + properties.put("replication_allocation", "tag.location.default: 1"); + return properties; + } + + private Distribution createDistribution() { + return Distributions.hash(2, NamedReference.field(DORIS_COL_NAME1)); + } + + @Test + void testDorisSchemaBasicOperation() { + SupportsSchemas schemas = catalog.asSchemas(); + Namespace namespace = Namespace.of(metalakeName, catalogName); + + // test list schemas + NameIdentifier[] nameIdentifiers = schemas.listSchemas(namespace); + Set schemaNames = + Arrays.stream(nameIdentifiers).map(NameIdentifier::name).collect(Collectors.toSet()); + Assertions.assertTrue(schemaNames.contains(schemaName)); + + // test create schema already exists + String testSchemaName = GravitinoITUtils.genRandomName("create_schema_test"); + NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); + schemas.createSchema(schemaIdent, schema_comment, Collections.emptyMap()); + + nameIdentifiers = schemas.listSchemas(Namespace.of(metalakeName, catalogName)); + Map schemaMap = + Arrays.stream(nameIdentifiers).collect(Collectors.toMap(NameIdentifier::name, v -> v)); + Assertions.assertTrue(schemaMap.containsKey(testSchemaName)); + + Assertions.assertThrows( + SchemaAlreadyExistsException.class, + () -> { + schemas.createSchema(schemaIdent, schema_comment, Collections.emptyMap()); + }); + + // test drop schema + Assertions.assertTrue(schemas.dropSchema(schemaIdent, false)); + + // check schema is deleted + // 1. check by load schema + Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(schemaIdent)); + + // 2. check by list schema + nameIdentifiers = schemas.listSchemas(Namespace.of(metalakeName, catalogName)); + schemaMap = + Arrays.stream(nameIdentifiers).collect(Collectors.toMap(NameIdentifier::name, v -> v)); + Assertions.assertFalse(schemaMap.containsKey(testSchemaName)); + + // test drop schema not exists + NameIdentifier notExistsSchemaIdent = NameIdentifier.of(metalakeName, catalogName, "no-exits"); + Assertions.assertFalse(schemas.dropSchema(notExistsSchemaIdent, false)); + } + + @Test + void testDropDorisSchema() { + String schemaName = GravitinoITUtils.genRandomName("doris_it_schema_dropped").toLowerCase(); + + catalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), + "test_comment", + ImmutableMap.of("key", "value")); + + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + createColumns(), + "Created by gravitino client", + createTableProperties(), + Transforms.EMPTY_TRANSFORM, + createDistribution(), + null); + + // Try to drop a database, and cascade equals to false, it should not be allowed. + Assertions.assertFalse( + catalog + .asSchemas() + .dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), false)); + + // Check the database still exists + catalog.asSchemas().loadSchema(NameIdentifier.of(metalakeName, catalogName, schemaName)); + + // Try to drop a database, and cascade equals to true, it should be allowed. + Assertions.assertTrue( + catalog + .asSchemas() + .dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), true)); + + // Check database has been dropped + SupportsSchemas schemas = catalog.asSchemas(); + NameIdentifier of = NameIdentifier.of(metalakeName, catalogName, schemaName); + Assertions.assertThrows( + NoSuchSchemaException.class, + () -> { + schemas.loadSchema(of); + }); + } + + @Test + void testDorisTableBasicOperation() { + // create a table + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + Column[] columns = createColumns(); + + Distribution distribution = createDistribution(); + + Index[] indexes = + new Index[] { + Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}) + }; + + Map properties = createTableProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + distribution, + null, + indexes); + + ITUtils.assertionsTableInfo( + tableName, table_comment, Arrays.asList(columns), properties, indexes, createdTable); + + // load table + Table loadTable = tableCatalog.loadTable(tableIdentifier); + ITUtils.assertionsTableInfo( + tableName, table_comment, Arrays.asList(columns), properties, indexes, loadTable); + + // rename table + String newTableName = GravitinoITUtils.genRandomName("new_table_name"); + tableCatalog.alterTable(tableIdentifier, TableChange.rename(newTableName)); + NameIdentifier newTableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, newTableName); + Table renamedTable = tableCatalog.loadTable(newTableIdentifier); + ITUtils.assertionsTableInfo( + newTableName, table_comment, Arrays.asList(columns), properties, indexes, renamedTable); + } + + @Test + void testAlterDorisTable() { + // create a table + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + Column[] columns = createColumns(); + + Distribution distribution = createDistribution(); + + Index[] indexes = + new Index[] { + Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}) + }; + + Map properties = createTableProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + distribution, + null, + indexes); + + ITUtils.assertionsTableInfo( + tableName, table_comment, Arrays.asList(columns), properties, indexes, createdTable); + + // Alter column type + tableCatalog.alterTable( + tableIdentifier, + TableChange.updateColumnType(new String[] {DORIS_COL_NAME3}, Types.VarCharType.of(255))); + + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + ITUtils.assertColumn( + Column.of(DORIS_COL_NAME3, Types.VarCharType.of(255), "col_3_comment"), + tableCatalog.loadTable(tableIdentifier).columns()[2])); + + // update column comment + // Alter column type + tableCatalog.alterTable( + tableIdentifier, + TableChange.updateColumnComment(new String[] {DORIS_COL_NAME3}, "new_comment")); + + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + ITUtils.assertColumn( + Column.of(DORIS_COL_NAME3, Types.VarCharType.of(255), "new_comment"), + tableCatalog.loadTable(tableIdentifier).columns()[2])); + + // add new column + tableCatalog.alterTable( + tableIdentifier, + TableChange.addColumn( + new String[] {"col_4"}, Types.VarCharType.of(255), "col_4_comment", true)); + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + 4, tableCatalog.loadTable(tableIdentifier).columns().length)); + + ITUtils.assertColumn( + Column.of("col_4", Types.VarCharType.of(255), "col_4_comment"), + tableCatalog.loadTable(tableIdentifier).columns()[3]); + + // change column position + // TODO: change column position is unstable, add it later + + // drop column + tableCatalog.alterTable( + tableIdentifier, TableChange.deleteColumn(new String[] {"col_4"}, true)); + + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + 3, tableCatalog.loadTable(tableIdentifier).columns().length)); + } + + @Test + void testDorisIndex() { + String tableName = GravitinoITUtils.genRandomName("test_add_index"); + + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + Column[] columns = createColumns(); + + Distribution distribution = createDistribution(); + + Map properties = createTableProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + distribution, + null); + Assertions.assertEquals(createdTable.name(), tableName); + + // add index test. + tableCatalog.alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.addIndex( + Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}})); + + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertEquals( + 1, + tableCatalog + .loadTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)) + .index() + .length)); + + // delete index and add new column and index. + tableCatalog.alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.deleteIndex("k1_index", true), + TableChange.addIndex( + Index.IndexType.PRIMARY_KEY, "k2_index", new String[][] {{DORIS_COL_NAME2}})); + + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertEquals( + 1, + tableCatalog + .loadTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)) + .index() + .length)); + } +} From a4c657816f173a6984aeefed0729c38d51e00d11 Mon Sep 17 00:00:00 2001 From: XiaoZ <57973980+xiaozcy@users.noreply.github.com> Date: Sat, 20 Apr 2024 11:57:59 +0800 Subject: [PATCH 076/106] [#3036] fix(catalog-hive): remove call to Steams.peek (#3046) ### What changes were proposed in this pull request? replace call to Steams.peek with Streams.map in HiveTableOperations.java ### Why are the changes needed? Fix: #3036 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? existing unit test Co-authored-by: zhanghan18 Co-authored-by: Qi Yu --- .../gravitino/catalog/hive/HiveTableOperations.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java index 8f2388753f6..97ebbbf909f 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java @@ -306,9 +306,9 @@ private List getFilterPartitionValueList(Table dropTable, String partiti // Split and process the partition specification string Map partSpecMap = Arrays.stream(partitionSpec.split(PARTITION_NAME_DELIMITER)) - .map(part -> part.split(PARTITION_VALUE_DELIMITER, 2)) - .peek( - keyValue -> { + .map( + part -> { + String[] keyValue = part.split(PARTITION_VALUE_DELIMITER, 2); if (keyValue.length != 2) { throw new IllegalArgumentException("Error partition format: " + partitionSpec); } @@ -316,6 +316,7 @@ private List getFilterPartitionValueList(Table dropTable, String partiti throw new NoSuchPartitionException( "Hive partition %s does not exist in Hive Metastore", partitionSpec); } + return keyValue; }) .collect(Collectors.toMap(keyValue -> keyValue[0], keyValue -> keyValue[1])); From 80545466db70686da2116f3440a2759c6dc59396 Mon Sep 17 00:00:00 2001 From: YiJhen Lin <44198696+yijhenlin@users.noreply.github.com> Date: Sat, 20 Apr 2024 12:15:39 +0800 Subject: [PATCH 077/106] [#3056] fix(catalog-hadoop): fix improper character class in regex (#3055) ### What changes were proposed in this pull request? Correct the regex ### Why are the changes needed? "gvfs://fileset/catalog1/schema1/fileset1//" should not pass the matches but it passed. Fix: #3056 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add UT --- .../filesystem/hadoop/GravitinoVirtualFileSystem.java | 2 +- .../gravitino/filesystem/hadoop/TestGvfsBase.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 37d4a90a41b..7372a2a1223 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -57,7 +57,7 @@ public class GravitinoVirtualFileSystem extends FileSystem { // gvfs://fileset/fileset_catalog/fileset_schema/fileset1/file.txt // /fileset_catalog/fileset_schema/fileset1/sub_dir/ private static final Pattern IDENTIFIER_PATTERN = - Pattern.compile("^(?:gvfs://fileset)?/([^/]+)/([^/]+)/([^/]+)(?:[/[^/]+]*)$"); + Pattern.compile("^(?:gvfs://fileset)?/([^/]+)/([^/]+)/([^/]+)(?:/[^/]+)*/?$"); @Override public void initialize(URI name, Configuration configuration) throws IOException { diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java index d190db5fbdc..0a5889caa9b 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java @@ -558,6 +558,12 @@ public void testExtractIdentifier() throws IOException, URISyntaxException { assertThrows( IllegalArgumentException.class, () -> fs.extractIdentifier(new URI("/catalog1/schema1/"))); + assertThrows( + IllegalArgumentException.class, + () -> fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1//"))); + assertThrows( + IllegalArgumentException.class, + () -> fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir//"))); } } } From 8119053caed82c876c261764e16a12a99ef23f72 Mon Sep 17 00:00:00 2001 From: cai can <94670132+caican00@users.noreply.github.com> Date: Sat, 20 Apr 2024 14:29:21 +0800 Subject: [PATCH 078/106] [#2586] feat(spark-connector): Support iceberg partition (#2709) ### What changes were proposed in this pull request? support iceberg partition, such as `bucket(num, column)`,`truncate(width, column)`, `years(column)`, `months(column)`, etc ### Why are the changes needed? Support iceberg partition Fix: https://github.com/datastrato/gravitino/issues/2586 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? New unit tests and ITs. --- .../lakehouse/iceberg/IcebergTable.java | 5 +- .../integration/test/spark/SparkCommonIT.java | 6 +- .../spark/iceberg/SparkIcebergCatalogIT.java | 92 ++++++- .../test/util/spark/SparkTableInfo.java | 34 ++- .../util/spark/SparkTableInfoChecker.java | 74 ++++-- .../lakehouse-iceberg/00000_create_table.txt | 9 +- .../lakehouse-iceberg/00002_alter_table.txt | 34 ++- .../lakehouse-iceberg/00006_datatype.txt | 10 +- .../lakehouse-iceberg/00007_varchar.txt | 5 +- .../connector/SparkTransformConverter.java | 201 ++++++++++++++- .../spark/connector/catalog/BaseCatalog.java | 27 ++- .../connector/hive/GravitinoHiveCatalog.java | 12 +- .../spark/connector/hive/SparkHiveTable.java | 6 +- .../iceberg/GravitinoIcebergCatalog.java | 12 +- .../connector/iceberg/SparkIcebergTable.java | 11 +- .../spark/connector/table/SparkBaseTable.java | 13 +- .../TestSparkTransformConverter.java | 228 ++++++++++++------ 17 files changed, 646 insertions(+), 133 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java index 4c2e3cf9c50..18fcdbfd118 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.catalog.lakehouse.iceberg; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata.DISTRIBUTION_MODE; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata.LOCATION; import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; import com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.ConvertUtil; @@ -22,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import java.util.HashMap; import java.util.Map; import lombok.Getter; import lombok.ToString; @@ -131,7 +133,8 @@ String transformDistribution(Distribution distribution) { * @return A new IcebergTable instance. */ public static IcebergTable fromIcebergTable(TableMetadata table, String tableName) { - Map properties = table.properties(); + Map properties = new HashMap<>(table.properties()); + properties.put(LOCATION, table.location()); Schema schema = table.schema(); Transform[] partitionSpec = FromIcebergPartitionSpec.fromPartitionSpec(table.spec(), schema); SortOrder[] sortOrder = FromIcebergSortOrder.fromSortOrder(table.sortOrder()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkCommonIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkCommonIT.java index eb5013c18f4..9dab1b46839 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkCommonIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkCommonIT.java @@ -641,12 +641,16 @@ void testInsertDatasourceFormatPartitionTableAsSelect() { protected void checkPartitionDirExists(SparkTableInfo table) { Assertions.assertTrue(table.isPartitionTable(), "Not a partition table"); - String tableLocation = table.getTableLocation(); + String tableLocation = getTableLocation(table); String partitionExpression = getPartitionExpression(table, "/").replace("'", ""); Path partitionPath = new Path(tableLocation, partitionExpression); checkDirExists(partitionPath); } + protected String getTableLocation(SparkTableInfo table) { + return table.getTableLocation(); + } + protected void checkDirExists(Path dir) { try { Assertions.assertTrue(hdfs.exists(dir), "HDFS directory not exists," + dir); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java index ed5df8f08ac..27cc184ce6f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java @@ -6,15 +6,22 @@ import com.datastrato.gravitino.integration.test.spark.SparkCommonIT; import com.datastrato.gravitino.integration.test.util.spark.SparkTableInfo; +import com.datastrato.gravitino.integration.test.util.spark.SparkTableInfoChecker; import com.google.common.collect.ImmutableList; +import java.io.File; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.Path; import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException; import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; import org.apache.spark.sql.connector.catalog.CatalogPlugin; import org.apache.spark.sql.connector.catalog.FunctionCatalog; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.functions.UnboundFunction; +import org.apache.spark.sql.types.DataTypes; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -41,7 +48,12 @@ protected boolean supportsSparkSQLClusteredBy() { @Override protected boolean supportsPartition() { - return false; + return true; + } + + @Override + protected String getTableLocation(SparkTableInfo table) { + return String.join(File.separator, table.getTableLocation(), "data"); } @Test @@ -126,4 +138,82 @@ void testIcebergFunction() { Assertions.assertEquals("ab", bucket.get(0)); }); } + + @Test + void testIcebergPartitions() { + Map partitionPaths = new HashMap<>(); + partitionPaths.put("years", "name=a/name_trunc=a/id_bucket=4/ts_year=2024"); + partitionPaths.put("months", "name=a/name_trunc=a/id_bucket=4/ts_month=2024-01"); + partitionPaths.put("days", "name=a/name_trunc=a/id_bucket=4/ts_day=2024-01-01"); + partitionPaths.put("hours", "name=a/name_trunc=a/id_bucket=4/ts_hour=2024-01-01-12"); + + partitionPaths + .keySet() + .forEach( + func -> { + String tableName = String.format("test_iceberg_%s_partition_table", func); + dropTableIfExists(tableName); + String createTableSQL = getCreateIcebergSimpleTableString(tableName); + createTableSQL = + createTableSQL + + String.format( + " PARTITIONED BY (name, truncate(1, name), bucket(16, id), %s(ts));", + func); + sql(createTableSQL); + SparkTableInfo tableInfo = getTableInfo(tableName); + SparkTableInfoChecker checker = + SparkTableInfoChecker.create() + .withName(tableName) + .withColumns(getIcebergSimpleTableColumn()) + .withIdentifyPartition(Collections.singletonList("name")) + .withTruncatePartition(1, "name") + .withBucketPartition(16, Collections.singletonList("id")); + switch (func) { + case "years": + checker.withYearPartition("ts"); + break; + case "months": + checker.withMonthPartition("ts"); + break; + case "days": + checker.withDayPartition("ts"); + break; + case "hours": + checker.withHourPartition("ts"); + break; + default: + throw new IllegalArgumentException("UnSupported partition function: " + func); + } + checker.check(tableInfo); + + String insertData = + String.format( + "INSERT into %s values(2,'a',cast('2024-01-01 12:00:00.0' as timestamp));", + tableName); + sql(insertData); + List queryResult = getTableData(tableName); + Assertions.assertEquals(1, queryResult.size()); + Assertions.assertEquals("2,a,2024-01-01 12:00:00.0", queryResult.get(0)); + String partitionExpression = partitionPaths.get(func); + Path partitionPath = new Path(getTableLocation(tableInfo), partitionExpression); + checkDirExists(partitionPath); + }); + } + + private List getIcebergSimpleTableColumn() { + return Arrays.asList( + SparkTableInfo.SparkColumnInfo.of("id", DataTypes.IntegerType, "id comment"), + SparkTableInfo.SparkColumnInfo.of("name", DataTypes.StringType, ""), + SparkTableInfo.SparkColumnInfo.of("ts", DataTypes.TimestampType, null)); + } + + /** + * Here we build a new `createIcebergSql` String for creating a table with a field of timestamp + * type to create the year/month,etc partitions + */ + private String getCreateIcebergSimpleTableString(String tableName) { + return String.format( + "CREATE TABLE %s (id INT COMMENT 'id comment', name STRING COMMENT '', ts TIMESTAMP)", + tableName); + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfo.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfo.java index 8d32c8ef1ca..09343ed1d0e 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfo.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfo.java @@ -18,10 +18,15 @@ import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.connector.expressions.ApplyTransform; import org.apache.spark.sql.connector.expressions.BucketTransform; +import org.apache.spark.sql.connector.expressions.DaysTransform; +import org.apache.spark.sql.connector.expressions.HoursTransform; import org.apache.spark.sql.connector.expressions.IdentityTransform; +import org.apache.spark.sql.connector.expressions.MonthsTransform; import org.apache.spark.sql.connector.expressions.SortedBucketTransform; import org.apache.spark.sql.connector.expressions.Transform; +import org.apache.spark.sql.connector.expressions.YearsTransform; import org.apache.spark.sql.types.DataType; import org.junit.jupiter.api.Assertions; @@ -69,10 +74,17 @@ void setBucket(Transform bucket) { void addPartition(Transform partition) { if (partition instanceof IdentityTransform) { partitionColumnNames.add(((IdentityTransform) partition).reference().fieldNames()[0]); + this.partitions.add(partition); + } else if (partition instanceof BucketTransform + || partition instanceof HoursTransform + || partition instanceof DaysTransform + || partition instanceof MonthsTransform + || partition instanceof YearsTransform + || (partition instanceof ApplyTransform && "truncate".equalsIgnoreCase(partition.name()))) { + this.partitions.add(partition); } else { throw new NotSupportedException("Doesn't support " + partition.name()); } - this.partitions.add(partition); } static SparkTableInfo create(SparkBaseTable baseTable) { @@ -95,13 +107,25 @@ static SparkTableInfo create(SparkBaseTable baseTable) { .collect(Collectors.toList()); sparkTableInfo.comment = baseTable.properties().remove(ConnectorConstants.COMMENT); sparkTableInfo.tableProperties = baseTable.properties(); + boolean supportsBucketPartition = + baseTable.getSparkTransformConverter().isSupportsBucketPartition(); Arrays.stream(baseTable.partitioning()) .forEach( transform -> { if (transform instanceof BucketTransform || transform instanceof SortedBucketTransform) { - sparkTableInfo.setBucket(transform); - } else if (transform instanceof IdentityTransform) { + if (isBucketPartition(supportsBucketPartition, transform)) { + sparkTableInfo.addPartition(transform); + } else { + sparkTableInfo.setBucket(transform); + } + } else if (transform instanceof IdentityTransform + || transform instanceof HoursTransform + || transform instanceof DaysTransform + || transform instanceof MonthsTransform + || transform instanceof YearsTransform + || (transform instanceof ApplyTransform + && "truncate".equalsIgnoreCase(transform.name()))) { sparkTableInfo.addPartition(transform); } else { throw new NotSupportedException( @@ -111,6 +135,10 @@ static SparkTableInfo create(SparkBaseTable baseTable) { return sparkTableInfo; } + private static boolean isBucketPartition(boolean supportsBucketPartition, Transform transform) { + return supportsBucketPartition && !(transform instanceof SortedBucketTransform); + } + public List getUnPartitionedColumns() { return columns.stream() .filter(column -> !partitionColumnNames.contains(column.name)) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfoChecker.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfoChecker.java index c41ccd23213..65e4c36579a 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfoChecker.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkTableInfoChecker.java @@ -7,9 +7,10 @@ import com.datastrato.gravitino.integration.test.util.spark.SparkTableInfo.SparkColumnInfo; import com.datastrato.gravitino.spark.connector.SparkTransformConverter; -import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.spark.sql.connector.expressions.Expressions; import org.apache.spark.sql.connector.expressions.IdentityTransform; import org.apache.spark.sql.connector.expressions.Transform; @@ -21,7 +22,7 @@ */ public class SparkTableInfoChecker { private SparkTableInfo expectedTableInfo = new SparkTableInfo(); - private List checkFields = new ArrayList<>(); + private Set checkFields = new LinkedHashSet<>(); private SparkTableInfoChecker() {} @@ -50,6 +51,23 @@ public SparkTableInfoChecker withColumns(List columns) { return this; } + public SparkTableInfoChecker withBucket(int bucketNum, List bucketColumns) { + Transform bucketTransform = Expressions.bucket(bucketNum, bucketColumns.toArray(new String[0])); + this.expectedTableInfo.setBucket(bucketTransform); + this.checkFields.add(CheckField.BUCKET); + return this; + } + + public SparkTableInfoChecker withBucket( + int bucketNum, List bucketColumns, List sortColumns) { + Transform sortBucketTransform = + SparkTransformConverter.createSortBucketTransform( + bucketNum, bucketColumns.toArray(new String[0]), sortColumns.toArray(new String[0])); + this.expectedTableInfo.setBucket(sortBucketTransform); + this.checkFields.add(CheckField.BUCKET); + return this; + } + public SparkTableInfoChecker withIdentifyPartition(List partitionColumns) { partitionColumns.forEach( columnName -> { @@ -61,20 +79,47 @@ public SparkTableInfoChecker withIdentifyPartition(List partitionColumns return this; } - public SparkTableInfoChecker withBucket(int bucketNum, List bucketColumns) { + public SparkTableInfoChecker withBucketPartition(int bucketNum, List bucketColumns) { Transform bucketTransform = Expressions.bucket(bucketNum, bucketColumns.toArray(new String[0])); - this.expectedTableInfo.setBucket(bucketTransform); - this.checkFields.add(CheckField.BUCKET); + this.expectedTableInfo.addPartition(bucketTransform); + this.checkFields.add(CheckField.PARTITION); return this; } - public SparkTableInfoChecker withBucket( - int bucketNum, List bucketColumns, List sortColumns) { - Transform sortBucketTransform = - SparkTransformConverter.createSortBucketTransform( - bucketNum, bucketColumns.toArray(new String[0]), sortColumns.toArray(new String[0])); - this.expectedTableInfo.setBucket(sortBucketTransform); - this.checkFields.add(CheckField.BUCKET); + public SparkTableInfoChecker withHourPartition(String partitionColumn) { + Transform hourTransform = Expressions.hours(partitionColumn); + this.expectedTableInfo.addPartition(hourTransform); + this.checkFields.add(CheckField.PARTITION); + return this; + } + + public SparkTableInfoChecker withDayPartition(String partitionColumn) { + Transform dayTransform = Expressions.days(partitionColumn); + this.expectedTableInfo.addPartition(dayTransform); + this.checkFields.add(CheckField.PARTITION); + return this; + } + + public SparkTableInfoChecker withMonthPartition(String partitionColumn) { + Transform monthTransform = Expressions.months(partitionColumn); + this.expectedTableInfo.addPartition(monthTransform); + this.checkFields.add(CheckField.PARTITION); + return this; + } + + public SparkTableInfoChecker withYearPartition(String partitionColumn) { + Transform yearTransform = Expressions.years(partitionColumn); + this.expectedTableInfo.addPartition(yearTransform); + this.checkFields.add(CheckField.PARTITION); + return this; + } + + public SparkTableInfoChecker withTruncatePartition(int width, String partitionColumn) { + Transform truncateTransform = + Expressions.apply( + "truncate", Expressions.literal(width), Expressions.column(partitionColumn)); + this.expectedTableInfo.addPartition(truncateTransform); + this.checkFields.add(CheckField.PARTITION); return this; } @@ -104,8 +149,9 @@ public void check(SparkTableInfo realTableInfo) { expectedTableInfo.getColumns(), realTableInfo.getColumns()); break; case PARTITION: - Assertions.assertEquals( - expectedTableInfo.getPartitions(), realTableInfo.getPartitions()); + Assertions.assertArrayEquals( + expectedTableInfo.getPartitions().toArray(), + realTableInfo.getPartitions().toArray()); break; case BUCKET: Assertions.assertEquals(expectedTableInfo.getBucket(), realTableInfo.getBucket()); diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt index c0ba8a4044b..d7cbbec0bb3 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt @@ -6,7 +6,10 @@ CREATE TABLE name varchar, salary integer ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" CREATE TABLE @@ -16,6 +19,7 @@ CREATE TABLE ) COMMENT '' WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb02', partitioning = ARRAY['name'], sorted_by = ARRAY['salary'] )" @@ -30,6 +34,7 @@ CREATE TABLE ) COMMENT '' WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb03', partitioning = ARRAY['name'], sorted_by = ARRAY['name'] )" @@ -42,6 +47,7 @@ CREATE TABLE ) COMMENT '' WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb04', sorted_by = ARRAY['name'] )" @@ -53,6 +59,7 @@ CREATE TABLE ) COMMENT '' WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb05', partitioning = ARRAY['name'] )" diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt index 969d40b0eb8..db285827f59 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt @@ -16,7 +16,10 @@ DROP COLUMN name varchar, salary integer ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" RENAME COLUMN @@ -24,7 +27,10 @@ RENAME COLUMN s varchar, salary integer ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" SET COLUMN TYPE @@ -32,7 +38,10 @@ SET COLUMN TYPE s varchar, salary bigint ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" COMMENT @@ -40,7 +49,10 @@ COMMENT s varchar, salary bigint ) -COMMENT 'test table comments'" +COMMENT 'test table comments' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" COMMENT @@ -48,7 +60,10 @@ COMMENT s varchar COMMENT 'test column comments', salary bigint ) -COMMENT 'test table comments'" +COMMENT 'test table comments' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" ADD COLUMN @@ -57,10 +72,11 @@ ADD COLUMN salary bigint, city varchar COMMENT 'aaa' ) -COMMENT 'test table comments'" +COMMENT 'test table comments' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" DROP TABLE -DROP SCHEMA - - +DROP SCHEMA \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt index 915539adaf5..ee97371a6b0 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt @@ -18,7 +18,10 @@ CREATE TABLE f14 time(3), f15 timestamp(3) ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb01' +)" INSERT: 1 row @@ -44,7 +47,10 @@ CREATE TABLE f14 time(3) NOT NULL, f15 timestamp(3) NOT NULL ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2.db/tb02' +)" INSERT: 1 row diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt index c7f7ab14e44..a32a0e8604a 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt @@ -14,7 +14,10 @@ CREATE TABLE id integer, name varchar ) -COMMENT ''" +COMMENT '' +WITH ( + location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/varchar_db2.db/tb04' +)" DROP TABLE diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/SparkTransformConverter.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/SparkTransformConverter.java index 9afad670b76..a636699024d 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/SparkTransformConverter.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/SparkTransformConverter.java @@ -13,6 +13,7 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.Arrays; @@ -21,11 +22,19 @@ import lombok.Getter; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.spark.sql.connector.expressions.ApplyTransform; import org.apache.spark.sql.connector.expressions.BucketTransform; +import org.apache.spark.sql.connector.expressions.DaysTransform; import org.apache.spark.sql.connector.expressions.Expressions; +import org.apache.spark.sql.connector.expressions.HoursTransform; import org.apache.spark.sql.connector.expressions.IdentityTransform; +import org.apache.spark.sql.connector.expressions.Literal; import org.apache.spark.sql.connector.expressions.LogicalExpressions; +import org.apache.spark.sql.connector.expressions.MonthsTransform; import org.apache.spark.sql.connector.expressions.SortedBucketTransform; +import org.apache.spark.sql.connector.expressions.YearsTransform; +import org.apache.spark.sql.types.IntegerType; +import org.apache.spark.sql.types.LongType; import scala.collection.JavaConverters; /** @@ -39,6 +48,22 @@ */ public class SparkTransformConverter { + /** + * If supportsBucketPartition is ture, BucketTransform is transfromed to partition, and + * SortedBucketTransform is not supported. If false, BucketTransform and SortedBucketTransform is + * transformed to Distribution and SortOrder. + */ + private final boolean supportsBucketPartition; + + public SparkTransformConverter(boolean supportsBucketPartition) { + this.supportsBucketPartition = supportsBucketPartition; + } + + @VisibleForTesting + public boolean isSupportsBucketPartition() { + return supportsBucketPartition; + } + @Getter public static class DistributionAndSortOrdersInfo { private Distribution distribution; @@ -55,19 +80,48 @@ private void setSortOrders(SortOrder[] sortOrdersInfo) { } } - public static Transform[] toGravitinoPartitionings( + public Transform[] toGravitinoPartitionings( org.apache.spark.sql.connector.expressions.Transform[] transforms) { if (ArrayUtils.isEmpty(transforms)) { return Transforms.EMPTY_TRANSFORM; } return Arrays.stream(transforms) - .filter(transform -> !isBucketTransform(transform)) + .filter(this::isPartitionTransform) .map( transform -> { if (transform instanceof IdentityTransform) { IdentityTransform identityTransform = (IdentityTransform) transform; return Transforms.identity(identityTransform.reference().fieldNames()); + } else if (transform instanceof BucketTransform) { + BucketTransform bucketTransform = (BucketTransform) transform; + int numBuckets = (int) bucketTransform.numBuckets().value(); + String[][] fieldNames = + Arrays.stream(bucketTransform.references()) + .map(org.apache.spark.sql.connector.expressions.NamedReference::fieldNames) + .toArray(String[][]::new); + return Transforms.bucket(numBuckets, fieldNames); + } else if (transform instanceof HoursTransform) { + HoursTransform hoursTransform = (HoursTransform) transform; + return Transforms.hour(hoursTransform.reference().fieldNames()); + } else if (transform instanceof DaysTransform) { + DaysTransform daysTransform = (DaysTransform) transform; + return Transforms.day(daysTransform.reference().fieldNames()); + } else if (transform instanceof MonthsTransform) { + MonthsTransform monthsTransform = (MonthsTransform) transform; + return Transforms.month(monthsTransform.reference().fieldNames()); + } else if (transform instanceof YearsTransform) { + YearsTransform yearsTransform = (YearsTransform) transform; + return Transforms.year(yearsTransform.reference().fieldNames()); + } else if (transform instanceof ApplyTransform + && "truncate".equalsIgnoreCase(transform.name())) { + Preconditions.checkArgument( + transform.references().length == 1, + "Truncate transform should have only one reference"); + return Transforms.truncate( + findWidth(transform), + getFieldNameFromGravitinoNamedReference( + (NamedReference) toGravitinoNamedReference(transform.references()[0]))); } else { throw new NotSupportedException( "Doesn't support Spark transform: " + transform.name()); @@ -76,7 +130,7 @@ public static Transform[] toGravitinoPartitionings( .toArray(Transform[]::new); } - public static DistributionAndSortOrdersInfo toGravitinoDistributionAndSortOrders( + public DistributionAndSortOrdersInfo toGravitinoDistributionAndSortOrders( org.apache.spark.sql.connector.expressions.Transform[] transforms) { DistributionAndSortOrdersInfo distributionAndSortOrdersInfo = new DistributionAndSortOrdersInfo(); @@ -85,7 +139,7 @@ public static DistributionAndSortOrdersInfo toGravitinoDistributionAndSortOrders } Arrays.stream(transforms) - .filter(transform -> isBucketTransform(transform)) + .filter(transform -> !isPartitionTransform(transform)) .forEach( transform -> { if (transform instanceof SortedBucketTransform) { @@ -104,10 +158,16 @@ public static DistributionAndSortOrdersInfo toGravitinoDistributionAndSortOrders } }); + if (distributionAndSortOrdersInfo.getDistribution() == null) { + distributionAndSortOrdersInfo.setDistribution(Distributions.NONE); + } + if (distributionAndSortOrdersInfo.getSortOrders() == null) { + distributionAndSortOrdersInfo.setSortOrders(new SortOrder[0]); + } return distributionAndSortOrdersInfo; } - public static org.apache.spark.sql.connector.expressions.Transform[] toSparkTransform( + public org.apache.spark.sql.connector.expressions.Transform[] toSparkTransform( com.datastrato.gravitino.rel.expressions.transforms.Transform[] partitions, Distribution distribution, SortOrder[] sortOrder) { @@ -117,11 +177,54 @@ public static org.apache.spark.sql.connector.expressions.Transform[] toSparkTran .forEach( transform -> { if (transform instanceof Transforms.IdentityTransform) { + Preconditions.checkArgument( + transform.references().length == 1, + "Identity transform should have only one reference"); Transforms.IdentityTransform identityTransform = (Transforms.IdentityTransform) transform; sparkTransforms.add( createSparkIdentityTransform( - String.join(ConnectorConstants.DOT, identityTransform.fieldName()))); + getFieldNameFromGravitinoNamedReference( + identityTransform.references()[0]))); + } else if (transform instanceof Transforms.HourTransform) { + Preconditions.checkArgument( + transform.references().length == 1, + "Hour transform should have only one reference"); + Transforms.HourTransform hourTransform = (Transforms.HourTransform) transform; + sparkTransforms.add(createSparkHoursTransform(hourTransform.references()[0])); + } else if (transform instanceof Transforms.BucketTransform) { + Transforms.BucketTransform bucketTransform = + (Transforms.BucketTransform) transform; + int numBuckets = bucketTransform.numBuckets(); + String[] fieldNames = + Arrays.stream(bucketTransform.fieldNames()) + .map(f -> String.join(ConnectorConstants.DOT, f)) + .toArray(String[]::new); + sparkTransforms.add(createSparkBucketTransform(numBuckets, fieldNames)); + } else if (transform instanceof Transforms.DayTransform) { + Preconditions.checkArgument( + transform.references().length == 1, + "Day transform should have only one reference"); + Transforms.DayTransform dayTransform = (Transforms.DayTransform) transform; + sparkTransforms.add(createSparkDaysTransform(dayTransform.references()[0])); + } else if (transform instanceof Transforms.MonthTransform) { + Preconditions.checkArgument( + transform.references().length == 1, + "Month transform should have only one reference"); + Transforms.MonthTransform monthTransform = (Transforms.MonthTransform) transform; + sparkTransforms.add(createSparkMonthsTransform(monthTransform.references()[0])); + } else if (transform instanceof Transforms.YearTransform) { + Preconditions.checkArgument( + transform.references().length == 1, + "Year transform should have only one reference"); + Transforms.YearTransform yearTransform = (Transforms.YearTransform) transform; + sparkTransforms.add(createSparkYearsTransform(yearTransform.references()[0])); + } else if (transform instanceof Transforms.TruncateTransform) { + Transforms.TruncateTransform truncateTransform = + (Transforms.TruncateTransform) transform; + int width = truncateTransform.width(); + String[] fieldName = truncateTransform.fieldName(); + sparkTransforms.add(createSparkTruncateTransform(width, fieldName)); } else { throw new UnsupportedOperationException( "Doesn't support Gravitino partition: " @@ -132,10 +235,12 @@ public static org.apache.spark.sql.connector.expressions.Transform[] toSparkTran }); } - org.apache.spark.sql.connector.expressions.Transform bucketTransform = - toSparkBucketTransform(distribution, sortOrder); - if (bucketTransform != null) { - sparkTransforms.add(bucketTransform); + if (!supportsBucketPartition) { + org.apache.spark.sql.connector.expressions.Transform bucketTransform = + toSparkBucketTransform(distribution, sortOrder); + if (bucketTransform != null) { + sparkTransforms.add(bucketTransform); + } } return sparkTransforms.toArray(new org.apache.spark.sql.connector.expressions.Transform[0]); @@ -209,10 +314,15 @@ private static org.apache.spark.sql.connector.expressions.Transform toSparkBucke private static Expression[] toGravitinoNamedReference( List sparkNamedReferences) { return sparkNamedReferences.stream() - .map(sparkReference -> NamedReference.field(sparkReference.fieldNames())) + .map(SparkTransformConverter::toGravitinoNamedReference) .toArray(Expression[]::new); } + private static Expression toGravitinoNamedReference( + org.apache.spark.sql.connector.expressions.NamedReference sparkNamedReference) { + return NamedReference.field(sparkNamedReference.fieldNames()); + } + public static org.apache.spark.sql.connector.expressions.Transform createSortBucketTransform( int bucketNum, String[] bucketFields, String[] sortFields) { return LogicalExpressions.bucket( @@ -224,6 +334,38 @@ public static IdentityTransform createSparkIdentityTransform(String columnName) return IdentityTransform.apply(Expressions.column(columnName)); } + public static HoursTransform createSparkHoursTransform(NamedReference gravitinoNamedReference) { + return LogicalExpressions.hours( + Expressions.column(getFieldNameFromGravitinoNamedReference(gravitinoNamedReference))); + } + + public static BucketTransform createSparkBucketTransform(int numBuckets, String[] fieldNames) { + return LogicalExpressions.bucket(numBuckets, createSparkNamedReference(fieldNames)); + } + + public static DaysTransform createSparkDaysTransform(NamedReference gravitinoNamedReference) { + return LogicalExpressions.days( + Expressions.column(getFieldNameFromGravitinoNamedReference(gravitinoNamedReference))); + } + + public static MonthsTransform createSparkMonthsTransform(NamedReference gravitinoNamedReference) { + return LogicalExpressions.months( + Expressions.column(getFieldNameFromGravitinoNamedReference(gravitinoNamedReference))); + } + + public static YearsTransform createSparkYearsTransform(NamedReference gravitinoNamedReference) { + return LogicalExpressions.years( + Expressions.column(getFieldNameFromGravitinoNamedReference(gravitinoNamedReference))); + } + + public static org.apache.spark.sql.connector.expressions.Transform createSparkTruncateTransform( + int width, String[] fieldName) { + return Expressions.apply( + "truncate", + Expressions.literal(width), + Expressions.column(String.join(ConnectorConstants.DOT, fieldName))); + } + private static org.apache.spark.sql.connector.expressions.NamedReference[] createSparkNamedReference(String[] fields) { return Arrays.stream(fields) @@ -237,8 +379,41 @@ private static String getFieldNameFromGravitinoNamedReference( return String.join(ConnectorConstants.DOT, gravitinoNamedReference.fieldName()); } - private static boolean isBucketTransform( + private boolean isPartitionTransform( org.apache.spark.sql.connector.expressions.Transform transform) { - return transform instanceof BucketTransform || transform instanceof SortedBucketTransform; + if (supportsBucketPartition) { + Preconditions.checkArgument( + !(transform instanceof SortedBucketTransform), + "Spark doesn't support SortedBucketTransform as partition transform"); + return true; + } + return !(transform instanceof BucketTransform || transform instanceof SortedBucketTransform); + } + + // Referred from org.apache.iceberg.spark.Spark3Util + private static int findWidth(org.apache.spark.sql.connector.expressions.Transform transform) { + for (org.apache.spark.sql.connector.expressions.Expression expr : transform.arguments()) { + if (expr instanceof Literal) { + if (((Literal) expr).dataType() instanceof IntegerType) { + Literal lit = (Literal) expr; + Preconditions.checkArgument( + lit.value() > 0, "Unsupported width for transform: %s", transform.describe()); + return lit.value(); + + } else if (((Literal) expr).dataType() instanceof LongType) { + Literal lit = (Literal) expr; + Preconditions.checkArgument( + lit.value() > 0 && lit.value() < Integer.MAX_VALUE, + "Unsupported width for transform: %s", + transform.describe()); + if (lit.value() > Integer.MAX_VALUE) { + throw new IllegalArgumentException(); + } + return lit.value().intValue(); + } + } + } + + throw new IllegalArgumentException("Cannot find width for transform: " + transform.describe()); } } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java index bd6c26f9241..f5994b4ce86 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/catalog/BaseCatalog.java @@ -65,6 +65,7 @@ public abstract class BaseCatalog implements TableCatalog, SupportsNamespaces { // The Gravitino catalog client to do schema operations. protected Catalog gravitinoCatalogClient; protected PropertiesConverter propertiesConverter; + protected SparkTransformConverter sparkTransformConverter; private final String metalakeName; private String catalogName; @@ -94,13 +95,16 @@ protected abstract TableCatalog createAndInitSparkCatalog( * @param gravitinoTable Gravitino table to do DDL operations * @param sparkCatalog specific Spark catalog to do IO operations * @param propertiesConverter transform properties between Gravitino and Spark + * @param sparkTransformConverter sparkTransformConverter convert transforms between Gravitino and + * Spark * @return a specific Spark table */ protected abstract SparkBaseTable createSparkTable( Identifier identifier, com.datastrato.gravitino.rel.Table gravitinoTable, TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter); + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter); /** * Get a PropertiesConverter to transform properties between Gravitino and Spark. @@ -109,6 +113,13 @@ protected abstract SparkBaseTable createSparkTable( */ protected abstract PropertiesConverter getPropertiesConverter(); + /** + * Get a SparkTransformConverter to convert transforms between Gravitino and Spark. + * + * @return an SparkTransformConverter + */ + protected abstract SparkTransformConverter getSparkTransformConverter(); + @Override public void initialize(String name, CaseInsensitiveStringMap options) { this.catalogName = name; @@ -119,6 +130,7 @@ public void initialize(String name, CaseInsensitiveStringMap options) { this.sparkCatalog = createAndInitSparkCatalog(name, options, gravitinoCatalogClient.properties()); this.propertiesConverter = getPropertiesConverter(); + this.sparkTransformConverter = getSparkTransformConverter(); } @Override @@ -167,9 +179,9 @@ public Table createTable( String comment = gravitinoProperties.remove(ConnectorConstants.COMMENT); DistributionAndSortOrdersInfo distributionAndSortOrdersInfo = - SparkTransformConverter.toGravitinoDistributionAndSortOrders(transforms); + sparkTransformConverter.toGravitinoDistributionAndSortOrders(transforms); com.datastrato.gravitino.rel.expressions.transforms.Transform[] partitionings = - SparkTransformConverter.toGravitinoPartitionings(transforms); + sparkTransformConverter.toGravitinoPartitionings(transforms); try { com.datastrato.gravitino.rel.Table table = @@ -183,7 +195,8 @@ public Table createTable( partitionings, distributionAndSortOrdersInfo.getDistribution(), distributionAndSortOrdersInfo.getSortOrders()); - return createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable( + ident, table, sparkCatalog, propertiesConverter, sparkTransformConverter); } catch (NoSuchSchemaException e) { throw new NoSuchNamespaceException(ident.namespace()); } catch (com.datastrato.gravitino.exceptions.TableAlreadyExistsException e) { @@ -200,7 +213,8 @@ public Table loadTable(Identifier ident) throws NoSuchTableException { .asTableCatalog() .loadTable(NameIdentifier.of(metalakeName, catalogName, database, ident.name())); // Will create a catalog specific table - return createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable( + ident, table, sparkCatalog, propertiesConverter, sparkTransformConverter); } catch (com.datastrato.gravitino.exceptions.NoSuchTableException e) { throw new NoSuchTableException(ident); } @@ -227,7 +241,8 @@ public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchT .alterTable( NameIdentifier.of(metalakeName, catalogName, getDatabase(ident), ident.name()), gravitinoTableChanges); - return createSparkTable(ident, table, sparkCatalog, propertiesConverter); + return createSparkTable( + ident, table, sparkCatalog, propertiesConverter, sparkTransformConverter); } catch (com.datastrato.gravitino.exceptions.NoSuchTableException e) { throw new NoSuchTableException(ident); } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java index 64b61754a2e..6ffca1ff9f4 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/GravitinoHiveCatalog.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.spark.connector.GravitinoSparkConfig; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.catalog.BaseCatalog; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import com.google.common.base.Preconditions; @@ -45,12 +46,19 @@ protected SparkBaseTable createSparkTable( Identifier identifier, Table gravitinoTable, TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { - return new SparkHiveTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter) { + return new SparkHiveTable( + identifier, gravitinoTable, sparkCatalog, propertiesConverter, sparkTransformConverter); } @Override protected PropertiesConverter getPropertiesConverter() { return new HivePropertiesConverter(); } + + @Override + protected SparkTransformConverter getSparkTransformConverter() { + return new SparkTransformConverter(false); + } } diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/SparkHiveTable.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/SparkHiveTable.java index 5843b06584e..91f9468178b 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/SparkHiveTable.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/hive/SparkHiveTable.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; @@ -17,8 +18,9 @@ public SparkHiveTable( Identifier identifier, Table gravitinoTable, TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { - super(identifier, gravitinoTable, sparkCatalog, propertiesConverter); + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter) { + super(identifier, gravitinoTable, sparkCatalog, propertiesConverter, sparkTransformConverter); } @Override diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java index d1a8a7a9fcf..f7a028cad7a 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.catalog.BaseCatalog; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import com.google.common.base.Preconditions; @@ -69,8 +70,10 @@ protected SparkBaseTable createSparkTable( Identifier identifier, Table gravitinoTable, TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { - return new SparkIcebergTable(identifier, gravitinoTable, sparkCatalog, propertiesConverter); + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter) { + return new SparkIcebergTable( + identifier, gravitinoTable, sparkCatalog, propertiesConverter, sparkTransformConverter); } @Override @@ -78,6 +81,11 @@ protected PropertiesConverter getPropertiesConverter() { return new IcebergPropertiesConverter(); } + @Override + protected SparkTransformConverter getSparkTransformConverter() { + return new SparkTransformConverter(true); + } + @Override public Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceException { return ((SparkCatalog) sparkCatalog).listFunctions(namespace); diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/SparkIcebergTable.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/SparkIcebergTable.java index d4a496c409e..75176071707 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/SparkIcebergTable.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/iceberg/SparkIcebergTable.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.spark.connector.PropertiesConverter; +import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.SupportsDelete; @@ -19,8 +20,14 @@ public SparkIcebergTable( Identifier identifier, Table gravitinoTable, TableCatalog sparkIcebergCatalog, - PropertiesConverter propertiesConverter) { - super(identifier, gravitinoTable, sparkIcebergCatalog, propertiesConverter); + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter) { + super( + identifier, + gravitinoTable, + sparkIcebergCatalog, + propertiesConverter, + sparkTransformConverter); } @Override diff --git a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/table/SparkBaseTable.java b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/table/SparkBaseTable.java index 0d057656e86..d1333135f19 100644 --- a/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/table/SparkBaseTable.java +++ b/spark-connector/spark-connector/src/main/java/com/datastrato/gravitino/spark/connector/table/SparkBaseTable.java @@ -11,6 +11,7 @@ import com.datastrato.gravitino.spark.connector.PropertiesConverter; import com.datastrato.gravitino.spark.connector.SparkTransformConverter; import com.datastrato.gravitino.spark.connector.SparkTypeConverter; +import com.google.common.annotations.VisibleForTesting; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -47,16 +48,19 @@ public abstract class SparkBaseTable implements Table, SupportsRead, SupportsWri private TableCatalog sparkCatalog; private Table lazySparkTable; private PropertiesConverter propertiesConverter; + private SparkTransformConverter sparkTransformConverter; public SparkBaseTable( Identifier identifier, com.datastrato.gravitino.rel.Table gravitinoTable, TableCatalog sparkCatalog, - PropertiesConverter propertiesConverter) { + PropertiesConverter propertiesConverter, + SparkTransformConverter sparkTransformConverter) { this.identifier = identifier; this.gravitinoTable = gravitinoTable; this.sparkCatalog = sparkCatalog; this.propertiesConverter = propertiesConverter; + this.sparkTransformConverter = sparkTransformConverter; } @Override @@ -127,7 +131,7 @@ public Transform[] partitioning() { gravitinoTable.partitioning(); Distribution distribution = gravitinoTable.distribution(); SortOrder[] sortOrders = gravitinoTable.sortOrder(); - return SparkTransformConverter.toSparkTransform(partitions, distribution, sortOrders); + return sparkTransformConverter.toSparkTransform(partitions, distribution, sortOrders); } protected Table getSparkTable() { @@ -141,6 +145,11 @@ protected Table getSparkTable() { return lazySparkTable; } + @VisibleForTesting + public SparkTransformConverter getSparkTransformConverter() { + return sparkTransformConverter; + } + protected boolean isCaseSensitive() { return true; } diff --git a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/TestSparkTransformConverter.java b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/TestSparkTransformConverter.java index ea00eeb5b58..0618e4f8f5a 100644 --- a/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/TestSparkTransformConverter.java +++ b/spark-connector/spark-connector/src/test/java/com/datastrato/gravitino/spark/connector/TestSparkTransformConverter.java @@ -25,9 +25,10 @@ import org.apache.spark.sql.connector.expressions.SortedBucketTransform; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import scala.collection.JavaConverters; @TestInstance(Lifecycle.PER_CLASS) @@ -40,83 +41,114 @@ void init() { initSparkToGravitinoTransformMap(); } - @Test - void testPartition() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testPartition(boolean supportsBucketPartition) { + SparkTransformConverter sparkTransformConverter = + new SparkTransformConverter(supportsBucketPartition); + sparkToGravitinoPartitionTransformMaps.forEach( (sparkTransform, gravitinoTransform) -> { Transform[] gravitinoPartitionings = - SparkTransformConverter.toGravitinoPartitionings( + sparkTransformConverter.toGravitinoPartitionings( new org.apache.spark.sql.connector.expressions.Transform[] {sparkTransform}); - Assertions.assertTrue( - gravitinoPartitionings != null && gravitinoPartitionings.length == 1); - Assertions.assertEquals(gravitinoTransform, gravitinoPartitionings[0]); + if (sparkTransform instanceof BucketTransform && !supportsBucketPartition) { + Assertions.assertTrue( + gravitinoPartitionings != null && gravitinoPartitionings.length == 0); + } else { + Assertions.assertTrue( + gravitinoPartitionings != null && gravitinoPartitionings.length == 1); + Assertions.assertEquals(gravitinoTransform, gravitinoPartitionings[0]); + } }); sparkToGravitinoPartitionTransformMaps.forEach( (sparkTransform, gravitinoTransform) -> { org.apache.spark.sql.connector.expressions.Transform[] sparkTransforms = - SparkTransformConverter.toSparkTransform( + sparkTransformConverter.toSparkTransform( new Transform[] {gravitinoTransform}, null, null); - Assertions.assertTrue(sparkTransforms.length == 1); + Assertions.assertEquals(1, sparkTransforms.length); Assertions.assertEquals(sparkTransform, sparkTransforms[0]); }); } - @Test - void testGravitinoToSparkDistributionWithoutSortOrder() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testGravitinoToSparkDistributionWithoutSortOrder(boolean supportsBucketPartition) { + SparkTransformConverter sparkTransformConverter = + new SparkTransformConverter(supportsBucketPartition); int bucketNum = 16; String[][] columnNames = createGravitinoFieldReferenceNames("a", "b.c"); Distribution gravitinoDistribution = createHashDistribution(bucketNum, columnNames); org.apache.spark.sql.connector.expressions.Transform[] sparkTransforms = - SparkTransformConverter.toSparkTransform(null, gravitinoDistribution, null); - Assertions.assertTrue(sparkTransforms != null && sparkTransforms.length == 1); - Assertions.assertTrue(sparkTransforms[0] instanceof BucketTransform); - BucketTransform bucket = (BucketTransform) sparkTransforms[0]; - Assertions.assertEquals(bucketNum, (Integer) bucket.numBuckets().value()); - String[][] columns = - JavaConverters.seqAsJavaList(bucket.columns()).stream() - .map(namedReference -> namedReference.fieldNames()) - .toArray(String[][]::new); - Assertions.assertArrayEquals(columnNames, columns); + sparkTransformConverter.toSparkTransform(null, gravitinoDistribution, null); + if (supportsBucketPartition) { + Assertions.assertTrue(sparkTransforms != null && sparkTransforms.length == 0); + } else { + Assertions.assertTrue(sparkTransforms != null && sparkTransforms.length == 1); + Assertions.assertTrue(sparkTransforms[0] instanceof BucketTransform); + BucketTransform bucket = (BucketTransform) sparkTransforms[0]; + Assertions.assertEquals(bucketNum, (Integer) bucket.numBuckets().value()); + String[][] columns = + JavaConverters.seqAsJavaList(bucket.columns()).stream() + .map(namedReference -> namedReference.fieldNames()) + .toArray(String[][]::new); + Assertions.assertArrayEquals(columnNames, columns); + } // none and null distribution - sparkTransforms = SparkTransformConverter.toSparkTransform(null, null, null); + sparkTransforms = sparkTransformConverter.toSparkTransform(null, null, null); Assertions.assertEquals(0, sparkTransforms.length); - sparkTransforms = SparkTransformConverter.toSparkTransform(null, Distributions.NONE, null); + sparkTransforms = sparkTransformConverter.toSparkTransform(null, Distributions.NONE, null); Assertions.assertEquals(0, sparkTransforms.length); - // range and even distribution - Assertions.assertThrowsExactly( - NotSupportedException.class, - () -> SparkTransformConverter.toSparkTransform(null, Distributions.RANGE, null)); - Distribution evenDistribution = Distributions.even(bucketNum, NamedReference.field("")); - Assertions.assertThrowsExactly( - NotSupportedException.class, - () -> SparkTransformConverter.toSparkTransform(null, evenDistribution, null)); + if (!supportsBucketPartition) { + // range and even distribution + Assertions.assertThrowsExactly( + NotSupportedException.class, + () -> sparkTransformConverter.toSparkTransform(null, Distributions.RANGE, null)); + Distribution evenDistribution = Distributions.even(bucketNum, NamedReference.field("")); + Assertions.assertThrowsExactly( + NotSupportedException.class, + () -> sparkTransformConverter.toSparkTransform(null, evenDistribution, null)); + } } - @Test - void testSparkToGravitinoDistributionWithoutSortOrder() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testSparkToGravitinoDistributionWithoutSortOrder(boolean supportsBucketPartition) { + SparkTransformConverter sparkTransformConverter = + new SparkTransformConverter(supportsBucketPartition); int bucketNum = 16; String[] sparkFieldReferences = new String[] {"a", "b.c"}; org.apache.spark.sql.connector.expressions.Transform sparkBucket = Expressions.bucket(bucketNum, sparkFieldReferences); DistributionAndSortOrdersInfo distributionAndSortOrdersInfo = - SparkTransformConverter.toGravitinoDistributionAndSortOrders( + sparkTransformConverter.toGravitinoDistributionAndSortOrders( new org.apache.spark.sql.connector.expressions.Transform[] {sparkBucket}); - Assertions.assertNull(distributionAndSortOrdersInfo.getSortOrders()); - - Distribution distribution = distributionAndSortOrdersInfo.getDistribution(); - String[][] gravitinoFieldReferences = createGravitinoFieldReferenceNames(sparkFieldReferences); - Assertions.assertEquals( - createHashDistribution(bucketNum, gravitinoFieldReferences), distribution); + if (!supportsBucketPartition) { + Distribution distribution = distributionAndSortOrdersInfo.getDistribution(); + String[][] gravitinoFieldReferences = + createGravitinoFieldReferenceNames(sparkFieldReferences); + Assertions.assertEquals( + createHashDistribution(bucketNum, gravitinoFieldReferences), distribution); + } else { + Assertions.assertNotNull(distributionAndSortOrdersInfo.getSortOrders()); + Assertions.assertEquals(0, distributionAndSortOrdersInfo.getSortOrders().length); + Assertions.assertNotNull(distributionAndSortOrdersInfo.getDistribution()); + Assertions.assertEquals(Distributions.NONE, distributionAndSortOrdersInfo.getDistribution()); + } } - @Test - void testSparkToGravitinoDistributionWithSortOrder() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testSparkToGravitinoDistributionWithSortOrder(boolean supportsBucketPartition) { + SparkTransformConverter sparkTransformConverter = + new SparkTransformConverter(supportsBucketPartition); + int bucketNum = 16; String[][] bucketColumnNames = createGravitinoFieldReferenceNames("a", "b.c"); String[][] sortColumnNames = createGravitinoFieldReferenceNames("f", "m.n"); @@ -126,20 +158,34 @@ void testSparkToGravitinoDistributionWithSortOrder() { createSparkFieldReference(bucketColumnNames), createSparkFieldReference(sortColumnNames)); - DistributionAndSortOrdersInfo distributionAndSortOrders = - SparkTransformConverter.toGravitinoDistributionAndSortOrders( - new org.apache.spark.sql.connector.expressions.Transform[] {sortedBucketTransform}); - Assertions.assertEquals( - createHashDistribution(bucketNum, bucketColumnNames), - distributionAndSortOrders.getDistribution()); + if (!supportsBucketPartition) { + DistributionAndSortOrdersInfo distributionAndSortOrders = + sparkTransformConverter.toGravitinoDistributionAndSortOrders( + new org.apache.spark.sql.connector.expressions.Transform[] {sortedBucketTransform}); - SortOrder[] sortOrders = - createSortOrders(sortColumnNames, ConnectorConstants.SPARK_DEFAULT_SORT_DIRECTION); - Assertions.assertArrayEquals(sortOrders, distributionAndSortOrders.getSortOrders()); + Assertions.assertEquals( + createHashDistribution(bucketNum, bucketColumnNames), + distributionAndSortOrders.getDistribution()); + + SortOrder[] sortOrders = + createSortOrders(sortColumnNames, ConnectorConstants.SPARK_DEFAULT_SORT_DIRECTION); + Assertions.assertArrayEquals(sortOrders, distributionAndSortOrders.getSortOrders()); + } else { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + sparkTransformConverter.toGravitinoDistributionAndSortOrders( + new org.apache.spark.sql.connector.expressions.Transform[] { + sortedBucketTransform + })); + } } - @Test - void testGravitinoToSparkDistributionWithSortOrder() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testGravitinoToSparkDistributionWithSortOrder(boolean supportsBucketPartition) { + SparkTransformConverter sparkTransformConverter = + new SparkTransformConverter(supportsBucketPartition); int bucketNum = 16; String[][] bucketColumnNames = createGravitinoFieldReferenceNames("a", "b.c"); String[][] sortColumnNames = createGravitinoFieldReferenceNames("f", "m.n"); @@ -148,24 +194,28 @@ void testGravitinoToSparkDistributionWithSortOrder() { createSortOrders(sortColumnNames, ConnectorConstants.SPARK_DEFAULT_SORT_DIRECTION); org.apache.spark.sql.connector.expressions.Transform[] transforms = - SparkTransformConverter.toSparkTransform(null, distribution, sortOrders); - Assertions.assertTrue(transforms.length == 1); - Assertions.assertTrue(transforms[0] instanceof SortedBucketTransform); - - SortedBucketTransform sortedBucketTransform = (SortedBucketTransform) transforms[0]; - Assertions.assertEquals(bucketNum, (Integer) sortedBucketTransform.numBuckets().value()); - String[][] sparkSortColumns = - JavaConverters.seqAsJavaList(sortedBucketTransform.sortedColumns()).stream() - .map(sparkNamedReference -> sparkNamedReference.fieldNames()) - .toArray(String[][]::new); - - String[][] sparkBucketColumns = - JavaConverters.seqAsJavaList(sortedBucketTransform.columns()).stream() - .map(sparkNamedReference -> sparkNamedReference.fieldNames()) - .toArray(String[][]::new); - - Assertions.assertArrayEquals(bucketColumnNames, sparkBucketColumns); - Assertions.assertArrayEquals(sortColumnNames, sparkSortColumns); + sparkTransformConverter.toSparkTransform(null, distribution, sortOrders); + if (!supportsBucketPartition) { + Assertions.assertTrue(transforms.length == 1); + Assertions.assertTrue(transforms[0] instanceof SortedBucketTransform); + + SortedBucketTransform sortedBucketTransform = (SortedBucketTransform) transforms[0]; + Assertions.assertEquals(bucketNum, (Integer) sortedBucketTransform.numBuckets().value()); + String[][] sparkSortColumns = + JavaConverters.seqAsJavaList(sortedBucketTransform.sortedColumns()).stream() + .map(sparkNamedReference -> sparkNamedReference.fieldNames()) + .toArray(String[][]::new); + + String[][] sparkBucketColumns = + JavaConverters.seqAsJavaList(sortedBucketTransform.columns()).stream() + .map(sparkNamedReference -> sparkNamedReference.fieldNames()) + .toArray(String[][]::new); + + Assertions.assertArrayEquals(bucketColumnNames, sparkBucketColumns); + Assertions.assertArrayEquals(sortColumnNames, sparkSortColumns); + } else { + Assertions.assertEquals(0, transforms.length); + } } private org.apache.spark.sql.connector.expressions.NamedReference[] createSparkFieldReference( @@ -202,5 +252,41 @@ private void initSparkToGravitinoTransformMap() { sparkToGravitinoPartitionTransformMaps.put( SparkTransformConverter.createSparkIdentityTransform("a.b"), Transforms.identity(new String[] {"a", "b"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkBucketTransform(10, new String[] {"a"}), + Transforms.bucket(10, new String[] {"a"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkBucketTransform(10, new String[] {"msg.a"}), + Transforms.bucket(10, new String[] {"msg", "a"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkHoursTransform(NamedReference.field("date")), + Transforms.hour("date")); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkHoursTransform(NamedReference.field("msg.date")), + Transforms.hour(new String[] {"msg", "date"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkDaysTransform(NamedReference.field("date")), + Transforms.day("date")); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkDaysTransform(NamedReference.field("msg.date")), + Transforms.day(new String[] {"msg", "date"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkMonthsTransform(NamedReference.field("date")), + Transforms.month("date")); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkMonthsTransform(NamedReference.field("msg.date")), + Transforms.month(new String[] {"msg", "date"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkYearsTransform(NamedReference.field("date")), + Transforms.year("date")); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkYearsTransform(NamedReference.field("msg.date")), + Transforms.year(new String[] {"msg", "date"})); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkTruncateTransform(10, new String[] {"package"}), + Transforms.truncate(10, "package")); + sparkToGravitinoPartitionTransformMaps.put( + SparkTransformConverter.createSparkTruncateTransform(10, new String[] {"msg.package"}), + Transforms.truncate(10, "msg.package")); } } From 008adb4b7710df41815618f74dbd6acb7c76b4a6 Mon Sep 17 00:00:00 2001 From: lwyang <1670906161@qq.com> Date: Sat, 20 Apr 2024 14:54:23 +0800 Subject: [PATCH 079/106] [#2705] feat(core): Add the relational backend for User Entity (#2850) ### What changes were proposed in this pull request? add relational backend for User Entity ### Why are the changes needed? Fix: #2705 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? ut --------- Co-authored-by: yangliwei --- .../authorization/AuthorizationUtils.java | 2 +- .../storage/relational/JDBCBackend.java | 14 + .../relational/mapper/RoleMetaMapper.java | 99 +++ .../relational/mapper/UserMetaMapper.java | 119 ++++ .../relational/mapper/UserRoleRelMapper.java | 105 +++ .../storage/relational/po/RolePO.java | 182 ++++++ .../storage/relational/po/UserPO.java | 143 +++++ .../storage/relational/po/UserRoleRelPO.java | 131 ++++ .../service/MetalakeMetaService.java | 28 +- .../relational/service/RoleMetaService.java | 77 +++ .../relational/service/UserMetaService.java | 226 +++++++ .../session/SqlSessionFactoryHelper.java | 6 + .../relational/utils/POConverters.java | 137 ++++ .../gravitino/storage/TestEntityStorage.java | 78 ++- .../storage/memory/TestMemoryEntityStore.java | 15 + .../storage/relational/TestJDBCBackend.java | 43 ++ .../service/TestUserMetaService.java | 604 ++++++++++++++++++ core/src/test/resources/h2/schema-h2.sql | 40 ++ scripts/mysql/schema-0.5.0-mysql.sql | 42 +- 19 files changed, 2074 insertions(+), 17 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserRoleRelPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java index d6ded111e3a..05475abc9fa 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java @@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory; /* The utilization class of authorization module*/ -class AuthorizationUtils { +public class AuthorizationUtils { static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in th metalake %s"; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 14a0df4fcac..9d884fa8087 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -18,16 +18,20 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.relational.converters.SQLExceptionConverterFactory; import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; +import com.datastrato.gravitino.storage.relational.service.RoleMetaService; import com.datastrato.gravitino.storage.relational.service.SchemaMetaService; import com.datastrato.gravitino.storage.relational.service.TableMetaService; import com.datastrato.gravitino.storage.relational.service.TopicMetaService; +import com.datastrato.gravitino.storage.relational.service.UserMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import java.io.IOException; import java.util.List; @@ -95,6 +99,10 @@ public void insert(E e, boolean overwritten) FilesetMetaService.getInstance().insertFileset((FilesetEntity) e, overwritten); } else if (e instanceof TopicEntity) { TopicMetaService.getInstance().insertTopic((TopicEntity) e, overwritten); + } else if (e instanceof UserEntity) { + UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); + } else if (e instanceof RoleEntity) { + RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); @@ -118,6 +126,8 @@ public E update( return (E) FilesetMetaService.getInstance().updateFileset(ident, updater); case TOPIC: return (E) TopicMetaService.getInstance().updateTopic(ident, updater); + case USER: + return (E) UserMetaService.getInstance().updateUser(ident, updater); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for update operation", entityType); @@ -140,6 +150,8 @@ public E get( return (E) FilesetMetaService.getInstance().getFilesetByIdentifier(ident); case TOPIC: return (E) TopicMetaService.getInstance().getTopicByIdentifier(ident); + case USER: + return (E) UserMetaService.getInstance().getUserByIdentifier(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for get operation", entityType); @@ -161,6 +173,8 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea return FilesetMetaService.getInstance().deleteFileset(ident); case TOPIC: return TopicMetaService.getInstance().deleteTopic(ident); + case USER: + return UserMetaService.getInstance().deleteUser(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for delete operation", entityType); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java new file mode 100644 index 00000000000..f9732d040e1 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java @@ -0,0 +1,99 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.RolePO; +import java.util.List; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface RoleMetaMapper { + String ROLE_TABLE_NAME = "role_meta"; + String RELATION_TABLE_NAME = "user_role_rel"; + + @Select( + "SELECT role_id as roleId FROM " + + ROLE_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" + + " AND deleted_at = 0") + Long selectRoleIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String name); + + @Select( + "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.securable_object as securableObject, ro.privileges as privileges," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + RELATION_TABLE_NAME + + " re ON ro.role_id = re.role_id" + + " WHERE re.user_id = #{userId}" + + " AND ro.deleted_at = 0 AND re.deleted_at = 0") + List listRolesByUserId(@Param("userId") Long userId); + + @Insert( + "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " securable_object, privileges," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.securableObject}," + + " #{roleMeta.privileges}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " )") + void insertRoleMeta(@Param("roleMeta") RolePO rolePO); + + @Insert( + "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " securable_object, privileges," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.securableObject}," + + " #{roleMeta.privileges}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " ) ON DUPLICATE KEY UPDATE" + + " role_name = #{roleMeta.roleName}," + + " metalake_id = #{roleMeta.metalakeId}," + + " properties = #{roleMeta.properties}," + + " securable_object = #{roleMeta.securableObject}," + + " privileges = #{roleMeta.privileges}," + + " audit_info = #{roleMeta.auditInfo}," + + " current_version = #{roleMeta.currentVersion}," + + " last_version = #{roleMeta.lastVersion}," + + " deleted_at = #{roleMeta.deletedAt}") + void insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java new file mode 100644 index 00000000000..83b94fa24c5 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.UserPO; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface UserMetaMapper { + String TABLE_NAME = "user_meta"; + + @Select( + "SELECT user_id as userId FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0") + Long selectUserIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name); + + @Select( + "SELECT user_id as userId, user_name as userName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0") + UserPO selectUserMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(user_id, user_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )") + void insertUserMeta(@Param("userMeta") UserPO userPO); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(user_id, user_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " user_name = #{userMeta.userName}," + + " metalake_id = #{userMeta.metalakeId}," + + " audit_info = #{userMeta.auditInfo}," + + " current_version = #{userMeta.currentVersion}," + + " last_version = #{userMeta.lastVersion}," + + " deleted_at = #{userMeta.deletedAt}") + void insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE user_id = #{userId} AND deleted_at = 0") + void softDeleteUserMetaByUserId(@Param("userId") Long userId); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + void softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET user_name = #{newUserMeta.userName}," + + " metalake_id = #{newUserMeta.metalakeId}," + + " audit_info = #{newUserMeta.auditInfo}," + + " current_version = #{newUserMeta.currentVersion}," + + " last_version = #{newUserMeta.lastVersion}," + + " deleted_at = #{newUserMeta.deletedAt}" + + " WHERE user_id = #{oldUserMeta.userId}" + + " AND user_name = #{oldUserMeta.userName}" + + " AND metalake_id = #{oldUserMeta.metalakeId}" + + " AND audit_info = #{oldUserMeta.auditInfo}" + + " AND current_version = #{oldUserMeta.currentVersion}" + + " AND last_version = #{oldUserMeta.lastVersion}" + + " AND deleted_at = 0") + Integer updateUserMeta( + @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java new file mode 100644 index 00000000000..637a34753f9 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO; +import java.util.List; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface UserRoleRelMapper { + String RELATION_TABLE_NAME = "user_role_rel"; + String USER_TABLE_NAME = "user_meta"; + + @Insert({ + "" + }) + void batchInsertUserRoleRel(@Param("userRoleRels") List userRoleRelPOs); + + @Insert({ + "" + }) + void batchInsertUserRoleRelOnDuplicateKeyUpdate( + @Param("userRoleRels") List userRoleRelPOs); + + @Update( + "UPDATE " + + RELATION_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE user_id = #{userId} AND deleted_at = 0") + void softDeleteUserRoleRelByUserId(@Param("userId") Long userId); + + @Update({ + "" + }) + void softDeleteUserRoleRelByUserAndRoles( + @Param("userId") Long userId, @Param("roleIds") List roleIds); + + @Update( + "UPDATE " + + RELATION_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE user_id IN (SELECT user_id FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0") + void softDeleteUserRoleRelByMetalakeId(Long metalakeId); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java new file mode 100644 index 00000000000..721e30a561a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java @@ -0,0 +1,182 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class RolePO { + private Long roleId; + private String roleName; + private Long metalakeId; + private String properties; + private String securableObject; + private String privileges; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getRoleId() { + return roleId; + } + + public String getRoleName() { + return roleName; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public String getProperties() { + return properties; + } + + public String getSecurableObject() { + return securableObject; + } + + public String getPrivileges() { + return privileges; + } + + public String getAuditInfo() { + return auditInfo; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RolePO)) { + return false; + } + RolePO tablePO = (RolePO) o; + return Objects.equal(getRoleId(), tablePO.getRoleId()) + && Objects.equal(getRoleName(), tablePO.getRoleName()) + && Objects.equal(getMetalakeId(), tablePO.getMetalakeId()) + && Objects.equal(getProperties(), tablePO.getProperties()) + && Objects.equal(getSecurableObject(), tablePO.getSecurableObject()) + && Objects.equal(getPrivileges(), tablePO.getPrivileges()) + && Objects.equal(getAuditInfo(), tablePO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), tablePO.getCurrentVersion()) + && Objects.equal(getLastVersion(), tablePO.getLastVersion()) + && Objects.equal(getDeletedAt(), tablePO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getRoleId(), + getRoleName(), + getMetalakeId(), + getProperties(), + getSecurableObject(), + getPrivileges(), + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final RolePO rolePO; + + private Builder() { + rolePO = new RolePO(); + } + + public Builder withRoleId(Long roleId) { + rolePO.roleId = roleId; + return this; + } + + public Builder withRoleName(String roleName) { + rolePO.roleName = roleName; + return this; + } + + public Builder withMetalakeId(Long metalakeId) { + rolePO.metalakeId = metalakeId; + return this; + } + + public Builder withProperties(String properties) { + rolePO.properties = properties; + return this; + } + + public Builder withSecurableObject(String securableObject) { + rolePO.securableObject = securableObject; + return this; + } + + public Builder withPrivileges(String privileges) { + rolePO.privileges = privileges; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + rolePO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + rolePO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + rolePO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + rolePO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(rolePO.roleId != null, "Role id is required"); + Preconditions.checkArgument(rolePO.roleName != null, "Role name is required"); + Preconditions.checkArgument(rolePO.metalakeId != null, "Metalake id is required"); + Preconditions.checkArgument(rolePO.properties != null, "Properties is required"); + Preconditions.checkArgument(rolePO.securableObject != null, "Securable object is required"); + Preconditions.checkArgument(rolePO.privileges != null, "Privileges is required"); + Preconditions.checkArgument(rolePO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument(rolePO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(rolePO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(rolePO.deletedAt != null, "Deleted at is required"); + } + + public RolePO build() { + validate(); + return rolePO; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserPO.java new file mode 100644 index 00000000000..cf22bf3432b --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserPO.java @@ -0,0 +1,143 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class UserPO { + private Long userId; + private String userName; + private Long metalakeId; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getUserId() { + return userId; + } + + public String getUserName() { + return userName; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public String getAuditInfo() { + return auditInfo; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof UserPO)) { + return false; + } + UserPO tablePO = (UserPO) o; + return Objects.equal(getUserId(), tablePO.getUserId()) + && Objects.equal(getUserName(), tablePO.getUserName()) + && Objects.equal(getMetalakeId(), tablePO.getMetalakeId()) + && Objects.equal(getAuditInfo(), tablePO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), tablePO.getCurrentVersion()) + && Objects.equal(getLastVersion(), tablePO.getLastVersion()) + && Objects.equal(getDeletedAt(), tablePO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getUserId(), + getUserName(), + getMetalakeId(), + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final UserPO userPO; + + private Builder() { + userPO = new UserPO(); + } + + public Builder withUserId(Long userId) { + userPO.userId = userId; + return this; + } + + public Builder withUserName(String userName) { + userPO.userName = userName; + return this; + } + + public Builder withMetalakeId(Long metalakeId) { + userPO.metalakeId = metalakeId; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + userPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + userPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + userPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + userPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(userPO.userId != null, "User id is required"); + Preconditions.checkArgument(userPO.userName != null, "User name is required"); + Preconditions.checkArgument(userPO.metalakeId != null, "Metalake id is required"); + Preconditions.checkArgument(userPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument(userPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(userPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(userPO.deletedAt != null, "Deleted at is required"); + } + + public UserPO build() { + validate(); + return userPO; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserRoleRelPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserRoleRelPO.java new file mode 100644 index 00000000000..48e225a7ace --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/UserRoleRelPO.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class UserRoleRelPO { + private Long userId; + private Long roleId; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getUserId() { + return userId; + } + + public Long getRoleId() { + return roleId; + } + + public String getAuditInfo() { + return auditInfo; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof UserRoleRelPO)) { + return false; + } + UserRoleRelPO userRoleRelPO = (UserRoleRelPO) o; + return Objects.equal(getUserId(), userRoleRelPO.getUserId()) + && Objects.equal(getRoleId(), userRoleRelPO.getRoleId()) + && Objects.equal(getAuditInfo(), userRoleRelPO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), userRoleRelPO.getCurrentVersion()) + && Objects.equal(getLastVersion(), userRoleRelPO.getLastVersion()) + && Objects.equal(getDeletedAt(), userRoleRelPO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getUserId(), + getRoleId(), + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final UserRoleRelPO userRoleRelPO; + + private Builder() { + userRoleRelPO = new UserRoleRelPO(); + } + + public Builder withUserId(Long userId) { + userRoleRelPO.userId = userId; + return this; + } + + public Builder withRoleId(Long roleId) { + userRoleRelPO.roleId = roleId; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + userRoleRelPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + userRoleRelPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + userRoleRelPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + userRoleRelPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(userRoleRelPO.userId != null, "User id is required"); + Preconditions.checkArgument(userRoleRelPO.roleId != null, "Role id is required"); + Preconditions.checkArgument(userRoleRelPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument( + userRoleRelPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(userRoleRelPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(userRoleRelPO.deletedAt != null, "Deleted at is required"); + } + + public UserRoleRelPO build() { + validate(); + return userRoleRelPO; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 7c76ab4ebb5..c35dd23c508 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -20,6 +20,8 @@ import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TableMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TopicMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.UserRoleRelMapper; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; import com.datastrato.gravitino.storage.relational.utils.POConverters; @@ -170,7 +172,15 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { () -> SessionUtils.doWithoutCommit( TopicMetaMapper.class, - mapper -> mapper.softDeleteTopicMetasByMetalakeId(metalakeId))); + mapper -> mapper.softDeleteTopicMetasByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> mapper.softDeleteUserRoleRelByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, + mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId))); } else { List catalogEntities = CatalogMetaService.getInstance() @@ -179,9 +189,19 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { throw new NonEmptyEntityException( "Entity %s has sub-entities, you should remove sub-entities first", ident); } - SessionUtils.doWithCommit( - MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> mapper.softDeleteUserRoleRelByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, + mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId))); } } return true; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java new file mode 100644 index 00000000000..c0656628c9f --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; +import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.base.Preconditions; +import java.util.List; + +/** The service class for role metadata. It provides the basic database operations for role. */ +public class RoleMetaService { + private static final RoleMetaService INSTANCE = new RoleMetaService(); + + public static RoleMetaService getInstance() { + return INSTANCE; + } + + private RoleMetaService() {} + + public Long getRoleIdByMetalakeIdAndName(Long metalakeId, String roleName) { + Long roleId = + SessionUtils.getWithoutCommit( + RoleMetaMapper.class, + mapper -> mapper.selectRoleIdByMetalakeIdAndName(metalakeId, roleName)); + + if (roleId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.ROLE.name().toLowerCase(), + roleName); + } + return roleId; + } + + public List listRolesByUserId(Long userId) { + return SessionUtils.getWithoutCommit( + RoleMetaMapper.class, mapper -> mapper.listRolesByUserId(userId)); + } + + public void insertRole(RoleEntity roleEntity, boolean overwritten) { + try { + Preconditions.checkArgument( + roleEntity.namespace() != null + && !roleEntity.namespace().isEmpty() + && roleEntity.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(roleEntity.namespace().level(0)); + RolePO.Builder builder = RolePO.builder().withMetalakeId(metalakeId); + RolePO rolePO = POConverters.initializeRolePOWithVersion(roleEntity, builder); + + SessionUtils.doWithCommit( + RoleMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertRoleMetaOnDuplicateKeyUpdate(rolePO); + } else { + mapper.insertRoleMeta(rolePO); + } + }); + + } catch (RuntimeException re) { + ExceptionUtils.checkSQLException( + re, Entity.EntityType.ROLE, roleEntity.nameIdentifier().toString()); + throw re; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java new file mode 100644 index 00000000000..cc1de84a19a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java @@ -0,0 +1,226 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.UserEntity; +import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.UserRoleRelMapper; +import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.po.UserPO; +import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO; +import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** The service class for user metadata. It provides the basic database operations for user. */ +public class UserMetaService { + private static final UserMetaService INSTANCE = new UserMetaService(); + + public static UserMetaService getInstance() { + return INSTANCE; + } + + private UserMetaService() {} + + private UserPO getUserPOByMetalakeIdAndName(Long metalakeId, String userName) { + UserPO userPO = + SessionUtils.getWithoutCommit( + UserMetaMapper.class, + mapper -> mapper.selectUserMetaByMetalakeIdAndName(metalakeId, userName)); + + if (userPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.USER.name().toLowerCase(), + userName); + } + return userPO; + } + + private Long getUserIdByMetalakeIdAndName(Long metalakeId, String userName) { + Long userId = + SessionUtils.getWithoutCommit( + UserMetaMapper.class, + mapper -> mapper.selectUserIdByMetalakeIdAndName(metalakeId, userName)); + + if (userId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.USER.name().toLowerCase(), + userName); + } + return userId; + } + + public UserEntity getUserByIdentifier(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + UserPO userPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); + List rolePOs = RoleMetaService.getInstance().listRolesByUserId(userPO.getUserId()); + + return POConverters.fromUserPO(userPO, rolePOs, identifier.namespace()); + } + + public void insertUser(UserEntity userEntity, boolean overwritten) { + try { + Preconditions.checkArgument( + userEntity.namespace() != null + && !userEntity.namespace().isEmpty() + && userEntity.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(userEntity.namespace().level(0)); + UserPO.Builder builder = UserPO.builder().withMetalakeId(metalakeId); + UserPO userPO = POConverters.initializeUserPOWithVersion(userEntity, builder); + + List roleIds = userEntity.roleIds(); + if (roleIds == null) { + roleIds = + Optional.ofNullable(userEntity.roleNames()).orElse(Lists.newArrayList()).stream() + .map( + roleName -> + RoleMetaService.getInstance() + .getRoleIdByMetalakeIdAndName(metalakeId, roleName)) + .collect(Collectors.toList()); + } + List userRoleRelPOs = + POConverters.initializeUserRoleRelsPOWithVersion(userEntity, roleIds); + + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertUserMetaOnDuplicateKeyUpdate(userPO); + } else { + mapper.insertUserMeta(userPO); + } + }), + () -> { + if (userRoleRelPOs.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> { + if (overwritten) { + mapper.batchInsertUserRoleRelOnDuplicateKeyUpdate(userRoleRelPOs); + } else { + mapper.batchInsertUserRoleRel(userRoleRelPOs); + } + }); + }); + } catch (RuntimeException re) { + ExceptionUtils.checkSQLException( + re, Entity.EntityType.USER, userEntity.nameIdentifier().toString()); + throw re; + } + } + + public boolean deleteUser(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + Long userId = getUserIdByMetalakeIdAndName(metalakeId, identifier.name()); + + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, mapper -> mapper.softDeleteUserMetaByUserId(userId)), + () -> + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, mapper -> mapper.softDeleteUserRoleRelByUserId(userId))); + return true; + } + + public UserEntity updateUser( + NameIdentifier identifier, Function updater) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + UserPO oldUserPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); + List rolePOs = RoleMetaService.getInstance().listRolesByUserId(oldUserPO.getUserId()); + UserEntity oldUserEntity = POConverters.fromUserPO(oldUserPO, rolePOs, identifier.namespace()); + + UserEntity newEntity = (UserEntity) updater.apply((E) oldUserEntity); + Preconditions.checkArgument( + Objects.equals(oldUserEntity.id(), newEntity.id()), + "The updated user entity id: %s should be same with the user entity id before: %s", + newEntity.id(), + oldUserEntity.id()); + + Set oldRoleIds = + oldUserEntity.roleIds() == null + ? Sets.newHashSet() + : Sets.newHashSet(oldUserEntity.roleIds()); + Set newRoleIds = + newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + + Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); + Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); + + if (insertRoleIds.isEmpty() && deleteRoleIds.isEmpty()) { + return newEntity; + } + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, + mapper -> + mapper.updateUserMeta( + POConverters.updateUserPOWithVersion(oldUserPO, newEntity), oldUserPO)), + () -> { + if (insertRoleIds.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> + mapper.batchInsertUserRoleRel( + POConverters.initializeUserRoleRelsPOWithVersion( + newEntity, Lists.newArrayList(insertRoleIds)))); + }, + () -> { + if (deleteRoleIds.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> + mapper.softDeleteUserRoleRelByUserAndRoles( + newEntity.id(), Lists.newArrayList(deleteRoleIds))); + }); + return newEntity; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index aa9b6f0f1fa..485435e1e8e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -11,9 +11,12 @@ import com.datastrato.gravitino.storage.relational.mapper.FilesetMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.FilesetVersionMapper; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TableMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TopicMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.UserRoleRelMapper; import com.google.common.base.Preconditions; import java.sql.SQLException; import java.time.Duration; @@ -86,6 +89,9 @@ public void init(Config config) { configuration.addMapper(FilesetMetaMapper.class); configuration.addMapper(FilesetVersionMapper.class); configuration.addMapper(TopicMetaMapper.class); + configuration.addMapper(UserMetaMapper.class); + configuration.addMapper(RoleMetaMapper.class); + configuration.addMapper(UserRoleRelMapper.class); // Create the SqlSessionFactory object, it is a singleton object if (sqlSessionFactory == null) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 2d3a3275850..591131071bb 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -13,18 +13,24 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.FilesetPO; import com.datastrato.gravitino.storage.relational.po.FilesetVersionPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.po.RolePO; import com.datastrato.gravitino.storage.relational.po.SchemaPO; import com.datastrato.gravitino.storage.relational.po.TablePO; import com.datastrato.gravitino.storage.relational.po.TopicPO; +import com.datastrato.gravitino.storage.relational.po.UserPO; +import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Lists; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -611,4 +617,135 @@ public static TopicPO updateTopicPOWithVersion(TopicPO oldTopicPO, TopicEntity n throw new RuntimeException("Failed to serialize json object:", e); } } + + /** + * Initialize UserPO + * + * @param userEntity UserEntity object + * @return UserPO object with version initialized + */ + public static UserPO initializeUserPOWithVersion(UserEntity userEntity, UserPO.Builder builder) { + try { + return builder + .withUserId(userEntity.id()) + .withUserName(userEntity.name()) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(userEntity.auditInfo())) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Update UserPO version + * + * @param oldUserPO the old UserEntity object + * @param newUser the new TableEntity object + * @return UserPO object with updated version + */ + public static UserPO updateUserPOWithVersion(UserPO oldUserPO, UserEntity newUser) { + Long lastVersion = oldUserPO.getLastVersion(); + // Will set the version to the last version + 1 when having some fields need be multiple version + Long nextVersion = lastVersion; + try { + return UserPO.builder() + .withUserId(oldUserPO.getUserId()) + .withUserName(newUser.name()) + .withMetalakeId(oldUserPO.getMetalakeId()) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(newUser.auditInfo())) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Convert {@link UserPO} to {@link UserEntity} + * + * @param userPO UserPo object to be converted + * @param rolePOs list of rolePO + * @param namespace Namespace object to be associated with the user + * @return UserEntity object from UserPO object + */ + public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespace namespace) { + try { + List roleNames = + rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); + List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); + return UserEntity.builder() + .withId(userPO.getUserId()) + .withName(userPO.getUserName()) + .withRoleNames(roleNames.isEmpty() ? null : roleNames) + .withRoleIds(roleIds.isEmpty() ? null : roleIds) + .withNamespace(namespace) + .withAuditInfo( + JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to deserialize json object:", e); + } + } + + /** + * Initialize UserRoleRelPO + * + * @param userEntity UserEntity object + * @param roleIds list of role ids + * @return UserRoleRelPO object with version initialized + */ + public static List initializeUserRoleRelsPOWithVersion( + UserEntity userEntity, List roleIds) { + try { + List userRoleRelPOs = Lists.newArrayList(); + for (Long roleId : roleIds) { + UserRoleRelPO roleRelPO = + UserRoleRelPO.builder() + .withUserId(userEntity.id()) + .withRoleId(roleId) + .withAuditInfo( + JsonUtils.anyFieldMapper().writeValueAsString(userEntity.auditInfo())) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + userRoleRelPOs.add(roleRelPO); + } + return userRoleRelPOs; + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Initialize RolePO + * + * @param roleEntity RoleEntity object + * @return RolePO object with version initialized + */ + public static RolePO initializeRolePOWithVersion(RoleEntity roleEntity, RolePO.Builder builder) { + try { + return builder + .withRoleId(roleEntity.id()) + .withRoleName(roleEntity.name()) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(roleEntity.properties())) + .withSecurableObject(roleEntity.securableObject().toString()) + .withPrivileges( + roleEntity.privileges().stream() + .map(privilege -> privilege.name().toString()) + .collect(Collectors.joining(","))) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(roleEntity.auditInfo())) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 3bd7cf8a70d..c5dcae5e85d 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -28,6 +28,7 @@ import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.authorization.AuthorizationUtils; import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.exceptions.AlreadyExistsException; @@ -229,6 +230,8 @@ void testRestart(String type) throws IOException { Namespace.of("metalake", "catalog", "schema1"), "topic1", auditInfo); + UserEntity user1 = + createUser(RandomIdGenerator.INSTANCE.nextId(), "metalake", "user1", auditInfo); // Store all entities store.put(metalake); @@ -238,6 +241,7 @@ void testRestart(String type) throws IOException { store.put(table1); store.put(fileset1); store.put(topic1); + store.put(user1); Assertions.assertDoesNotThrow( () -> @@ -273,6 +277,13 @@ void testRestart(String type) throws IOException { NameIdentifier.of("metalake", "catalog", "schema1", "topic1"), Entity.EntityType.TOPIC, TopicEntity.class)); + + Assertions.assertDoesNotThrow( + () -> + store.get( + AuthorizationUtils.ofUser("metalake", "user1"), + Entity.EntityType.USER, + UserEntity.class)); } // It will automatically close the store we create before, then we reopen the entity store @@ -313,6 +324,12 @@ void testRestart(String type) throws IOException { NameIdentifier.of("metalake", "catalog", "schema1", "topic1"), Entity.EntityType.TOPIC, TopicEntity.class)); + Assertions.assertDoesNotThrow( + () -> + store.get( + AuthorizationUtils.ofUser("metalake", "user1"), + Entity.EntityType.USER, + UserEntity.class)); destroy(type); } } @@ -439,9 +456,9 @@ public void testAuthorizationEntityDelete(String type) throws IOException { BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); store.put(metalake); - UserEntity oneUser = createUser("metalake", "oneUser", auditInfo); + UserEntity oneUser = createUser(1L, "metalake", "oneUser", auditInfo); store.put(oneUser); - UserEntity anotherUser = createUser("metalake", "anotherUser", auditInfo); + UserEntity anotherUser = createUser(2L, "metalake", "anotherUser", auditInfo); store.put(anotherUser); GroupEntity oneGroup = createGroup("metalake", "oneGroup", auditInfo); store.put(oneGroup); @@ -510,6 +527,8 @@ void testEntityDelete(String type) throws IOException { TopicEntity topic1InSchema2 = createTopicEntity( 2L, Namespace.of("metalake", "catalog", "schema2"), "topic1", auditInfo); + UserEntity user1 = createUser(1L, "metalake", "user1", auditInfo); + UserEntity user2 = createUser(2L, "metalake", "user2", auditInfo); // Store all entities store.put(metalake); @@ -523,6 +542,8 @@ void testEntityDelete(String type) throws IOException { store.put(fileset1InSchema2); store.put(topic1); store.put(topic1InSchema2); + store.put(user1); + store.put(user2); validateAllEntityExist( metalake, @@ -536,7 +557,11 @@ void testEntityDelete(String type) throws IOException { fileset1, fileset1InSchema2, topic1, - topic1InSchema2); + topic1InSchema2, + user1, + user2); + + validateDeleteUser(store, user1); validateDeleteTable(store, schema2, table1, table1InSchema2); @@ -558,7 +583,7 @@ void testEntityDelete(String type) throws IOException { topic1, topic1InSchema2); - validateDeleteMetalake(store, metalake, catalogCopy); + validateDeleteMetalake(store, metalake, catalogCopy, user2); // Store all entities again // metalake @@ -643,6 +668,9 @@ void testEntityDelete(String type) throws IOException { topic1InSchema2.name(), topic1InSchema2.auditInfo()); store.put(topic1InSchema2New); + UserEntity userNew = + createUser(RandomIdGenerator.INSTANCE.nextId(), "metalake", "userNew", auditInfo); + store.put(userNew); validateDeleteTableCascade(store, table1New); @@ -654,7 +682,7 @@ void testEntityDelete(String type) throws IOException { validateDeleteCatalogCascade(store, catalogNew, schema2New); - validateDeleteMetalakeCascade(store, metalakeNew, catalogNew, schema2New); + validateDeleteMetalakeCascade(store, metalakeNew, catalogNew, schema2New, userNew); destroy(type); } @@ -1153,14 +1181,14 @@ public static TopicEntity createTopicEntity( .build(); } - private static UserEntity createUser(String metalake, String name, AuditInfo auditInfo) { + private static UserEntity createUser(Long id, String metalake, String name, AuditInfo auditInfo) { return UserEntity.builder() - .withId(1L) + .withId(id) .withNamespace( Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList()) + .withRoleNames(null) .build(); } @@ -1247,8 +1275,14 @@ private void validateDeleteTopic( } private void validateDeleteMetalakeCascade( - EntityStore store, BaseMetalake metalake, CatalogEntity catalog, SchemaEntity schema2) + EntityStore store, + BaseMetalake metalake, + CatalogEntity catalog, + SchemaEntity schema2, + UserEntity userNew) throws IOException { + Assertions.assertTrue(store.exists(userNew.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertTrue( store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, true)); @@ -1256,6 +1290,7 @@ private void validateDeleteMetalakeCascade( Assertions.assertFalse(store.exists(catalog.nameIdentifier(), Entity.EntityType.CATALOG)); Assertions.assertFalse(store.exists(schema2.nameIdentifier(), Entity.EntityType.SCHEMA)); Assertions.assertFalse(store.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); + Assertions.assertFalse(store.exists(userNew.nameIdentifier(), Entity.EntityType.USER)); } private void validateDeleteCatalogCascade( @@ -1334,8 +1369,11 @@ private void validateDeleteSchemaCascade( } private static void validateDeleteMetalake( - EntityStore store, BaseMetalake metalake, CatalogEntity catalogCopy) throws IOException { + EntityStore store, BaseMetalake metalake, CatalogEntity catalogCopy, UserEntity user2) + throws IOException { // Now delete catalog 'catalogCopy' and metalake + Assertions.assertTrue(store.exists(user2.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertThrowsExactly( NonEmptyEntityException.class, () -> store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); @@ -1344,6 +1382,7 @@ private static void validateDeleteMetalake( store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE); Assertions.assertFalse(store.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); + Assertions.assertFalse(store.exists(user2.nameIdentifier(), Entity.EntityType.USER)); } private static void validateDeleteCatalog( @@ -1454,6 +1493,17 @@ private static void validateDeleteSchema( topic1New, store.get(topic1.nameIdentifier(), Entity.EntityType.TOPIC, TopicEntity.class)); } + private void validateDeleteUser(EntityStore store, UserEntity user1) throws IOException { + Assertions.assertTrue(store.exists(user1.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertTrue(store.delete(user1.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertFalse(store.exists(user1.nameIdentifier(), Entity.EntityType.USER)); + + UserEntity user = + createUser(RandomIdGenerator.INSTANCE.nextId(), "metalake", "user1", user1.auditInfo()); + store.put(user); + Assertions.assertTrue(store.exists(user.nameIdentifier(), Entity.EntityType.USER)); + } + private void validateDeleteTable( EntityStore store, SchemaEntity schema2, TableEntity table1, TableEntity table1InSchema2) throws IOException { @@ -1490,7 +1540,9 @@ private static void validateAllEntityExist( FilesetEntity fileset1, FilesetEntity fileset1InSchema2, TopicEntity topic1, - TopicEntity topic1InSchema2) + TopicEntity topic1InSchema2, + UserEntity user1, + UserEntity user2) throws IOException { // Now try to get Assertions.assertEquals( @@ -1523,6 +1575,10 @@ private static void validateAllEntityExist( Assertions.assertEquals( topic1InSchema2, store.get(topic1InSchema2.nameIdentifier(), Entity.EntityType.TOPIC, TopicEntity.class)); + Assertions.assertEquals( + user1, store.get(user1.nameIdentifier(), Entity.EntityType.USER, UserEntity.class)); + Assertions.assertEquals( + user2, store.get(user2.nameIdentifier(), Entity.EntityType.USER, UserEntity.class)); } private void validateDeletedFileset(EntityStore store) throws IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java index 7f6d6c5172b..9ec63ed5c07 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java @@ -24,6 +24,7 @@ import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; +import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.Maps; import java.io.IOException; @@ -197,6 +198,15 @@ public void testEntityStoreAndRetrieve() throws Exception { .withAuditInfo(auditInfo) .build(); + UserEntity userEntity = + UserEntity.builder() + .withId(1L) + .withName("user") + .withNamespace(Namespace.of("metalake", "catalog", "db")) + .withAuditInfo(auditInfo) + .withRoleNames(null) + .build(); + InMemoryEntityStore store = new InMemoryEntityStore(); store.initialize(Mockito.mock(Config.class)); store.setSerDe(Mockito.mock(EntitySerDe.class)); @@ -206,6 +216,7 @@ public void testEntityStoreAndRetrieve() throws Exception { store.put(schemaEntity); store.put(tableEntity); store.put(filesetEntity); + store.put(userEntity); Metalake retrievedMetalake = store.get(metalake.nameIdentifier(), EntityType.METALAKE, BaseMetalake.class); @@ -227,6 +238,10 @@ public void testEntityStoreAndRetrieve() throws Exception { store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class); Assertions.assertEquals(filesetEntity, retrievedFileset); + UserEntity retrievedUser = + store.get(userEntity.nameIdentifier(), EntityType.USER, UserEntity.class); + Assertions.assertEquals(userEntity, retrievedUser); + store.delete(metalake.nameIdentifier(), EntityType.METALAKE); NameIdentifier id = metalake.nameIdentifier(); Assertions.assertThrows( diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 965e6043678..4b4a4f31d1c 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -19,19 +19,25 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.authorization.Privilege; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.RandomIdGenerator; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -466,4 +472,41 @@ public static TopicEntity createTopicEntity( .withAuditInfo(auditInfo) .build(); } + + public static UserEntity createUserEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return UserEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withRoleNames(null) + .withRoleIds(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static UserEntity createUserEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo, List roleNames) { + return UserEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withRoleNames(roleNames) + .withRoleIds(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static RoleEntity createRoleEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return RoleEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withProperties(null) + .withAuditInfo(auditInfo) + .withSecurableObject(SecurableObjects.ofAllCatalogs()) + .withPrivileges(Lists.newArrayList(Privileges.fromName(Privilege.Name.LOAD_CATALOG))) + .build(); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java new file mode 100644 index 00000000000..9a509e0dcc8 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java @@ -0,0 +1,604 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.authorization.AuthorizationUtils; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.meta.UserEntity; +import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.TestJDBCBackend; +import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; +import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.time.Instant; +import java.util.List; +import java.util.function.Function; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TestUserMetaService extends TestJDBCBackend { + + String metalakeName = "metalake"; + + @Test + void getUserByIdentifier() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // get not exist user + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + userMetaService.getUserByIdentifier(AuthorizationUtils.ofUser(metalakeName, "user1"))); + + // get user + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo); + userMetaService.insertUser(user1, false); + Assertions.assertEquals(user1, userMetaService.getUserByIdentifier(user1.nameIdentifier())); + + // get user with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + UserEntity user2 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo, + Lists.newArrayList("role1", "role2")); + userMetaService.insertUser(user2, false); + UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); + Assertions.assertEquals(user2.name(), actualUser.name()); + Assertions.assertEquals( + Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + } + + @Test + void insertUser() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // insert user + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user1.nameIdentifier())); + Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user1, false)); + Assertions.assertEquals(user1, userMetaService.getUserByIdentifier(user1.nameIdentifier())); + + // insert duplicate user + UserEntity user1Exist = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo); + Assertions.assertThrows( + AlreadyExistsException.class, () -> userMetaService.insertUser(user1Exist, false)); + + // insert overwrite + UserEntity user1Overwrite = + createUserEntity( + user1.id(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1Overwrite", + auditInfo); + Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user1Overwrite, true)); + Assertions.assertEquals( + "user1Overwrite", + userMetaService.getUserByIdentifier(user1Overwrite.nameIdentifier()).name()); + Assertions.assertEquals( + user1Overwrite, userMetaService.getUserByIdentifier(user1Overwrite.nameIdentifier())); + + // insert user with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + UserEntity user2 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo, + Lists.newArrayList("role1", "role2")); + Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2, false)); + UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); + Assertions.assertEquals(user2.name(), actualUser.name()); + Assertions.assertEquals( + Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + + // insert duplicate user with roles + UserEntity user2Exist = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo); + Assertions.assertThrows( + AlreadyExistsException.class, () -> userMetaService.insertUser(user2Exist, false)); + + // insert overwrite user with roles + UserEntity user2Overwrite = + createUserEntity( + user1.id(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2Overwrite", + auditInfo); + Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2Overwrite, true)); + Assertions.assertEquals( + "user2Overwrite", + userMetaService.getUserByIdentifier(user2Overwrite.nameIdentifier()).name()); + Assertions.assertEquals( + user2Overwrite, userMetaService.getUserByIdentifier(user2Overwrite.nameIdentifier())); + } + + @Test + void deleteUser() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // delete user + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user1.nameIdentifier())); + Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user1, false)); + Assertions.assertEquals(user1, userMetaService.getUserByIdentifier(user1.nameIdentifier())); + Assertions.assertTrue(userMetaService.deleteUser(user1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user1.nameIdentifier())); + + // delete user with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + UserEntity user2 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo, + Lists.newArrayList("role1", "role2")); + userMetaService.insertUser(user2, false); + List rolePOs = + SessionUtils.doWithCommitAndFetchResult( + RoleMetaMapper.class, mapper -> mapper.listRolesByUserId(user2.id())); + Assertions.assertEquals(2, rolePOs.size()); + UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); + Assertions.assertEquals(user2.name(), actualUser.name()); + Assertions.assertEquals( + Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + + Assertions.assertTrue(userMetaService.deleteUser(user2.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user2.nameIdentifier())); + rolePOs = + SessionUtils.doWithCommitAndFetchResult( + RoleMetaMapper.class, mapper -> mapper.listRolesByUserId(user2.id())); + Assertions.assertEquals(0, rolePOs.size()); + } + + @Test + void updateUser() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo, + Lists.newArrayList("role1", "role2")); + userMetaService.insertUser(user1, false); + UserEntity actualUser = userMetaService.getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(user1.name(), actualUser.name()); + Assertions.assertEquals( + Sets.newHashSet(user1.roleNames()), Sets.newHashSet(actualUser.roleNames())); + + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role3, false); + + // update user (grant) + Function grantUpdater = + user -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(user.auditInfo().creator()) + .withCreateTime(user.auditInfo().createTime()) + .withLastModifier("grantUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(user.roleNames()); + List roleIds = Lists.newArrayList(user.roleIds()); + roleNames.add(role3.name()); + roleIds.add(role3.id()); + + return UserEntity.builder() + .withNamespace(user.namespace()) + .withId(user.id()) + .withName(user.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + + Assertions.assertNotNull(userMetaService.updateUser(user1.nameIdentifier(), grantUpdater)); + UserEntity grantUser = + UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(user1.id(), grantUser.id()); + Assertions.assertEquals(user1.name(), grantUser.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantUser.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantUser.roleIds())); + Assertions.assertEquals("creator", grantUser.auditInfo().creator()); + Assertions.assertEquals("grantUser", grantUser.auditInfo().lastModifier()); + + // update user (revoke) + Function revokeUpdater = + user -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(user.auditInfo().creator()) + .withCreateTime(user.auditInfo().createTime()) + .withLastModifier("revokeUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(user.roleNames()); + List roleIds = Lists.newArrayList(user.roleIds()); + roleIds.remove(roleNames.indexOf("role2")); + roleNames.remove("role2"); + + return UserEntity.builder() + .withNamespace(user.namespace()) + .withId(user.id()) + .withName(user.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + + Assertions.assertNotNull(userMetaService.updateUser(user1.nameIdentifier(), revokeUpdater)); + UserEntity revokeUser = + UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(user1.id(), revokeUser.id()); + Assertions.assertEquals(user1.name(), revokeUser.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeUser.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeUser.roleIds())); + Assertions.assertEquals("creator", revokeUser.auditInfo().creator()); + Assertions.assertEquals("revokeUser", revokeUser.auditInfo().lastModifier()); + + RoleEntity role4 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role4", + auditInfo); + roleMetaService.insertRole(role4, false); + + // update user (grant & revoke) + Function grantRevokeUpdater = + user -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(user.auditInfo().creator()) + .withCreateTime(user.auditInfo().createTime()) + .withLastModifier("grantRevokeUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(user.roleNames()); + List roleIds = Lists.newArrayList(user.roleIds()); + roleIds.remove(roleNames.indexOf("role3")); + roleNames.remove("role3"); + roleIds.add(role4.id()); + roleNames.add(role4.name()); + + return UserEntity.builder() + .withNamespace(user.namespace()) + .withId(user.id()) + .withName(user.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + Assertions.assertNotNull( + userMetaService.updateUser(user1.nameIdentifier(), grantRevokeUpdater)); + UserEntity grantRevokeUser = + UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(user1.id(), grantRevokeUser.id()); + Assertions.assertEquals(user1.name(), grantRevokeUser.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeUser.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeUser.roleIds())); + Assertions.assertEquals("creator", grantRevokeUser.auditInfo().creator()); + Assertions.assertEquals("grantRevokeUser", grantRevokeUser.auditInfo().lastModifier()); + + Function noUpdater = + user -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(user.auditInfo().creator()) + .withCreateTime(user.auditInfo().createTime()) + .withLastModifier("noUpdateUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(user.roleNames()); + List roleIds = Lists.newArrayList(user.roleIds()); + + return UserEntity.builder() + .withNamespace(user.namespace()) + .withId(user.id()) + .withName(user.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + Assertions.assertNotNull(userMetaService.updateUser(user1.nameIdentifier(), noUpdater)); + UserEntity noUpdaterUser = + UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(user1.id(), noUpdaterUser.id()); + Assertions.assertEquals(user1.name(), noUpdaterUser.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterUser.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterUser.roleIds())); + Assertions.assertEquals("creator", noUpdaterUser.auditInfo().creator()); + Assertions.assertEquals("grantRevokeUser", noUpdaterUser.auditInfo().lastModifier()); + } + + @Test + void deleteMetalake() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + roleMetaService.insertRole(role3, false); + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo, + Lists.newArrayList("role1", "role2")); + UserEntity user2 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo, + Lists.newArrayList("role3")); + userMetaService.insertUser(user1, false); + userMetaService.insertUser(user2, false); + + Assertions.assertEquals( + user1.name(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).name()); + Assertions.assertEquals(2, roleMetaService.listRolesByUserId(user1.id()).size()); + Assertions.assertEquals( + user2.name(), userMetaService.getUserByIdentifier(user2.nameIdentifier()).name()); + Assertions.assertEquals(1, roleMetaService.listRolesByUserId(user2.id()).size()); + + // delete metalake without cascade + Assertions.assertTrue( + MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), false)); + + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user2.nameIdentifier())); + Assertions.assertEquals(0, roleMetaService.listRolesByUserId(user1.id()).size()); + Assertions.assertEquals(0, roleMetaService.listRolesByUserId(user2.id()).size()); + } + + @Test + void deleteMetalakeCascade() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + UserMetaService userMetaService = UserMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + roleMetaService.insertRole(role3, false); + UserEntity user1 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user1", + auditInfo, + Lists.newArrayList("role1", "role2")); + UserEntity user2 = + createUserEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofUserNamespace(metalakeName), + "user2", + auditInfo, + Lists.newArrayList("role3")); + userMetaService.insertUser(user1, false); + userMetaService.insertUser(user2, false); + + Assertions.assertEquals( + user1.name(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).name()); + Assertions.assertEquals(2, roleMetaService.listRolesByUserId(user1.id()).size()); + Assertions.assertEquals( + user2.name(), userMetaService.getUserByIdentifier(user2.nameIdentifier()).name()); + Assertions.assertEquals(1, roleMetaService.listRolesByUserId(user2.id()).size()); + + // delete metalake with cascade + Assertions.assertTrue( + MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), true)); + + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> userMetaService.getUserByIdentifier(user2.nameIdentifier())); + Assertions.assertEquals(0, roleMetaService.listRolesByUserId(user1.id()).size()); + Assertions.assertEquals(0, roleMetaService.listRolesByUserId(user2.id()).size()); + } +} diff --git a/core/src/test/resources/h2/schema-h2.sql b/core/src/test/resources/h2/schema-h2.sql index 6e2796c3164..0b2c269b9bc 100644 --- a/core/src/test/resources/h2/schema-h2.sql +++ b/core/src/test/resources/h2/schema-h2.sql @@ -124,4 +124,44 @@ CREATE TABLE IF NOT EXISTS `topic_meta` ( -- Aliases are used here, and indexes with the same name in H2 can only be created once. KEY idx_tvmid (metalake_id), KEY idx_tvcid (catalog_id) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `user_meta` ( + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `user_name` VARCHAR(128) NOT NULL COMMENT 'username', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', + PRIMARY KEY (`user_id`), + CONSTRAINT `uk_mid_us_del` UNIQUE (`metalake_id`, `user_name`, `deleted_at`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `role_meta` ( + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `securable_object` VARCHAR(256) NOT NULL COMMENT 'securable object', + `privileges` VARCHAR(64) NOT NULL COMMENT 'securable privileges', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', + PRIMARY KEY (`role_id`), + CONSTRAINT `uk_mid_rn_del` UNIQUE (`metalake_id`, `role_name`, `deleted_at`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `user_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + CONSTRAINT `uk_ui_ri_del` UNIQUE (`user_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`role_id`) ) ENGINE=InnoDB; \ No newline at end of file diff --git a/scripts/mysql/schema-0.5.0-mysql.sql b/scripts/mysql/schema-0.5.0-mysql.sql index f3a1e323d10..f78942a65c9 100644 --- a/scripts/mysql/schema-0.5.0-mysql.sql +++ b/scripts/mysql/schema-0.5.0-mysql.sql @@ -116,4 +116,44 @@ CREATE TABLE IF NOT EXISTS `topic_meta` ( UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `topic_name`, `deleted_at`), KEY `idx_mid` (`metalake_id`), KEY `idx_cid` (`catalog_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'topic metadata'; \ No newline at end of file + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'topic metadata'; + +CREATE TABLE IF NOT EXISTS `user_meta` ( + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `user_name` VARCHAR(128) NOT NULL COMMENT 'username', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', + PRIMARY KEY (`user_id`), + UNIQUE KEY `uk_mid_us_del` (`metalake_id`, `user_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user metadata'; + +CREATE TABLE IF NOT EXISTS `role_meta` ( + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `securable_object` VARCHAR(256) NOT NULL COMMENT 'securable object', + `privileges` VARCHAR(64) NOT NULL COMMENT 'securable privileges', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', + PRIMARY KEY (`role_id`), + UNIQUE KEY `uk_mid_rn_del` (`metalake_id`, `role_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'role metadata'; + +CREATE TABLE IF NOT EXISTS `user_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_ui_ri_del` (`user_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`role_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; \ No newline at end of file From a1dd2797a5f1763caea74606dbdc8560a605cdb8 Mon Sep 17 00:00:00 2001 From: YongXing Date: Sat, 20 Apr 2024 16:01:57 +0800 Subject: [PATCH 080/106] [#2385] feat(core): Add the Relational Garbage Collector (#3016) ### What changes were proposed in this pull request? - Add the JDBC Backend Garbage Collector ### Why are the changes needed? Fix: #2385 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? UT --------- Co-authored-by: YxAc --- .../com/datastrato/gravitino/Configs.java | 1 + .../storage/relational/JDBCBackend.java | 80 ++++++ .../storage/relational/RelationalBackend.java | 22 +- .../relational/RelationalEntityStore.java | 4 + .../RelationalGarbageCollector.java | 97 +++++++ .../relational/mapper/CatalogMetaMapper.java | 8 + .../relational/mapper/FilesetMetaMapper.java | 8 + .../mapper/FilesetVersionMapper.java | 31 +++ .../relational/mapper/MetalakeMetaMapper.java | 8 + .../relational/mapper/SchemaMetaMapper.java | 8 + .../relational/mapper/TableMetaMapper.java | 8 + .../relational/mapper/TopicMetaMapper.java | 8 + .../service/CatalogMetaService.java | 8 + .../service/FilesetMetaService.java | 53 ++++ .../service/MetalakeMetaService.java | 8 + .../relational/service/SchemaMetaService.java | 8 + .../relational/service/TableMetaService.java | 8 + .../relational/service/TopicMetaService.java | 8 + .../gravitino/storage/TestEntityStorage.java | 4 + .../storage/relational/TestJDBCBackend.java | 261 +++++++++++++++++- 20 files changed, 639 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 4d361f43436..f92faee88da 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -57,6 +57,7 @@ public interface Configs { String DEFAULT_KV_ROCKSDB_BACKEND_PATH = String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "rocksdb"); + int GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT = 100; long MAX_NODE_IN_MEMORY = 100000L; long MIN_NODE_IN_MEMORY = 1000L; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 9d884fa8087..bcd56709e66 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -5,6 +5,8 @@ package com.datastrato.gravitino.storage.relational; +import static com.datastrato.gravitino.Configs.GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT; + import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; @@ -36,6 +38,8 @@ import java.io.IOException; import java.util.List; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * {@link JDBCBackend} is a jdbc implementation of {@link RelationalBackend} interface. You can use @@ -45,6 +49,8 @@ */ public class JDBCBackend implements RelationalBackend { + private static final Logger LOG = LoggerFactory.getLogger(JDBCBackend.class); + /** Initialize the jdbc backend instance. */ @Override public void initialize(Config config) { @@ -181,6 +187,80 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } } + @Override + public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLine) { + LOG.info( + "Try to physically delete {} legacy data that has been marked deleted before {}", + entityType, + legacyTimeLine); + + switch (entityType) { + case METALAKE: + return MetalakeMetaService.getInstance() + .deleteMetalakeMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case CATALOG: + return CatalogMetaService.getInstance() + .deleteCatalogMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case SCHEMA: + return SchemaMetaService.getInstance() + .deleteSchemaMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case TABLE: + return TableMetaService.getInstance() + .deleteTableMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case FILESET: + return FilesetMetaService.getInstance() + .deleteFilesetAndVersionMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case TOPIC: + return TopicMetaService.getInstance() + .deleteTopicMetasByLegacyTimeLine( + legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + + case COLUMN: + case USER: + case GROUP: + case AUDIT: + case ROLE: + return 0; + // TODO: Implement hard delete logic for these entity types. + + default: + throw new IllegalArgumentException( + "Unsupported entity type when collectAndRemoveLegacyData: " + entityType); + } + } + + @Override + public int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount) { + switch (entityType) { + case METALAKE: + case CATALOG: + case SCHEMA: + case TABLE: + case COLUMN: + case TOPIC: + case USER: + case GROUP: + case AUDIT: + case ROLE: + // These entity types have not implemented multi-versions, so we can skip. + return 0; + + case FILESET: + return FilesetMetaService.getInstance() + .deleteFilesetVersionsByRetentionCount( + versionRetentionCount, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + + default: + throw new IllegalArgumentException( + "Unsupported entity type when collectAndRemoveOldVersionData: " + entityType); + } + } + @Override public void close() throws IOException { SqlSessionFactoryHelper.getInstance().close(); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index ea9506ed147..17e77533e42 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -84,7 +84,7 @@ E get(NameIdentifier ident, Entity.EntityType throws IOException; /** - * Deletes the entity associated with the identifier and the entity type. + * Soft deletes the entity associated with the identifier and the entity type. * * @param ident The identifier of the entity. * @param entityType The type of the entity. @@ -92,4 +92,24 @@ E get(NameIdentifier ident, Entity.EntityType * @return True, if the entity was successfully deleted, else false. */ boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade); + + /** + * Permanently deletes the legacy data that has been marked as deleted before the given legacy + * timeline. + * + * @param entityType The type of the entity. + * @param legacyTimeLine The time before which the data has been marked as deleted. + * @return The count of the deleted data. + */ + int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLine); + + /** + * Soft deletes the old version data that is older than or equal to the given version retention + * count. + * + * @param entityType The type of the entity. + * @param versionRetentionCount The count of versions to retain. + * @return The count of the deleted data. + */ + int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index 8427e7484af..8e6a3d2b661 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -36,10 +36,13 @@ public class RelationalEntityStore implements EntityStore { ImmutableMap.of( Configs.DEFAULT_ENTITY_RELATIONAL_STORE, JDBCBackend.class.getCanonicalName()); private RelationalBackend backend; + private RelationalGarbageCollector garbageCollector; @Override public void initialize(Config config) throws RuntimeException { this.backend = createRelationalEntityBackend(config); + this.garbageCollector = new RelationalGarbageCollector(backend, config); + this.garbageCollector.start(); } private static RelationalBackend createRelationalEntityBackend(Config config) { @@ -109,6 +112,7 @@ public R executeInTransaction(Executable executab @Override public void close() throws IOException { + garbageCollector.close(); backend.close(); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java new file mode 100644 index 00000000000..3cc1a852782 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java @@ -0,0 +1,97 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational; + +import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; +import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.google.common.annotations.VisibleForTesting; +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class RelationalGarbageCollector implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(RelationalGarbageCollector.class); + private final RelationalBackend backend; + + private final long storeDeleteAfterTimeMillis; + private final long versionRetentionCount; + + @VisibleForTesting + final ScheduledExecutorService garbageCollectorPool = + new ScheduledThreadPoolExecutor( + 2, + r -> { + Thread t = new Thread(r, "RelationalBackend-Garbage-Collector"); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.AbortPolicy()); + + public RelationalGarbageCollector(RelationalBackend backend, Config config) { + this.backend = backend; + storeDeleteAfterTimeMillis = config.get(STORE_DELETE_AFTER_TIME); + versionRetentionCount = config.get(VERSION_RETENTION_COUNT); + } + + public void start() { + long dateTimeLineMinute = storeDeleteAfterTimeMillis / 1000 / 60; + + // We will collect garbage every 10 minutes at least. If the dateTimeLineMinute is larger than + // 100 minutes, we would collect garbage every dateTimeLineMinute/10 minutes. + long frequency = Math.max(dateTimeLineMinute / 10, 10); + garbageCollectorPool.scheduleAtFixedRate(this::collectAndClean, 5, frequency, TimeUnit.MINUTES); + } + + private void collectAndClean() { + long threadId = Thread.currentThread().getId(); + LOG.info("Thread {} start to collect garbage...", threadId); + + try { + LOG.info("Start to collect and delete legacy data by thread {}", threadId); + long legacyTimeLine = System.currentTimeMillis() - storeDeleteAfterTimeMillis; + for (Entity.EntityType entityType : Entity.EntityType.values()) { + long deletedCount = Long.MAX_VALUE; + while (deletedCount > 0) { + deletedCount = backend.hardDeleteLegacyData(entityType, legacyTimeLine); + } + } + + LOG.info("Start to collect and delete old version data by thread {}", threadId); + for (Entity.EntityType entityType : Entity.EntityType.values()) { + long deletedCount = Long.MAX_VALUE; + while (deletedCount > 0) { + deletedCount = backend.deleteOldVersionData(entityType, versionRetentionCount); + } + } + } catch (Exception e) { + LOG.error("Thread {} failed to collect and clean garbage.", threadId, e); + } finally { + LOG.info("Thread {} finish to collect garbage.", threadId); + } + } + + @Override + public void close() throws IOException { + this.garbageCollectorPool.shutdown(); + try { + if (!this.garbageCollectorPool.awaitTermination(5, TimeUnit.SECONDS)) { + this.garbageCollectorPool.shutdownNow(); + } + } catch (InterruptedException ex) { + this.garbageCollectorPool.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java index 038b4256f0b..5e40deb39b1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.storage.relational.po.CatalogPO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -147,4 +148,11 @@ Integer updateCatalogMeta( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Delete( + "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteCatalogMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java index 885a8d89908..ee87daa43fd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.storage.relational.po.FilesetPO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Result; @@ -214,4 +215,11 @@ Integer updateFilesetMeta( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") Integer softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId); + + @Delete( + "DELETE FROM " + + META_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteFilesetMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java index af08416ea09..d5d9053c330 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java @@ -6,8 +6,12 @@ package com.datastrato.gravitino.storage.relational.mapper; import com.datastrato.gravitino.storage.relational.po.FilesetVersionPO; +import java.util.List; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; /** @@ -97,4 +101,31 @@ void insertFilesetVersionOnDuplicateKeyUpdate( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") Integer softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId); + + @Delete( + "DELETE FROM " + + VERSION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteFilesetVersionsByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + @Select( + "SELECT fileset_id as filesetId," + + " Max(version) as version" + + " FROM " + + VERSION_TABLE_NAME + + " WHERE version > #{versionRetentionCount} AND deleted_at = 0" + + " GROUP BY fileset_id") + List> selectFilesetVersionsByRetentionCount( + @Param("versionRetentionCount") Long versionRetentionCount); + + @Update( + "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}") + Integer softDeleteFilesetVersionsByRetentionLine( + @Param("filesetId") Long filesetId, + @Param("versionRetentionLine") long versionRetentionLine, + @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index d1cb2d1ea73..29b87d64b50 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.storage.relational.po.MetalakePO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -125,4 +126,11 @@ Integer updateMetalakeMeta( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Delete( + "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteMetalakeMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java index 7f36478cf0a..8a27a440ab4 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.storage.relational.po.SchemaPO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -149,4 +150,11 @@ Integer updateSchemaMeta( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId); + + @Delete( + "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteSchemaMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java index 331847f8cdc..aeeaa1eac14 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.storage.relational.po.TablePO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -151,4 +152,11 @@ Integer updateTableMeta( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId); + + @Delete( + "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteTableMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java index 5db879f67e8..06f0d611db0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.storage.relational.po.TopicPO; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -152,4 +153,11 @@ Long selectTopicIdBySchemaIdAndName( + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId); + + @Delete( + "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteTopicMetasByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index a075dbf8d39..9264fac3464 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -209,4 +209,12 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { return true; } + + public int deleteCatalogMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + return SessionUtils.doWithCommitAndFetchResult( + CatalogMetaMapper.class, + mapper -> { + return mapper.deleteCatalogMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java index eb8b1924ac2..ad041aee418 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The service class for fileset metadata and version info. It provides the basic database @@ -29,6 +32,8 @@ public class FilesetMetaService { private static final FilesetMetaService INSTANCE = new FilesetMetaService(); + private static final Logger LOG = LoggerFactory.getLogger(FilesetMetaService.class); + public static FilesetMetaService getInstance() { return INSTANCE; } @@ -213,6 +218,54 @@ public boolean deleteFileset(NameIdentifier identifier) { return true; } + public int deleteFilesetAndVersionMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + int filesetDeletedCount = + SessionUtils.doWithCommitAndFetchResult( + FilesetMetaMapper.class, + mapper -> { + return mapper.deleteFilesetMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + int filesetVersionDeletedCount = + SessionUtils.doWithCommitAndFetchResult( + FilesetVersionMapper.class, + mapper -> { + return mapper.deleteFilesetVersionsByLegacyTimeLine(legacyTimeLine, limit); + }); + return filesetDeletedCount + filesetVersionDeletedCount; + } + + public int deleteFilesetVersionsByRetentionCount(Long versionRetentionCount, int limit) { + // get the current version of all filesets. + List> filesetCurVersions = + SessionUtils.getWithoutCommit( + FilesetVersionMapper.class, + mapper -> mapper.selectFilesetVersionsByRetentionCount(versionRetentionCount)); + + // soft delete old versions that are older than or equal to (currentVersion - + // versionRetentionCount). + int totalDeletedCount = 0; + for (ImmutablePair filesetCurVersion : filesetCurVersions) { + long versionRetentionLine = filesetCurVersion.getValue() - versionRetentionCount; + int deletedCount = + SessionUtils.doWithCommitAndFetchResult( + FilesetVersionMapper.class, + mapper -> + mapper.softDeleteFilesetVersionsByRetentionLine( + filesetCurVersion.getKey(), versionRetentionLine, limit)); + totalDeletedCount += deletedCount; + + // log the deletion by current fileset version. + LOG.info( + "Soft delete filesetVersions count: {} which versions are older than or equal to" + + " versionRetentionLine: {}, the current filesetId and version is: <{}, {}>.", + deletedCount, + versionRetentionLine, + filesetCurVersion.getKey(), + filesetCurVersion.getValue()); + } + return totalDeletedCount; + } + private void fillFilesetPOBuilderParentEntityId(FilesetPO.Builder builder, Namespace namespace) { Namespace.checkFileset(namespace); Long parentEntityId = null; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index c35dd23c508..066f4178332 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -206,4 +206,12 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { } return true; } + + public int deleteMetalakeMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + return SessionUtils.doWithCommitAndFetchResult( + MetalakeMetaMapper.class, + mapper -> { + return mapper.deleteMetalakeMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java index c1b8ba490d2..a5ae1063880 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java @@ -215,6 +215,14 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { return true; } + public int deleteSchemaMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + return SessionUtils.doWithCommitAndFetchResult( + SchemaMetaMapper.class, + mapper -> { + return mapper.deleteSchemaMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + } + private void fillSchemaPOBuilderParentEntityId(SchemaPO.Builder builder, Namespace namespace) { Namespace.checkSchema(namespace); Long parentEntityId = null; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java index fc6a03db757..f71bbc41b10 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java @@ -163,6 +163,14 @@ public boolean deleteTable(NameIdentifier identifier) { return true; } + public int deleteTableMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + return SessionUtils.doWithCommitAndFetchResult( + TableMetaMapper.class, + mapper -> { + return mapper.deleteTableMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + } + private void fillTablePOBuilderParentEntityId(TablePO.Builder builder, Namespace namespace) { Namespace.checkTable(namespace); Long parentEntityId = null; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java index cc60e266f2d..06042a9659f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java @@ -176,6 +176,14 @@ public boolean deleteTopic(NameIdentifier identifier) { return true; } + public int deleteTopicMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { + return SessionUtils.doWithCommitAndFetchResult( + TopicMetaMapper.class, + mapper -> { + return mapper.deleteTopicMetasByLegacyTimeLine(legacyTimeLine, limit); + }); + } + private Long getTopicIdBySchemaIdAndName(Long schemaId, String topicName) { Long topicId = SessionUtils.getWithoutCommit( diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index c5dcae5e85d..8ce48ab031b 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -18,6 +18,7 @@ import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; +import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Config; @@ -102,6 +103,7 @@ private void init(String type, Config config) { Assertions.assertEquals(KV_STORE_PATH, config.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)); Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + Mockito.when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); } else if (type.equals(Configs.RELATIONAL_ENTITY_STORE)) { File dir = new File(DB_DIR); if (dir.exists() || !dir.isDirectory()) { @@ -115,6 +117,8 @@ private void init(String type, Config config) { Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("123"); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + Mockito.when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); } else { throw new UnsupportedOperationException("Unsupported entity store type: " + type); } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 4b4a4f31d1c..569429829cd 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -12,7 +12,10 @@ import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Config; @@ -47,7 +50,9 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.UUID; import org.apache.commons.io.IOUtils; @@ -400,6 +405,260 @@ public void testUpdateAlreadyExistsException() { e -> createTopicEntity(topicCopy.id(), topicCopy.namespace(), "topic", auditInfo))); } + @Test + public void testMetaLifeCycleFromCreationToDeletion() throws IOException { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + // meta data creation + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "metalake", auditInfo); + backend.insert(metalake, false); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("metalake"), + "catalog", + auditInfo); + backend.insert(catalog, false); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("metalake", "catalog"), + "schema", + auditInfo); + backend.insert(schema, false); + + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofTable("metalake", "catalog", "schema"), + "table", + auditInfo); + backend.insert(table, false); + + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset", + auditInfo); + backend.insert(fileset, false); + + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "topic", + auditInfo); + backend.insert(topic, false); + + // update fileset properties and version + FilesetEntity filesetV2 = + createFilesetEntity( + fileset.id(), + Namespace.ofFileset("metalake", "catalog", "schema"), + "fileset", + auditInfo); + filesetV2.properties().put("version", "2"); + backend.update(fileset.nameIdentifier(), Entity.EntityType.FILESET, e -> filesetV2); + + // another meta data creation + BaseMetalake anotherMetaLake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), "another-metalake", auditInfo); + backend.insert(anotherMetaLake, false); + + CatalogEntity anotherCatalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofCatalog("another-metalake"), + "another-catalog", + auditInfo); + backend.insert(anotherCatalog, false); + + SchemaEntity anotherSchema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofSchema("another-metalake", "another-catalog"), + "another-schema", + auditInfo); + backend.insert(anotherSchema, false); + + FilesetEntity anotherFileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.ofFileset("another-metalake", "another-catalog", "another-schema"), + "anotherFileset", + auditInfo); + backend.insert(anotherFileset, false); + + FilesetEntity anotherFilesetV2 = + createFilesetEntity( + anotherFileset.id(), + Namespace.ofFileset("another-metalake", "another-catalog", "another-schema"), + "anotherFileset", + auditInfo); + anotherFilesetV2.properties().put("version", "2"); + backend.update( + anotherFileset.nameIdentifier(), Entity.EntityType.FILESET, e -> anotherFilesetV2); + + FilesetEntity anotherFilesetV3 = + createFilesetEntity( + anotherFileset.id(), + Namespace.ofFileset("another-metalake", "another-catalog", "another-schema"), + "anotherFileset", + auditInfo); + anotherFilesetV3.properties().put("version", "3"); + backend.update( + anotherFileset.nameIdentifier(), Entity.EntityType.FILESET, e -> anotherFilesetV3); + + // meta data list + List metaLakes = backend.list(metalake.namespace(), Entity.EntityType.METALAKE); + assertTrue(metaLakes.contains(metalake)); + + List catalogs = backend.list(catalog.namespace(), Entity.EntityType.CATALOG); + assertTrue(catalogs.contains(catalog)); + + List schemas = backend.list(schema.namespace(), Entity.EntityType.SCHEMA); + assertTrue(schemas.contains(schema)); + + List tables = backend.list(table.namespace(), Entity.EntityType.TABLE); + assertTrue(tables.contains(table)); + + List filesets = backend.list(fileset.namespace(), Entity.EntityType.FILESET); + assertFalse(filesets.contains(fileset)); + assertTrue(filesets.contains(filesetV2)); + assertEquals("2", filesets.get(filesets.indexOf(filesetV2)).properties().get("version")); + + List topics = backend.list(topic.namespace(), Entity.EntityType.TOPIC); + assertTrue(topics.contains(topic)); + + // meta data soft delete + backend.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, true); + + // check existence after soft delete + assertFalse(backend.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); + assertTrue(backend.exists(anotherMetaLake.nameIdentifier(), Entity.EntityType.METALAKE)); + + assertFalse(backend.exists(catalog.nameIdentifier(), Entity.EntityType.CATALOG)); + assertTrue(backend.exists(anotherCatalog.nameIdentifier(), Entity.EntityType.CATALOG)); + + assertFalse(backend.exists(schema.nameIdentifier(), Entity.EntityType.SCHEMA)); + assertTrue(backend.exists(anotherSchema.nameIdentifier(), Entity.EntityType.SCHEMA)); + + assertFalse(backend.exists(fileset.nameIdentifier(), Entity.EntityType.FILESET)); + assertTrue(backend.exists(anotherFileset.nameIdentifier(), Entity.EntityType.FILESET)); + + assertFalse(backend.exists(table.nameIdentifier(), Entity.EntityType.TABLE)); + assertFalse(backend.exists(topic.nameIdentifier(), Entity.EntityType.TOPIC)); + + // check legacy record after soft delete + assertTrue(legacyRecordExistsInDB(metalake.id(), Entity.EntityType.METALAKE)); + assertTrue(legacyRecordExistsInDB(catalog.id(), Entity.EntityType.CATALOG)); + assertTrue(legacyRecordExistsInDB(schema.id(), Entity.EntityType.SCHEMA)); + assertTrue(legacyRecordExistsInDB(table.id(), Entity.EntityType.TABLE)); + assertTrue(legacyRecordExistsInDB(topic.id(), Entity.EntityType.TOPIC)); + assertTrue(legacyRecordExistsInDB(fileset.id(), Entity.EntityType.FILESET)); + assertEquals(2, listFilesetVersions(fileset.id()).size()); + assertEquals(3, listFilesetVersions(anotherFileset.id()).size()); + + // meta data hard delete + for (Entity.EntityType entityType : Entity.EntityType.values()) { + backend.hardDeleteLegacyData(entityType, Instant.now().toEpochMilli() + 1000); + } + assertFalse(legacyRecordExistsInDB(metalake.id(), Entity.EntityType.METALAKE)); + assertFalse(legacyRecordExistsInDB(catalog.id(), Entity.EntityType.CATALOG)); + assertFalse(legacyRecordExistsInDB(schema.id(), Entity.EntityType.SCHEMA)); + assertFalse(legacyRecordExistsInDB(table.id(), Entity.EntityType.TABLE)); + assertFalse(legacyRecordExistsInDB(fileset.id(), Entity.EntityType.FILESET)); + assertFalse(legacyRecordExistsInDB(topic.id(), Entity.EntityType.TOPIC)); + assertEquals(0, listFilesetVersions(fileset.id()).size()); + + // soft delete for old version fileset + assertEquals(3, listFilesetVersions(anotherFileset.id()).size()); + for (Entity.EntityType entityType : Entity.EntityType.values()) { + backend.deleteOldVersionData(entityType, 1); + } + Map versionDeletedMap = listFilesetVersions(anotherFileset.id()); + assertEquals(3, versionDeletedMap.size()); + assertEquals(1, versionDeletedMap.values().stream().filter(value -> value == 0L).count()); + assertEquals(2, versionDeletedMap.values().stream().filter(value -> value != 0L).count()); + + // hard delete for old version fileset + backend.hardDeleteLegacyData(Entity.EntityType.FILESET, Instant.now().toEpochMilli() + 1000); + assertEquals(1, listFilesetVersions(anotherFileset.id()).size()); + } + + private boolean legacyRecordExistsInDB(Long id, Entity.EntityType entityType) { + String tableName; + String idColumnName; + + switch (entityType) { + case METALAKE: + tableName = "metalake_meta"; + idColumnName = "metalake_id"; + break; + case CATALOG: + tableName = "catalog_meta"; + idColumnName = "catalog_id"; + break; + case SCHEMA: + tableName = "schema_meta"; + idColumnName = "schema_id"; + break; + case TABLE: + tableName = "table_meta"; + idColumnName = "table_id"; + break; + case FILESET: + tableName = "fileset_meta"; + idColumnName = "fileset_id"; + break; + case TOPIC: + tableName = "topic_meta"; + idColumnName = "topic_id"; + break; + default: + throw new IllegalArgumentException("Unsupported entity type: " + entityType); + } + + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT * FROM %s WHERE %s = %d AND deleted_at != 0", + tableName, idColumnName, id))) { + return rs.next(); + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + } + + private Map listFilesetVersions(Long filesetId) { + Map versionDeletedTime = new HashMap<>(); + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT version, deleted_at FROM fileset_version_info WHERE fileset_id = %d", + filesetId))) { + while (rs.next()) { + versionDeletedTime.put(rs.getInt("version"), rs.getLong("deleted_at")); + } + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + return versionDeletedTime; + } + public static BaseMetalake createBaseMakeLake(Long id, String name, AuditInfo auditInfo) { return BaseMetalake.builder() .withId(id) @@ -456,7 +715,7 @@ public static FilesetEntity createFilesetEntity( .withFilesetType(Fileset.Type.MANAGED) .withStorageLocation("/tmp") .withComment("") - .withProperties(null) + .withProperties(new HashMap<>()) .withAuditInfo(auditInfo) .build(); } From 9d4695bf25fb605bbf2e381c869e931a77d4a58e Mon Sep 17 00:00:00 2001 From: mchades Date: Sat, 20 Apr 2024 16:32:32 +0800 Subject: [PATCH 081/106] [#2952] feat(core): support name spec capability (#3053) ### What changes were proposed in this pull request? - add name spec for entity name ### Why are the changes needed? Fix: #2952 ### Does this PR introduce _any_ user-facing change? yes, have more limitation in name now ### How was this patch tested? added tests --- .../catalog/FilesetOperationDispatcher.java | 5 -- .../catalog/SchemaOperationDispatcher.java | 4 -- .../catalog/TableOperationDispatcher.java | 5 -- .../catalog/TopicOperationDispatcher.java | 5 -- .../connector/capability/Capability.java | 29 +++++++++ .../TestFilesetNormalizeDispatcher.java | 29 +++++++++ .../TestSchemaNormalizeDispatcher.java | 22 +++++++ .../catalog/TestTableNormalizeDispatcher.java | 45 +++++++++++++ .../catalog/TestTopicNormalizeDispatcher.java | 25 ++++++++ .../connector/capability/TestCapability.java | 64 +++++++++++++++++++ 10 files changed, 214 insertions(+), 19 deletions(-) create mode 100644 core/src/test/java/com/datastrato/gravitino/connector/capability/TestCapability.java diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java index d462e6ac26a..30c012b1f5f 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java @@ -6,7 +6,6 @@ import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; -import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -100,10 +99,6 @@ public Fileset createFileset( Map properties) throws NoSuchSchemaException, FilesetAlreadyExistsException { NameIdentifier catalogIdent = getCatalogIdentifier(ident); - if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { - throw new IllegalArgumentException("Can't create a fileset with with reserved name `*`"); - } - doWithCatalog( catalogIdent, c -> diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index 41bca38c374..986de6bfa00 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -7,7 +7,6 @@ import static com.datastrato.gravitino.Entity.EntityType.SCHEMA; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; -import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -75,9 +74,6 @@ public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogExc @Override public Schema createSchema(NameIdentifier ident, String comment, Map properties) throws NoSuchCatalogException, SchemaAlreadyExistsException { - if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { - throw new IllegalArgumentException("Can't create a schema with with reserved name `*`"); - } NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index 0fc7e187dc3..5ba444e7770 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -9,7 +9,6 @@ import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; -import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -135,10 +134,6 @@ public Table createTable( SortOrder[] sortOrders, Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException { - if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { - throw new IllegalArgumentException("Can't create a table with with reserved name `*`"); - } - NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java index 8935f4f6518..c9188093d69 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java @@ -8,7 +8,6 @@ import static com.datastrato.gravitino.StringIdentifier.fromProperties; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; -import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; @@ -118,10 +117,6 @@ public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { public Topic createTopic( NameIdentifier ident, String comment, DataLayout dataLayout, Map properties) throws NoSuchSchemaException, TopicAlreadyExistsException { - if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(ident.name())) { - throw new IllegalArgumentException("Can't create a topic with with reserved name `*`"); - } - NameIdentifier catalogIdent = getCatalogIdentifier(ident); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java index 3303ea7b69c..3cd425b7fd3 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java @@ -4,7 +4,11 @@ */ package com.datastrato.gravitino.connector.capability; +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + import com.datastrato.gravitino.annotation.Evolving; +import com.google.common.collect.ImmutableSet; +import java.util.Set; /** * The Catalog interface to provide the capabilities of the catalog. If the implemented catalog has @@ -76,6 +80,22 @@ default CapabilityResult managedStorage(Scope scope) { /** The default implementation of the capability. */ class DefaultCapability implements Capability { + + private static final Set RESERVED_WORDS = + ImmutableSet.of(SECURABLE_ENTITY_RESERVED_NAME); + + /** + * Regular expression explanation: + * + *

^[a-zA-Z_] - Starts with a letter, digit, or underscore + * + *

[a-zA-Z0-9_/=-]{0,63} - Followed by 0 to 63 characters (making the total length at most + * 64) of letters (both cases), digits, underscores, slashes, hyphens, or equals signs + * + *

$ - End of the string + */ + private static final String DEFAULT_NAME_PATTERN = "^[a-zA-Z0-9_][a-zA-Z0-9_/=-]{0,63}$"; + @Override public CapabilityResult columnNotNull() { return CapabilityResult.SUPPORTED; @@ -93,6 +113,15 @@ public CapabilityResult caseSensitiveOnName(Scope scope) { @Override public CapabilityResult specificationOnName(Scope scope, String name) { + if (RESERVED_WORDS.contains(name.toLowerCase())) { + return CapabilityResult.unsupported( + String.format("The %s name '%s' is reserved.", scope, name)); + } + + if (!name.matches(DEFAULT_NAME_PATTERN)) { + return CapabilityResult.unsupported( + String.format("The %s name '%s' is illegal.", scope, name)); + } return CapabilityResult.SUPPORTED; } diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java index b0ea02cc4e6..355b736c120 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetNormalizeDispatcher.java @@ -4,6 +4,8 @@ */ package com.datastrato.gravitino.catalog; +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; @@ -72,4 +74,31 @@ public void testNameCaseInsensitive() { NameIdentifier.of(filesetNs, filesetIdent.name().toUpperCase()))); Assertions.assertFalse(filesetNormalizeDispatcher.filesetExists(filesetIdent)); } + + @Test + public void testNameSpec() { + Namespace filesetNs = Namespace.of(metalake, catalog, "testNameSpec"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent = NameIdentifier.of(filesetNs, SECURABLE_ENTITY_RESERVED_NAME); + Exception exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + filesetNormalizeDispatcher.createFileset( + filesetIdent, "comment", Fileset.Type.MANAGED, "fileset41", props)); + Assertions.assertEquals( + "The FILESET name '*' is reserved. Illegal name: *", exception.getMessage()); + + NameIdentifier filesetIdent2 = NameIdentifier.of(filesetNs, "a?"); + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + filesetNormalizeDispatcher.createFileset( + filesetIdent2, "comment", Fileset.Type.MANAGED, "fileset41", props)); + Assertions.assertEquals( + "The FILESET name 'a?' is illegal. Illegal name: a?", exception.getMessage()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java index 206ce794e5f..40c26d0e9d6 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestSchemaNormalizeDispatcher.java @@ -4,6 +4,8 @@ */ package com.datastrato.gravitino.catalog; +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.rel.Schema; @@ -54,4 +56,24 @@ public void testNameCaseInsensitive() { NameIdentifier.of(schemaIdent.namespace(), schemaIdent.name().toLowerCase()), false)); Assertions.assertFalse(schemaNormalizeDispatcher.schemaExists(schemaIdent)); } + + @Test + public void testNameSpec() { + NameIdentifier schemaIdent1 = + NameIdentifier.of(metalake, catalog, SECURABLE_ENTITY_RESERVED_NAME); + Exception exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> schemaNormalizeDispatcher.createSchema(schemaIdent1, null, null)); + Assertions.assertEquals( + "The SCHEMA name '*' is reserved. Illegal name: *", exception.getMessage()); + + NameIdentifier schemaIdent2 = NameIdentifier.of(metalake, catalog, "a?"); + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> schemaNormalizeDispatcher.createSchema(schemaIdent2, null, null)); + Assertions.assertEquals( + "The SCHEMA name 'a?' is illegal. Illegal name: a?", exception.getMessage()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java index be3132050f7..618b608f9ab 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTableNormalizeDispatcher.java @@ -4,6 +4,8 @@ */ package com.datastrato.gravitino.catalog; +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.TestColumn; @@ -97,6 +99,49 @@ public void testNameCaseInsensitive() { NameIdentifier.of(tableNs, tableIdent.name().toUpperCase()))); } + @Test + public void testNameSpec() { + Namespace tableNs = Namespace.of(metalake, catalog, "testNameSpec"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(tableNs.levels()), "comment", props); + + NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, SECURABLE_ENTITY_RESERVED_NAME); + Column[] columns = + new Column[] { + TestColumn.builder().withName("colNAME1").withType(Types.StringType.get()).build(), + TestColumn.builder().withName("colNAME2").withType(Types.StringType.get()).build() + }; + Exception exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> tableNormalizeDispatcher.createTable(tableIdent1, columns, "comment", props)); + Assertions.assertEquals( + "The TABLE name '*' is reserved. Illegal name: *", exception.getMessage()); + + NameIdentifier tableIdent2 = NameIdentifier.of(tableNs, "a?"); + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> tableNormalizeDispatcher.createTable(tableIdent2, columns, "comment", props)); + Assertions.assertEquals( + "The TABLE name 'a?' is illegal. Illegal name: a?", exception.getMessage()); + + NameIdentifier tableIdent3 = NameIdentifier.of(tableNs, "abc"); + Column[] columns1 = + new Column[] { + TestColumn.builder() + .withName(SECURABLE_ENTITY_RESERVED_NAME) + .withType(Types.StringType.get()) + .build() + }; + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> tableNormalizeDispatcher.createTable(tableIdent3, columns1, "comment", props)); + Assertions.assertEquals( + "The COLUMN name '*' is reserved. Illegal name: *", exception.getMessage()); + } + private void assertTableCaseInsensitive( NameIdentifier tableIdent, Column[] expectedColumns, Table table) { Assertions.assertEquals(tableIdent.name().toLowerCase(), table.name()); diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java index 8f8cd8e41f0..ec81f9ff036 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestTopicNormalizeDispatcher.java @@ -4,6 +4,8 @@ */ package com.datastrato.gravitino.catalog; +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.messaging.Topic; @@ -58,4 +60,27 @@ public void testNameCaseInsensitive() { NameIdentifier.of(topicNs, topicIdent.name().toUpperCase()))); Assertions.assertFalse(topicNormalizeDispatcher.topicExists(topicIdent)); } + + @Test + public void testNameSpec() { + Namespace topicNs = Namespace.of(metalake, catalog, "testNameSpec"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaNormalizeDispatcher.createSchema(NameIdentifier.of(topicNs.levels()), "comment", props); + + NameIdentifier topicIdent = NameIdentifier.of(topicNs, SECURABLE_ENTITY_RESERVED_NAME); + Exception exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> topicNormalizeDispatcher.createTopic(topicIdent, "comment", null, props)); + Assertions.assertEquals( + "The TOPIC name '*' is reserved. Illegal name: *", exception.getMessage()); + + NameIdentifier topicIdent2 = NameIdentifier.of(topicNs, "a?"); + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> topicNormalizeDispatcher.createTopic(topicIdent2, "comment", null, props)); + Assertions.assertEquals( + "The TOPIC name 'a?' is illegal. Illegal name: a?", exception.getMessage()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/connector/capability/TestCapability.java b/core/src/test/java/com/datastrato/gravitino/connector/capability/TestCapability.java new file mode 100644 index 00000000000..0d955adac59 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/connector/capability/TestCapability.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.connector.capability; + +import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCapability { + + @Test + void testDefaultNameSpecification() { + for (Capability.Scope scope : Capability.Scope.values()) { + // test for normal name + CapabilityResult result = Capability.DEFAULT.specificationOnName(scope, "_name_123_/_=-"); + Assertions.assertTrue(result.supported()); + + // test for reserved name + result = Capability.DEFAULT.specificationOnName(scope, SECURABLE_ENTITY_RESERVED_NAME); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is reserved")); + + // test for illegal name + result = Capability.DEFAULT.specificationOnName(scope, "name with space"); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + + result = Capability.DEFAULT.specificationOnName(scope, "name_with_@"); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + + result = Capability.DEFAULT.specificationOnName(scope, "name_with_#"); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + + result = Capability.DEFAULT.specificationOnName(scope, "name_with_$"); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + + result = Capability.DEFAULT.specificationOnName(scope, "name_with_%"); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + + // test for long name + StringBuilder longName = new StringBuilder(); + for (int i = 0; i < 64; i++) { + longName.append("a"); + } + + Assertions.assertEquals(64, longName.length()); + result = Capability.DEFAULT.specificationOnName(scope, longName.toString()); + Assertions.assertTrue(result.supported()); + + longName.append("a"); + Assertions.assertEquals(65, longName.length()); + result = Capability.DEFAULT.specificationOnName(scope, longName.toString()); + Assertions.assertFalse(result.supported()); + Assertions.assertTrue(result.unsupportedMessage().contains("is illegal")); + } + } +} From 6d598bc2052b77f85abd3f54d2339e789a39bee3 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Sat, 20 Apr 2024 16:37:52 +0800 Subject: [PATCH 082/106] [#2807]: Improvement(IT): manage mysql containers in ContainerSuite (#2813) ### What changes were proposed in this pull request? Manage MySQL containers (`mysql:5.7`, `mysql:8.0`) in `ContainerSuite`. And each test class will create a unique database named `TestClass.class.getSimpleName() + TestClass.class.hashCode()` for testing. ### Why are the changes needed? Because we want to share containers and manage their lifecycles in `ContainerSuite`. Fix: #2807 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Tested in integration tests. --------- Co-authored-by: Qi Yu --- .../integration/test/TestJdbcAbstractIT.java | 2 +- .../integration/test/AuditCatalogMysqlIT.java | 35 ++-- .../integration/test/CatalogMysqlIT.java | 46 +++-- .../test/MysqlTableOperationsIT.java | 174 ++++++++++-------- .../integration/test/TestMysqlAbstractIT.java | 46 +++-- .../test/service/MysqlService.java | 9 +- .../test/TestPostgreSqlAbstractIT.java | 2 +- .../test/container/ContainerSuite.java | 71 ++++++- .../test/container/MySQLContainer.java | 158 ++++++++++++++++ .../integration/test/util/AbstractIT.java | 34 ++-- .../test/util/TestDatabaseName.java | 40 ++++ 11 files changed, 475 insertions(+), 142 deletions(-) create mode 100644 integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java create mode 100644 integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/TestDatabaseName.java diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/integration/test/TestJdbcAbstractIT.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/integration/test/TestJdbcAbstractIT.java index 9749bab7f25..ec03143b93a 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/integration/test/TestJdbcAbstractIT.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/integration/test/TestJdbcAbstractIT.java @@ -47,7 +47,7 @@ public abstract class TestJdbcAbstractIT { protected static final String TEST_DB_NAME = RandomNameUtils.genRandomName("test_db_"); - public static void startup() { + public static void startup() throws Exception { CONTAINER.start(); HashMap properties = Maps.newHashMap(); properties.put(JdbcConfig.JDBC_DRIVER.getKey(), CONTAINER.getDriverClassName()); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 713790d2768..ec38006b7e5 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -13,10 +13,13 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.catalog.mysql.integration.test.service.MysqlService; import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.Table; @@ -26,30 +29,28 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.SQLException; import java.util.Collections; import java.util.Map; -import java.util.Random; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; @Tag("gravitino-docker-it") public class AuditCatalogMysqlIT extends AbstractIT { - + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake"); private static final String expectUser = System.getProperty("user.name"); public static final String DOWNLOAD_JDBC_DRIVER_URL = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; - public static final String mysqlImageName = "mysql:8.0"; - protected static final String TEST_DB_NAME = new Random().nextInt(1000) + "_test_db"; + protected static TestDatabaseName TEST_DB_NAME; private static final String provider = "jdbc-mysql"; private static MysqlService mysqlService; - private static MySQLContainer MYSQL_CONTAINER; + private static MySQLContainer MYSQL_CONTAINER; private static GravitinoMetalake metalake; @BeforeAll @@ -65,13 +66,10 @@ public static void startIntegrationTest() throws Exception { JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); } - MYSQL_CONTAINER = - new MySQLContainer<>(mysqlImageName) - .withDatabaseName(TEST_DB_NAME) - .withUsername("root") - .withPassword("root"); - MYSQL_CONTAINER.start(); - mysqlService = new MysqlService(MYSQL_CONTAINER); + containerSuite.startMySQLContainer(TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT); + MYSQL_CONTAINER = containerSuite.getMySQLContainer(); + TEST_DB_NAME = TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT; + mysqlService = new MysqlService(containerSuite.getMySQLContainer(), TEST_DB_NAME); createMetalake(); } @@ -80,13 +78,13 @@ public static void stopIntegrationTest() throws IOException, InterruptedExceptio AbstractIT.stopIntegrationTest(); client.dropMetalake(NameIdentifier.of(metalakeName)); mysqlService.close(); - MYSQL_CONTAINER.stop(); } @Test public void testAuditCatalog() throws Exception { String catalogName = GravitinoITUtils.genRandomName("audit_mysql_catalog"); Catalog catalog = createCatalog(catalogName); + Assertions.assertEquals(expectUser, catalog.auditInfo().creator()); Assertions.assertEquals(catalog.auditInfo().creator(), catalog.auditInfo().lastModifier()); Assertions.assertEquals( @@ -144,14 +142,17 @@ public void testAuditTable() throws Exception { Assertions.assertEquals(expectUser, table.auditInfo().lastModifier()); } - private static Catalog createCatalog(String catalogName) { + private static Catalog createCatalog(String catalogName) throws SQLException { Map catalogProperties = Maps.newHashMap(); catalogProperties.put( JdbcConfig.JDBC_URL.getKey(), StringUtils.substring( - MYSQL_CONTAINER.getJdbcUrl(), 0, MYSQL_CONTAINER.getJdbcUrl().lastIndexOf("/"))); - catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName()); + MYSQL_CONTAINER.getJdbcUrl(TEST_DB_NAME), + 0, + MYSQL_CONTAINER.getJdbcUrl(TEST_DB_NAME).lastIndexOf("/"))); + catalogProperties.put( + JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName(TEST_DB_NAME)); catalogProperties.put(JdbcConfig.USERNAME.getKey(), MYSQL_CONTAINER.getUsername()); catalogProperties.put(JdbcConfig.PASSWORD.getKey(), MYSQL_CONTAINER.getPassword()); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index 944822488ed..649d20b510b 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -18,10 +18,13 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NotFoundException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Column.ColumnImpl; import com.datastrato.gravitino.rel.Schema; @@ -46,6 +49,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -62,12 +66,11 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.condition.EnabledIf; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; @Tag("gravitino-docker-it") @TestInstance(Lifecycle.PER_CLASS) public class CatalogMysqlIT extends AbstractIT { + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String provider = "jdbc-mysql"; public static final String DOWNLOAD_JDBC_DRIVER_URL = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; @@ -93,9 +96,9 @@ public class CatalogMysqlIT extends AbstractIT { private MysqlService mysqlService; - private MySQLContainer MYSQL_CONTAINER; + private MySQLContainer MYSQL_CONTAINER; - protected final String TEST_DB_NAME = RandomUtils.nextInt(0, 10000) + "_test_db"; + private TestDatabaseName TEST_DB_NAME; public static final String defaultMysqlImageName = "mysql:8.0"; @@ -106,7 +109,7 @@ boolean SupportColumnDefaultValueExpression() { } @BeforeAll - public void startup() throws IOException { + public void startup() throws IOException, SQLException { if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { String gravitinoHome = System.getenv("GRAVITINO_HOME"); @@ -114,13 +117,17 @@ public void startup() throws IOException { JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); } - MYSQL_CONTAINER = - new MySQLContainer<>(mysqlImageName) - .withDatabaseName(TEST_DB_NAME) - .withUsername("root") - .withPassword("root"); - MYSQL_CONTAINER.start(); - mysqlService = new MysqlService(MYSQL_CONTAINER); + TEST_DB_NAME = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; + + if (mysqlImageName.equals("mysql:5.7")) { + containerSuite.startMySQLVersion5Container(TestDatabaseName.MYSQL_CATALOG_MYSQL_IT); + MYSQL_CONTAINER = containerSuite.getMySQLVersion5Container(); + } else { + containerSuite.startMySQLContainer(TEST_DB_NAME); + MYSQL_CONTAINER = containerSuite.getMySQLContainer(); + } + + mysqlService = new MysqlService(MYSQL_CONTAINER, TEST_DB_NAME); createMetalake(); createCatalog(); createSchema(); @@ -131,11 +138,10 @@ public void stop() { clearTableAndSchema(); client.dropMetalake(NameIdentifier.of(metalakeName)); mysqlService.close(); - MYSQL_CONTAINER.stop(); } @AfterEach - private void resetSchema() { + public void resetSchema() { clearTableAndSchema(); createSchema(); } @@ -161,14 +167,17 @@ private void createMetalake() { metalake = loadMetalake; } - private void createCatalog() { + private void createCatalog() throws SQLException { Map catalogProperties = Maps.newHashMap(); catalogProperties.put( JdbcConfig.JDBC_URL.getKey(), StringUtils.substring( - MYSQL_CONTAINER.getJdbcUrl(), 0, MYSQL_CONTAINER.getJdbcUrl().lastIndexOf("/"))); - catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName()); + MYSQL_CONTAINER.getJdbcUrl(TEST_DB_NAME), + 0, + MYSQL_CONTAINER.getJdbcUrl(TEST_DB_NAME).lastIndexOf("/"))); + catalogProperties.put( + JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName(TEST_DB_NAME)); catalogProperties.put(JdbcConfig.USERNAME.getKey(), MYSQL_CONTAINER.getUsername()); catalogProperties.put(JdbcConfig.PASSWORD.getKey(), MYSQL_CONTAINER.getPassword()); @@ -769,7 +778,8 @@ void testDropMySQLDatabase() { "Created by gravitino client", ImmutableMap.builder().build()); - // Try to drop a database, and cascade equals to false, it should not be allowed. + // Try to drop a database, and cascade equals to false, it should not be + // allowed. catalog.asSchemas().dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), false); // Check the database still exists catalog.asSchemas().loadSchema(NameIdentifier.of(metalakeName, catalogName, schemaName)); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/MysqlTableOperationsIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/MysqlTableOperationsIT.java index 0d577eadfed..fd4cf52fa31 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/MysqlTableOperationsIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/MysqlTableOperationsIT.java @@ -37,13 +37,11 @@ @Tag("gravitino-docker-it") public class MysqlTableOperationsIT extends TestMysqlAbstractIT { - private static Type VARCHAR = Types.VarCharType.of(255); private static Type INT = Types.IntegerType.get(); @Test public void testOperationTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); @@ -81,7 +79,7 @@ public void testOperationTable() { Index[] indexes = new Index[] {Indexes.unique("test", new String[][] {{"col_1"}, {"col_2"}})}; // create table TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns.toArray(new JdbcColumn[0]), tableComment, @@ -91,17 +89,18 @@ public void testOperationTable() { indexes); // list table - List tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); + List tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME.toString()); Assertions.assertTrue(tables.contains(tableName)); // load table - JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); // rename table String newName = "new_table"; - Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.rename(TEST_DB_NAME, tableName, newName)); - Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.load(TEST_DB_NAME, newName)); + Assertions.assertDoesNotThrow( + () -> TABLE_OPERATIONS.rename(TEST_DB_NAME.toString(), tableName, newName)); + Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName)); // alter table JdbcColumn newColumn = @@ -113,7 +112,7 @@ public void testOperationTable() { .withDefaultValue(Literals.of("hello test", VARCHAR)) .build(); TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), newName, TableChange.addColumn( new String[] {newColumn.name()}, @@ -123,7 +122,7 @@ public void testOperationTable() { newColumn.defaultValue()), TableChange.setProperty(MYSQL_ENGINE_KEY, "InnoDB")); properties.put(MYSQL_ENGINE_KEY, "InnoDB"); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName); List alterColumns = new ArrayList() { { @@ -141,36 +140,40 @@ public void testOperationTable() { GravitinoRuntimeException gravitinoRuntimeException = Assertions.assertThrows( GravitinoRuntimeException.class, - () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME, newName, setProperty)); + () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(), newName, setProperty)); Assertions.assertTrue( StringUtils.contains( gravitinoRuntimeException.getMessage(), "Unknown storage engine 'ABC'")); // delete column TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); + TEST_DB_NAME.toString(), + newName, + TableChange.deleteColumn(new String[] {newColumn.name()}, true)); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName); assertionsTableInfo(newName, tableComment, columns, properties, indexes, load); TableChange deleteColumn = TableChange.deleteColumn(new String[] {newColumn.name()}, false); IllegalArgumentException illegalArgumentException = Assertions.assertThrows( IllegalArgumentException.class, - () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME, newName, deleteColumn)); + () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(), newName, deleteColumn)); Assertions.assertEquals( "Delete column does not exist: " + newColumn.name(), illegalArgumentException.getMessage()); Assertions.assertDoesNotThrow( () -> TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true))); TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); - Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.drop(TEST_DB_NAME, newName)); + TEST_DB_NAME.toString(), + newName, + TableChange.deleteColumn(new String[] {newColumn.name()}, true)); + Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), newName)); Assertions.assertThrows( - NoSuchTableException.class, () -> TABLE_OPERATIONS.drop(TEST_DB_NAME, newName)); + NoSuchTableException.class, () -> TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), newName)); } @Test @@ -202,9 +205,9 @@ public void testAlterTable() { .withComment("name") .withDefaultValue(Literals.NULL) .build(); - // `col_1` int NOT NULL COMMENT 'id' , - // `col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'name' , - // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , + // `col_1` int NOT NULL COMMENT 'id' , + // `col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'name' , + // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , columns.add(col_3); Map properties = new HashMap<>(); @@ -215,7 +218,7 @@ public void testAlterTable() { }; // create table TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns.toArray(new JdbcColumn[0]), tableComment, @@ -223,17 +226,18 @@ public void testAlterTable() { null, Distributions.NONE, indexes); - JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.updateColumnType(new String[] {col_1.name()}, VARCHAR)); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); - // After modifying the type, some attributes of the corresponding column are not supported. + // After modifying the type, some attributes of the corresponding column are not + // supported. columns.clear(); col_1 = JdbcColumn.builder() @@ -250,15 +254,15 @@ public void testAlterTable() { String newComment = "new_comment"; // update table comment and column comment - // `col_1` int NOT NULL COMMENT 'id' , - // `col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'new_comment' , - // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , + // `col_1` int NOT NULL COMMENT 'id' , + // `col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'new_comment' , + // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.updateColumnType(new String[] {col_1.name()}, INT), TableChange.updateColumnComment(new String[] {col_2.name()}, newComment)); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); columns.clear(); col_1 = @@ -288,16 +292,17 @@ public void testAlterTable() { String newColName_2 = "new_col_2"; // rename column // update table comment and column comment - // `new_col_1` int NOT NULL COMMENT 'id' , - // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'new_comment' , - // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , + // `new_col_1` int NOT NULL COMMENT 'id' , + // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'new_comment' + // , + // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.renameColumn(new String[] {col_1.name()}, newColName_1), TableChange.renameColumn(new String[] {col_2.name()}, newColName_2)); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); columns.clear(); col_1 = @@ -326,13 +331,13 @@ public void testAlterTable() { newComment = "txt3"; String newCol2Comment = "xxx"; // update column position 、comment and add column、set table properties - // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'xxx' , - // `new_col_1` int NOT NULL COMMENT 'id' , - // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , - // `col_4` varchar(255) NOT NULL COMMENT 'txt4' , - // `col_5` varchar(255) COMMENT 'hello world' DEFAULT 'hello world' , + // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'xxx' , + // `new_col_1` int NOT NULL COMMENT 'id' , + // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , + // `col_4` varchar(255) NOT NULL COMMENT 'txt4' , + // `col_5` varchar(255) COMMENT 'hello world' DEFAULT 'hello world' , TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.updateColumnPosition( new String[] {newColName_1}, TableChange.ColumnPosition.after(newColName_2)), @@ -341,7 +346,7 @@ public void testAlterTable() { TableChange.updateColumnComment(new String[] {newColName_2}, newCol2Comment), TableChange.addColumn( new String[] {"col_5"}, VARCHAR, "txt5", Literals.of("hello world", VARCHAR))); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); columns.clear(); @@ -374,19 +379,19 @@ public void testAlterTable() { .build()); assertionsTableInfo(tableName, newComment, columns, properties, indexes, load); - // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'xxx' , - // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , - // `col_4` varchar(255) NULL COMMENT 'txt4' , - // `col_5` varchar(255) COMMENT 'hello world' DEFAULT 'hello world' , - // `new_col_1` int NOT NULL COMMENT 'id' , + // `new_col_2` varchar(255) NOT NULL DEFAULT 'hello world' COMMENT 'xxx' , + // `col_3` varchar(255) NULL DEFAULT NULL COMMENT 'name' , + // `col_4` varchar(255) NULL COMMENT 'txt4' , + // `col_5` varchar(255) COMMENT 'hello world' DEFAULT 'hello world' , + // `new_col_1` int NOT NULL COMMENT 'id' , TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.updateColumnPosition(new String[] {columns.get(1).name()}, null), TableChange.updateColumnNullability( new String[] {columns.get(3).name()}, !columns.get(3).nullable())); - load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); col_1 = columns.remove(1); JdbcColumn col3 = columns.remove(1); JdbcColumn col_4 = columns.remove(1); @@ -420,7 +425,7 @@ public void testAlterTable() { IllegalArgumentException exception = Assertions.assertThrows( IllegalArgumentException.class, - () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME, tableName, updateColumn)); + () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(), tableName, updateColumn)); Assertions.assertTrue( exception.getMessage().contains("with null default value cannot be changed to not null")); } @@ -472,7 +477,7 @@ public void testAlterTableUpdateColumnDefaultValue() { }; // create table TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns.toArray(new JdbcColumn[0]), tableComment, @@ -481,11 +486,11 @@ public void testAlterTableUpdateColumnDefaultValue() { Distributions.NONE, indexes); - JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo(tableName, tableComment, columns, properties, indexes, loaded); TABLE_OPERATIONS.alterTable( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, TableChange.updateColumnDefaultValue( new String[] {columns.get(0).name()}, @@ -498,7 +503,7 @@ public void testAlterTableUpdateColumnDefaultValue() { TableChange.updateColumnDefaultValue( new String[] {columns.get(3).name()}, Literals.of("world", Types.VarCharType.of(255)))); - loaded = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); Assertions.assertEquals( Literals.decimalLiteral(Decimal.of("1.234", 10, 2)), loaded.columns()[0].defaultValue()); Assertions.assertEquals(Literals.longLiteral(1L), loaded.columns()[1].defaultValue()); @@ -556,7 +561,7 @@ public void testCreateAndLoadTable() { }; // create table TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns.toArray(new JdbcColumn[0]), tableComment, @@ -565,7 +570,7 @@ public void testCreateAndLoadTable() { Distributions.NONE, indexes); - JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo(tableName, tableComment, columns, properties, indexes, loaded); } @@ -654,7 +659,7 @@ public void testCreateAllTypeTable() { // create table TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns.toArray(new JdbcColumn[0]), tableComment, @@ -663,7 +668,7 @@ public void testCreateAllTypeTable() { Distributions.NONE, Indexes.EMPTY_INDEXES); - JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), null, load); } @@ -698,7 +703,7 @@ public void testCreateNotSupportTypeTable() { IllegalArgumentException.class, () -> { TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, jdbcCols, tableComment, @@ -720,7 +725,7 @@ public void testCreateNotSupportTypeTable() { public void testCreateMultipleTables() { String test_table_1 = "test_table_1"; TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), test_table_1, new JdbcColumn[] { JdbcColumn.builder() @@ -762,7 +767,7 @@ public void testCreateMultipleTables() { Distributions.NONE, Indexes.EMPTY_INDEXES); - tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); + tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME.toString()); Assertions.assertFalse(tables.contains(test_table_2)); } @@ -770,7 +775,7 @@ public void testCreateMultipleTables() { public void testLoadTableDefaultProperties() { String test_table_1 = RandomNameUtils.genRandomName("properties_table_"); TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), test_table_1, new JdbcColumn[] { JdbcColumn.builder() @@ -785,7 +790,7 @@ public void testLoadTableDefaultProperties() { null, Distributions.NONE, Indexes.EMPTY_INDEXES); - JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, test_table_1); + JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), test_table_1); Assertions.assertEquals("InnoDB", load.properties().get(MYSQL_ENGINE_KEY)); } @@ -827,9 +832,16 @@ public void testAutoIncrement() { Indexes.unique("uk_1", new String[][] {{"col_1"}}) }; TABLE_OPERATIONS.create( - TEST_DB_NAME, tableName, columns, comment, properties, null, Distributions.NONE, indexes); + TEST_DB_NAME.toString(), + tableName, + columns, + comment, + properties, + null, + Distributions.NONE, + indexes); - JdbcTable table = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + JdbcTable table = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo( tableName, comment, @@ -837,7 +849,7 @@ public void testAutoIncrement() { properties, indexes, table); - TABLE_OPERATIONS.drop(TEST_DB_NAME, tableName); + TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), tableName); // Test create increment key for primary index. indexes = @@ -846,9 +858,16 @@ public void testAutoIncrement() { Indexes.unique("uk_2", new String[][] {{"col_2"}}) }; TABLE_OPERATIONS.create( - TEST_DB_NAME, tableName, columns, comment, properties, null, Distributions.NONE, indexes); + TEST_DB_NAME.toString(), + tableName, + columns, + comment, + properties, + null, + Distributions.NONE, + indexes); - table = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + table = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo( tableName, comment, @@ -856,14 +875,21 @@ public void testAutoIncrement() { properties, indexes, table); - TABLE_OPERATIONS.drop(TEST_DB_NAME, tableName); + TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), tableName); // Test create increment key for col_1 + col_3 uk. indexes = new Index[] {Indexes.unique("uk_2_3", new String[][] {{"col_1"}, {"col_3"}})}; TABLE_OPERATIONS.create( - TEST_DB_NAME, tableName, columns, comment, properties, null, Distributions.NONE, indexes); + TEST_DB_NAME.toString(), + tableName, + columns, + comment, + properties, + null, + Distributions.NONE, + indexes); - table = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); + table = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName); assertionsTableInfo( tableName, comment, @@ -871,7 +897,7 @@ public void testAutoIncrement() { properties, indexes, table); - TABLE_OPERATIONS.drop(TEST_DB_NAME, tableName); + TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), tableName); // Test create auto increment fail IllegalArgumentException exception = @@ -879,7 +905,7 @@ public void testAutoIncrement() { IllegalArgumentException.class, () -> TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, columns, comment, @@ -913,7 +939,7 @@ public void testAutoIncrement() { IllegalArgumentException.class, () -> TABLE_OPERATIONS.create( - TEST_DB_NAME, + TEST_DB_NAME.toString(), tableName, newColumns, comment, diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlAbstractIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlAbstractIT.java index eab3d2858d7..456f43054ca 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlAbstractIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/TestMysqlAbstractIT.java @@ -4,37 +4,61 @@ */ package com.datastrato.gravitino.catalog.mysql.integration.test; +import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.catalog.jdbc.integration.test.TestJdbcAbstractIT; +import com.datastrato.gravitino.catalog.jdbc.utils.DataSourceUtils; import com.datastrato.gravitino.catalog.mysql.converter.MysqlColumnDefaultValueConverter; import com.datastrato.gravitino.catalog.mysql.converter.MysqlExceptionConverter; import com.datastrato.gravitino.catalog.mysql.converter.MysqlTypeConverter; import com.datastrato.gravitino.catalog.mysql.operation.MysqlDatabaseOperations; import com.datastrato.gravitino.catalog.mysql.operation.MysqlTableOperations; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.MySQLContainer; +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; +import com.google.common.collect.Maps; +import java.sql.SQLException; import java.util.Collections; +import java.util.Map; +import javax.sql.DataSource; import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.MySQLContainer; public class TestMysqlAbstractIT extends TestJdbcAbstractIT { - public static final String defaultMysqlImageName = "mysql:8.0"; + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static TestDatabaseName TEST_DB_NAME; @BeforeAll - public static void startup() { - CONTAINER = - new MySQLContainer<>(defaultMysqlImageName) - .withDatabaseName(TEST_DB_NAME) - .withUsername("root") - .withPassword("root"); + public static void startup() throws Exception { + ContainerSuite containerSuite = ContainerSuite.getInstance(); + TEST_DB_NAME = TestDatabaseName.MYSQL_MYSQL_ABSTRACT_IT; + containerSuite.startMySQLContainer(TEST_DB_NAME); + DataSource dataSource = DataSourceUtils.createDataSource(getMySQLCatalogProperties()); + DATABASE_OPERATIONS = new MysqlDatabaseOperations(); TABLE_OPERATIONS = new MysqlTableOperations(); JDBC_EXCEPTION_CONVERTER = new MysqlExceptionConverter(); - TestJdbcAbstractIT.startup(); - DATABASE_OPERATIONS.initialize(DATA_SOURCE, JDBC_EXCEPTION_CONVERTER, Collections.emptyMap()); + DATABASE_OPERATIONS.initialize(dataSource, JDBC_EXCEPTION_CONVERTER, Collections.emptyMap()); TABLE_OPERATIONS.initialize( - DATA_SOURCE, + dataSource, JDBC_EXCEPTION_CONVERTER, new MysqlTypeConverter(), new MysqlColumnDefaultValueConverter(), Collections.emptyMap()); } + + private static Map getMySQLCatalogProperties() throws SQLException { + Map catalogProperties = Maps.newHashMap(); + + MySQLContainer mySQLContainer = containerSuite.getMySQLContainer(); + + String jdbcUrl = mySQLContainer.getJdbcUrl(TEST_DB_NAME); + + catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); + catalogProperties.put( + JdbcConfig.JDBC_DRIVER.getKey(), mySQLContainer.getDriverClassName(TEST_DB_NAME)); + catalogProperties.put(JdbcConfig.USERNAME.getKey(), mySQLContainer.getUsername()); + catalogProperties.put(JdbcConfig.PASSWORD.getKey(), mySQLContainer.getPassword()); + + return catalogProperties; + } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java index 0205bbdb5f7..228bdb21494 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java @@ -8,6 +8,8 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.jdbc.JdbcSchema; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.integration.test.container.MySQLContainer; +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.meta.AuditInfo; import java.sql.Connection; import java.sql.DriverManager; @@ -19,13 +21,12 @@ import java.util.List; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.testcontainers.containers.MySQLContainer; public class MysqlService { private Connection connection; - public MysqlService(MySQLContainer mysqlContainer) { + public MysqlService(MySQLContainer mysqlContainer, TestDatabaseName testDBName) { String username = mysqlContainer.getUsername(); String password = mysqlContainer.getPassword(); @@ -33,7 +34,9 @@ public MysqlService(MySQLContainer mysqlContainer) { connection = DriverManager.getConnection( StringUtils.substring( - mysqlContainer.getJdbcUrl(), 0, mysqlContainer.getJdbcUrl().lastIndexOf("/")), + mysqlContainer.getJdbcUrl(testDBName), + 0, + mysqlContainer.getJdbcUrl(testDBName).lastIndexOf("/")), username, password); } catch (Exception e) { diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestPostgreSqlAbstractIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestPostgreSqlAbstractIT.java index f93b703fe9a..4976f12f959 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestPostgreSqlAbstractIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestPostgreSqlAbstractIT.java @@ -23,7 +23,7 @@ public class TestPostgreSqlAbstractIT extends TestJdbcAbstractIT { public static final String DEFAULT_POSTGRES_IMAGE = "postgres:13"; @BeforeAll - public static void startup() { + public static void startup() throws Exception { CONTAINER = new PostgreSQLContainer<>(DEFAULT_POSTGRES_IMAGE) .withDatabaseName(TEST_DB_NAME) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 0eeb4962a29..2bd2ebbe8b7 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.integration.test.container; import com.datastrato.gravitino.integration.test.util.CloseableGroup; +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.RemoveNetworkCmd; import com.github.dockerjava.api.model.Info; @@ -25,7 +26,8 @@ public class ContainerSuite implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(ContainerSuite.class); private static volatile ContainerSuite instance = null; - // The subnet must match the configuration in `dev/docker/tools/mac-docker-connector.conf` + // The subnet must match the configuration in + // `dev/docker/tools/mac-docker-connector.conf` public static final String CONTAINER_NETWORK_SUBNET = "10.20.30.0/28"; private static final String CONTAINER_NETWORK_GATEWAY = "10.20.30.1"; private static final String CONTAINER_NETWORK_IPRANGE = "10.20.30.0/28"; @@ -38,6 +40,9 @@ public class ContainerSuite implements Closeable { private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; + private static volatile MySQLContainer mySQLContainer; + private static volatile MySQLContainer mySQLVersion5Container; + protected static final CloseableGroup closer = CloseableGroup.create(); private ContainerSuite() { @@ -150,6 +155,59 @@ public void startDorisContainer() { } } + public void startMySQLContainer(TestDatabaseName testDatabaseName) { + if (mySQLContainer == null) { + synchronized (ContainerSuite.class) { + if (mySQLContainer == null) { + // Start MySQL container + MySQLContainer.Builder mysqlBuilder = + MySQLContainer.builder() + .withHostName("gravitino-ci-mysql") + .withEnvVars( + ImmutableMap.builder() + .put("MYSQL_ROOT_PASSWORD", "root") + .build()) + .withExposePorts(ImmutableSet.of(MySQLContainer.MYSQL_PORT)) + .withNetwork(network); + + MySQLContainer container = closer.register(mysqlBuilder.build()); + container.start(); + mySQLContainer = container; + } + } + } + synchronized (MySQLContainer.class) { + mySQLContainer.createDatabase(testDatabaseName); + } + } + + public void startMySQLVersion5Container(TestDatabaseName testDatabaseName) { + if (mySQLVersion5Container == null) { + synchronized (ContainerSuite.class) { + if (mySQLVersion5Container == null) { + // Start MySQL container + MySQLContainer.Builder mysqlBuilder = + MySQLContainer.builder() + .withImage("mysql:5.7") + .withHostName("gravitino-ci-mysql-v5") + .withEnvVars( + ImmutableMap.builder() + .put("MYSQL_ROOT_PASSWORD", "root") + .build()) + .withExposePorts(ImmutableSet.of(MySQLContainer.MYSQL_PORT)) + .withNetwork(network); + + MySQLContainer container = closer.register(mysqlBuilder.build()); + container.start(); + mySQLVersion5Container = container; + } + } + } + synchronized (MySQLContainer.class) { + mySQLVersion5Container.createDatabase(testDatabaseName); + } + } + public void startKafkaContainer() { if (kafkaContainer == null) { synchronized (ContainerSuite.class) { @@ -192,7 +250,16 @@ public DorisContainer getDorisContainer() { return dorisContainer; } - // Let containers assign addresses in a fixed subnet to avoid `mac-docker-connector` needing to + public MySQLContainer getMySQLContainer() { + return mySQLContainer; + } + + public MySQLContainer getMySQLVersion5Container() { + return mySQLVersion5Container; + } + + // Let containers assign addresses in a fixed subnet to avoid + // `mac-docker-connector` needing to // refresh the configuration private static Network createDockerNetwork() { DockerClient dockerClient = DockerClientFactory.instance().client(); diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java new file mode 100644 index 00000000000..6547c9497ae --- /dev/null +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java @@ -0,0 +1,158 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.container; + +import static java.lang.String.format; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +import com.datastrato.gravitino.integration.test.util.TestDatabaseName; +import com.google.common.collect.ImmutableSet; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.rnorth.ducttape.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.Network; + +public class MySQLContainer extends BaseContainer { + public static final Logger LOG = LoggerFactory.getLogger(MySQLContainer.class); + + public static final String DEFAULT_IMAGE = "mysql:8.0"; + public static final String HOST_NAME = "gravitino-ci-mysql"; + public static final int MYSQL_PORT = 3306; + public static final String USER_NAME = "root"; + public static final String PASSWORD = "root"; + + public static Builder builder() { + return new Builder(); + } + + protected MySQLContainer( + String image, + String hostName, + Set ports, + Map extraHosts, + Map filesToMount, + Map envVars, + Optional network) { + super(image, hostName, ports, extraHosts, filesToMount, envVars, network); + } + + @Override + protected void setupContainer() { + super.setupContainer(); + withLogConsumer(new PrintingContainerLog(format("%-14s| ", "MySQLContainer"))); + } + + @Override + public void start() { + super.start(); + Preconditions.check("MySQL container startup failed!", checkContainerStatus(5)); + } + + @Override + protected boolean checkContainerStatus(int retryLimit) { + int nRetry = 0; + boolean isMySQLContainerReady = false; + int sleepTimeMillis = 20_00; + while (nRetry++ < retryLimit) { + try { + String[] commandAndArgs = + new String[] { + "mysqladmin", + "ping", + "-h", + "localhost", + "-u", + getUsername(), + String.format("-p%s", getPassword()) + }; + Container.ExecResult execResult = executeInContainer(commandAndArgs); + if (execResult.getExitCode() != 0) { + String message = + format( + "Command [%s] exited with %s", + String.join(" ", commandAndArgs), execResult.getExitCode()); + LOG.error("{}", message); + LOG.error("stderr: {}", execResult.getStderr()); + LOG.error("stdout: {}", execResult.getStdout()); + } else { + LOG.info("MySQL container startup success!"); + isMySQLContainerReady = true; + break; + } + LOG.info( + "MySQL container is not ready, recheck({}/{}) after {}ms", + nRetry, + retryLimit, + sleepTimeMillis); + await().atLeast(sleepTimeMillis, TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + } + } + + return isMySQLContainerReady; + } + + public void createDatabase(TestDatabaseName testDatabaseName) { + String mySQLJdbcUrl = + StringUtils.substring( + getJdbcUrl(testDatabaseName), 0, getJdbcUrl(testDatabaseName).lastIndexOf("/")); + + // change password for root user, Gravitino API must set password in catalog properties + try (Connection connection = + DriverManager.getConnection(mySQLJdbcUrl, USER_NAME, getPassword()); + Statement statement = connection.createStatement()) { + + String query = String.format("CREATE DATABASE IF NOT EXISTS %s;", testDatabaseName); + // FIXME: String, which is used in SQL, can be unsafe + statement.execute(query); + LOG.info(String.format("MySQL container database %s has been created", testDatabaseName)); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + public String getUsername() { + return USER_NAME; + } + + public String getPassword() { + return PASSWORD; + } + + public String getJdbcUrl(TestDatabaseName testDatabaseName) { + return format("jdbc:mysql://%s:%d/%s", getContainerIpAddress(), MYSQL_PORT, testDatabaseName); + } + + public String getDriverClassName(TestDatabaseName testDatabaseName) throws SQLException { + return DriverManager.getDriver(getJdbcUrl(testDatabaseName)).getClass().getName(); + } + + public static class Builder + extends BaseContainer.Builder { + + private Builder() { + this.image = DEFAULT_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(MYSQL_PORT); + } + + @Override + public MySQLContainer build() { + return new MySQLContainer( + image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); + } + } +} diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 898a52882fd..d9c95872ed5 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -14,6 +14,8 @@ import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.integration.test.MiniGravitino; import com.datastrato.gravitino.integration.test.MiniGravitinoContext; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.server.GravitinoServer; import com.datastrato.gravitino.server.ServerConfig; import com.datastrato.gravitino.server.web.JettyServerConfig; @@ -33,12 +35,15 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.MySQLContainer; @ExtendWith(PrintFuncNameExtension.class) public class AbstractIT { + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class); protected static GravitinoAdminClient client; @@ -56,12 +61,11 @@ public class AbstractIT { protected static boolean ignoreIcebergRestService = true; - private static final String MYSQL_DOCKER_IMAGE_VERSION = "mysql:8.0"; private static final String DOWNLOAD_JDBC_DRIVER_URL = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar"; - private static final String META_DATA = "metadata"; - private static MySQLContainer MYSQL_CONTAINER; + private static TestDatabaseName META_DATA; + private static MySQLContainer MYSQL_CONTAINER; protected static String serverUri; @@ -110,7 +114,7 @@ protected static void downLoadMySQLDriver(String relativeDeployLibsPath) throws } private static void setMySQLBackend() { - String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(); + String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(META_DATA); customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); customConfigs.put(Configs.ENTITY_RELATIONAL_STORE_KEY, "JDBCBackend"); customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, mysqlUrl); @@ -146,6 +150,13 @@ private static void setMySQLBackend() { } } + @ParameterizedTest + @CsvSource({ + "embedded, jdbcBackend", + "embedded, kvBackend", + "deploy, jdbcBackend", + "deploy, kvBackend" + }) @BeforeAll public static void startIntegrationTest() throws Exception { testMode = @@ -157,12 +168,9 @@ public static void startIntegrationTest() throws Exception { if ("true".equals(System.getenv("jdbcBackend"))) { // Start MySQL docker instance. - MYSQL_CONTAINER = - new MySQLContainer<>(MYSQL_DOCKER_IMAGE_VERSION) - .withDatabaseName(META_DATA) - .withUsername("root") - .withPassword("root"); - MYSQL_CONTAINER.start(); + META_DATA = TestDatabaseName.MYSQL_JDBC_BACKEND; + containerSuite.startMySQLContainer(META_DATA); + MYSQL_CONTAINER = containerSuite.getMySQLContainer(); setMySQLBackend(); } @@ -229,10 +237,6 @@ public static void stopIntegrationTest() throws IOException, InterruptedExceptio } customConfigs.clear(); LOG.info("Tearing down Gravitino Server"); - - if (MYSQL_CONTAINER != null) { - MYSQL_CONTAINER.stop(); - } } public static GravitinoAdminClient getGravitinoClient() { diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/TestDatabaseName.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/TestDatabaseName.java new file mode 100644 index 00000000000..65f28915811 --- /dev/null +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/TestDatabaseName.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.util; + +/** + * An enum representing the different test database names used for testing purposes in the Gravitino + * project. + * + *

This enum provides a set of predefined database names that can be used in test cases to + * specify the target MySQL database for testing various functionalities and components of + * Gravitino. + * + *

The available test database names are: + * + *

    + *
  • {@link #MYSQL_JDBC_BACKEND}: Represents the MySQL database used for testing the JDBC + * backend of Gravitino. + *
  • {@link #MYSQL_MYSQL_ABSTRACT_IT}: Represents the MySQL database used for testing the + * MysqlAbstractIT and its subclasses. + *
  • {@link #MYSQL_AUDIT_CATALOG_MYSQL_IT}: Represents the MySQL database used for testing the + * AuditCatalogMysqlIT. + *
  • {@link #MYSQL_CATALOG_MYSQL_IT}: Represents the MySQL database used for testing the catalog + * integration with MySQL. + *
+ */ +public enum TestDatabaseName { + /** Represents the MySQL database used for JDBC backend of Gravitino. */ + MYSQL_JDBC_BACKEND, + + /** Represents the MySQL database for MysqlAbstractIT and its subclasses. */ + MYSQL_MYSQL_ABSTRACT_IT, + + /** Represents the MySQL database for AudtCatalogMysqlIT. */ + MYSQL_AUDIT_CATALOG_MYSQL_IT, + + /** Represents the MySQL database used for testing the catalog integration with MySQL. */ + MYSQL_CATALOG_MYSQL_IT +} From 9429d1bbb23bcb86d66d0bf360f4050f38d85971 Mon Sep 17 00:00:00 2001 From: mchades Date: Sat, 20 Apr 2024 17:10:41 +0800 Subject: [PATCH 083/106] [#2942] docs(mysql-catalog): Fix incorrect document description (#3063) ### What changes were proposed in this pull request? Fix incorrect document description ### Why are the changes needed? Fix: #2942 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no need --- docs/jdbc-mysql-catalog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index 9dd3ee5a449..6ec1bf615ed 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -63,7 +63,7 @@ Refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metada - Gravitino's schema concept corresponds to the MySQL database. - Supports creating schema, but does not support setting comment. - Supports dropping schema. -- Doesn't support cascade dropping schema. +- Supports cascade dropping schema. ### Schema properties From 4ea073116ccc8cb164269600d83d424d8e3dc5e2 Mon Sep 17 00:00:00 2001 From: mchades Date: Sat, 20 Apr 2024 19:15:46 +0800 Subject: [PATCH 084/106] [#3051] fix(kafka-catalog): Make the catalog creation failure exception more accurate (#3060) ### What changes were proposed in this pull request? - postpone default schema creation after admin client - reduced exception nesting ### Why are the changes needed? Fix: #3015 ### Does this PR introduce _any_ user-facing change? yes, when the Kafka catalog creation fails, use the correct error message ### How was this patch tested? IT added --- .../catalog/kafka/KafkaCatalogOperations.java | 12 +++++++- .../integration/test/CatalogKafkaIT.java | 29 +++++++++++++++++++ .../gravitino/catalog/CatalogManager.java | 5 +++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java index e278c3bd463..63261d176fe 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java @@ -66,7 +66,9 @@ import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.InvalidConfigurationException; import org.apache.kafka.common.errors.InvalidReplicationFactorException; @@ -132,8 +134,16 @@ public void initialize(Map config, CatalogInfo info) throws Runt AdminClientConfig.CLIENT_ID_CONFIG, String.format(CLIENT_ID_TEMPLATE, config.get(ID_KEY), info.namespace(), info.name())); + try { + adminClient = AdminClient.create(adminClientConfig); + } catch (KafkaException e) { + if (e.getCause() instanceof ConfigException) { + throw new IllegalArgumentException( + "Invalid configuration for Kafka AdminClient: " + e.getCause().getMessage(), e); + } + throw new RuntimeException("Failed to create Kafka AdminClient", e); + } createDefaultSchemaIfNecessary(); - adminClient = AdminClient.create(adminClientConfig); } @Override diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 5c518b7fec5..e98b918cf78 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -145,6 +145,35 @@ public void testCatalog() throws ExecutionException, InterruptedException { Assertions.assertFalse(adminClient.listTopics().names().get().isEmpty()); } + @Test + public void testCatalogException() { + String catalogName = GravitinoITUtils.genRandomName("test-catalog"); + Exception exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + metalake.createCatalog( + NameIdentifier.of(METALAKE_NAME, catalogName), + Catalog.Type.MESSAGING, + PROVIDER, + "comment", + ImmutableMap.of(BOOTSTRAP_SERVERS, "2"))); + Assertions.assertTrue(exception.getMessage().contains("Invalid url in bootstrap.servers: 2")); + + exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + metalake.createCatalog( + NameIdentifier.of(METALAKE_NAME, catalogName), + Catalog.Type.MESSAGING, + PROVIDER, + "comment", + ImmutableMap.of("abc", "2"))); + Assertions.assertTrue( + exception.getMessage().contains("Missing configuration: bootstrap.servers")); + } + @Test public void testDefaultSchema() { NameIdentifier[] schemas = diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 266786da59d..4180e4a3c80 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -347,11 +347,14 @@ public Catalog createCatalog( } catch (Exception e3) { catalogCache.invalidate(ident); LOG.error("Failed to create catalog {}", ident, e3); + if (e3 instanceof RuntimeException) { + throw (RuntimeException) e3; + } throw new RuntimeException(e3); } finally { if (!createSuccess) { try { - store.delete(ident, EntityType.CATALOG); + store.delete(ident, EntityType.CATALOG, true); } catch (IOException e4) { LOG.error("Failed to clean up catalog {}", ident, e4); } From 6bf11d164aa051134fffcfed57c0bdd8f6f7e57a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Sat, 20 Apr 2024 21:26:53 +0800 Subject: [PATCH 085/106] [#3069] refactor(core): Refactor to move listener related code to package listener (#3070) ### What changes were proposed in this pull request? Move the listener related codes to package `listener` ### Why are the changes needed? Because they're more related to listener, not catalog. Fix: #3069 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing UTs. --- .../java/com/datastrato/gravitino/GravitinoEnv.java | 12 ++++++------ .../CatalogEventDispatcher.java | 9 +++++---- .../FilesetEventDispatcher.java | 9 +++++---- .../MetalakeEventDispatcher.java | 9 +++++---- .../{catalog => listener}/SchemaEventDispatcher.java | 9 +++++---- .../{catalog => listener}/TableEventDispatcher.java | 9 +++++---- .../{catalog => listener}/TopicEventDispatcher.java | 8 ++++---- .../listener/api/event/TestCatalogEvent.java | 2 +- .../listener/api/event/TestFilesetEvent.java | 2 +- .../listener/api/event/TestMetalakeEvent.java | 2 +- .../listener/api/event/TestSchemaEvent.java | 2 +- .../gravitino/listener/api/event/TestTableEvent.java | 2 +- .../gravitino/listener/api/event/TestTopicEvent.java | 2 +- 13 files changed, 41 insertions(+), 36 deletions(-) rename core/src/main/java/com/datastrato/gravitino/{catalog => listener}/CatalogEventDispatcher.java (94%) rename core/src/main/java/com/datastrato/gravitino/{catalog => listener}/FilesetEventDispatcher.java (94%) rename core/src/main/java/com/datastrato/gravitino/{metalake => listener}/MetalakeEventDispatcher.java (94%) rename core/src/main/java/com/datastrato/gravitino/{catalog => listener}/SchemaEventDispatcher.java (93%) rename core/src/main/java/com/datastrato/gravitino/{catalog => listener}/TableEventDispatcher.java (94%) rename core/src/main/java/com/datastrato/gravitino/{catalog => listener}/TopicEventDispatcher.java (94%) diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 9f801b054a3..b7c89865cf0 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -7,29 +7,29 @@ import com.datastrato.gravitino.authorization.AccessControlManager; import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager; import com.datastrato.gravitino.catalog.CatalogDispatcher; -import com.datastrato.gravitino.catalog.CatalogEventDispatcher; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.catalog.FilesetDispatcher; -import com.datastrato.gravitino.catalog.FilesetEventDispatcher; import com.datastrato.gravitino.catalog.FilesetNormalizeDispatcher; import com.datastrato.gravitino.catalog.FilesetOperationDispatcher; import com.datastrato.gravitino.catalog.SchemaDispatcher; -import com.datastrato.gravitino.catalog.SchemaEventDispatcher; import com.datastrato.gravitino.catalog.SchemaNormalizeDispatcher; import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.catalog.TableDispatcher; -import com.datastrato.gravitino.catalog.TableEventDispatcher; import com.datastrato.gravitino.catalog.TableNormalizeDispatcher; import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.catalog.TopicDispatcher; -import com.datastrato.gravitino.catalog.TopicEventDispatcher; import com.datastrato.gravitino.catalog.TopicNormalizeDispatcher; import com.datastrato.gravitino.catalog.TopicOperationDispatcher; +import com.datastrato.gravitino.listener.CatalogEventDispatcher; import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.EventListenerManager; +import com.datastrato.gravitino.listener.FilesetEventDispatcher; +import com.datastrato.gravitino.listener.MetalakeEventDispatcher; +import com.datastrato.gravitino.listener.SchemaEventDispatcher; +import com.datastrato.gravitino.listener.TableEventDispatcher; +import com.datastrato.gravitino.listener.TopicEventDispatcher; import com.datastrato.gravitino.lock.LockManager; import com.datastrato.gravitino.metalake.MetalakeDispatcher; -import com.datastrato.gravitino.metalake.MetalakeEventDispatcher; import com.datastrato.gravitino.metalake.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.JVMMetricsSource; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java index dc8e1dbb575..febad0a8c7e 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java @@ -3,16 +3,16 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.CatalogDispatcher; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterCatalogEvent; import com.datastrato.gravitino.listener.api.event.AlterCatalogFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateCatalogEvent; @@ -30,8 +30,9 @@ /** * {@code CatalogEventDispatcher} is a decorator for {@link CatalogDispatcher} that not only * delegates catalog operations to the underlying catalog dispatcher but also dispatches - * corresponding events to an {@link EventBus} after each operation is completed. This allows for - * event-driven workflows or monitoring of catalog operations. + * corresponding events to an {@link com.datastrato.gravitino.listener.EventBus} after each + * operation is completed. This allows for event-driven workflows or monitoring of catalog + * operations. */ public class CatalogEventDispatcher implements CatalogDispatcher { private final EventBus eventBus; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java index edc98bb1660..5ed8b6462f8 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java @@ -3,16 +3,16 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.FilesetDispatcher; import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchFilesetException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.file.FilesetChange; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterFilesetEvent; import com.datastrato.gravitino.listener.api.event.AlterFilesetFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateFilesetEvent; @@ -30,8 +30,9 @@ /** * {@code FilesetEventDispatcher} is a decorator for {@link FilesetDispatcher} that not only * delegates fileset operations to the underlying catalog dispatcher but also dispatches - * corresponding events to an {@link EventBus} after each operation is completed. This allows for - * event-driven workflows or monitoring of fileset operations. + * corresponding events to an {@link com.datastrato.gravitino.listener.EventBus} after each + * operation is completed. This allows for event-driven workflows or monitoring of fileset + * operations. */ public class FilesetEventDispatcher implements FilesetDispatcher { private final EventBus eventBus; diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java index 16744247eae..8c3d79d7da2 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java @@ -3,14 +3,13 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.metalake; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.Metalake; import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterMetalakeEvent; import com.datastrato.gravitino.listener.api.event.AlterMetalakeFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateMetalakeEvent; @@ -22,14 +21,16 @@ import com.datastrato.gravitino.listener.api.event.LoadMetalakeEvent; import com.datastrato.gravitino.listener.api.event.LoadMetalakeFailureEvent; import com.datastrato.gravitino.listener.api.info.MetalakeInfo; +import com.datastrato.gravitino.metalake.MetalakeDispatcher; import com.datastrato.gravitino.utils.PrincipalUtils; import java.util.Map; /** * {@code MetalakeEventDispatcher} is a decorator for {@link MetalakeDispatcher} that not only * delegates metalake operations to the underlying metalake dispatcher but also dispatches - * corresponding events to an {@link EventBus} after each operation is completed. This allows for - * event-driven workflows or monitoring of metalake operations. + * corresponding events to an {@link com.datastrato.gravitino.listener.EventBus} after each + * operation is completed. This allows for event-driven workflows or monitoring of metalake + * operations. */ public class MetalakeEventDispatcher implements MetalakeDispatcher { private final EventBus eventBus; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java similarity index 93% rename from core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java index 7ca34093942..f02f6f516f4 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java @@ -3,15 +3,16 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.SchemaDispatcher; +import com.datastrato.gravitino.catalog.SchemaOperationDispatcher; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterSchemaEvent; import com.datastrato.gravitino.listener.api.event.AlterSchemaFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateSchemaEvent; @@ -31,8 +32,8 @@ /** * {@code SchemaEventDispatcher} is a decorator for {@link SchemaDispatcher} that not only delegates * schema operations to the underlying schema dispatcher but also dispatches corresponding events to - * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or - * monitoring of schema operations. + * an {@link com.datastrato.gravitino.listener.EventBus} after each operation is completed. This + * allows for event-driven workflows or monitoring of schema operations. */ public class SchemaEventDispatcher implements SchemaDispatcher { private final EventBus eventBus; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java index bb2735527d9..deacbba2094 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java @@ -3,14 +3,15 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.TableDispatcher; +import com.datastrato.gravitino.catalog.TableOperationDispatcher; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterTableEvent; import com.datastrato.gravitino.listener.api.event.AlterTableFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateTableEvent; @@ -37,8 +38,8 @@ /** * {@code TableEventDispatcher} is a decorator for {@link TableDispatcher} that not only delegates * table operations to the underlying catalog dispatcher but also dispatches corresponding events to - * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or - * monitoring of table operations. + * an {@link com.datastrato.gravitino.listener.EventBus} after each operation is completed. This + * allows for event-driven workflows or monitoring of table operations. */ public class TableEventDispatcher implements TableDispatcher { private final EventBus eventBus; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java rename to core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java index 9375cae604d..dd628a534d2 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java @@ -3,13 +3,13 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog; +package com.datastrato.gravitino.listener; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.TopicDispatcher; import com.datastrato.gravitino.exceptions.NoSuchTopicException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; -import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.event.AlterTopicEvent; import com.datastrato.gravitino.listener.api.event.AlterTopicFailureEvent; import com.datastrato.gravitino.listener.api.event.CreateTopicEvent; @@ -30,8 +30,8 @@ /** * {@code TopicEventDispatcher} is a decorator for {@link TopicDispatcher} that not only delegates * topic operations to the underlying catalog dispatcher but also dispatches corresponding events to - * an {@link EventBus} after each operation is completed. This allows for event-driven workflows or - * monitoring of topic operations. + * an {@link com.datastrato.gravitino.listener.EventBus} after each operation is completed. This + * allows for event-driven workflows or monitoring of topic operations. */ public class TopicEventDispatcher implements TopicDispatcher { private final EventBus eventBus; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java index bccd7e705f9..ae1a743c14d 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestCatalogEvent.java @@ -14,8 +14,8 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.CatalogDispatcher; -import com.datastrato.gravitino.catalog.CatalogEventDispatcher; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.listener.CatalogEventDispatcher; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; import com.datastrato.gravitino.listener.api.info.CatalogInfo; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java index 7f5a679d5a7..e9101e6e723 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestFilesetEvent.java @@ -12,12 +12,12 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.FilesetDispatcher; -import com.datastrato.gravitino.catalog.FilesetEventDispatcher; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.FilesetEventDispatcher; import com.datastrato.gravitino.listener.api.info.FilesetInfo; import com.google.common.collect.ImmutableMap; import java.util.Arrays; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java index 09fb7aab534..d381ff5bb16 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestMetalakeEvent.java @@ -15,9 +15,9 @@ import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.MetalakeEventDispatcher; import com.datastrato.gravitino.listener.api.info.MetalakeInfo; import com.datastrato.gravitino.metalake.MetalakeDispatcher; -import com.datastrato.gravitino.metalake.MetalakeEventDispatcher; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Map; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java index 9bd7ef49efa..da99d5d75b3 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestSchemaEvent.java @@ -13,10 +13,10 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.SchemaDispatcher; -import com.datastrato.gravitino.catalog.SchemaEventDispatcher; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.SchemaEventDispatcher; import com.datastrato.gravitino.listener.api.info.SchemaInfo; import com.datastrato.gravitino.rel.Schema; import com.datastrato.gravitino.rel.SchemaChange; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java index 572cfa4746f..bc0a754680e 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTableEvent.java @@ -12,10 +12,10 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.TableDispatcher; -import com.datastrato.gravitino.catalog.TableEventDispatcher; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.TableEventDispatcher; import com.datastrato.gravitino.listener.api.info.TableInfo; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; diff --git a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java index 557c52a45eb..e2d26c9982f 100644 --- a/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java +++ b/core/src/test/java/com/datastrato/gravitino/listener/api/event/TestTopicEvent.java @@ -13,10 +13,10 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.TopicDispatcher; -import com.datastrato.gravitino.catalog.TopicEventDispatcher; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.listener.DummyEventListener; import com.datastrato.gravitino.listener.EventBus; +import com.datastrato.gravitino.listener.TopicEventDispatcher; import com.datastrato.gravitino.listener.api.info.TopicInfo; import com.datastrato.gravitino.messaging.Topic; import com.datastrato.gravitino.messaging.TopicChange; From 9299e74ff616ce3abe577e1dd283a7b86ac3750f Mon Sep 17 00:00:00 2001 From: FANNG Date: Sun, 21 Apr 2024 10:19:39 +0800 Subject: [PATCH 086/106] [#3059] fix(spark-connector): Failed to run spark sql with spark-connector-runtime.jar (#3073) ### What changes were proposed in this pull request? shade apache hc only ### Why are the changes needed? if shading `org.apache`, spark package will be shaded to `com/datastrato/gravitino/shaded/org/apache/spark`, but spark package is not build in spark-connector, leading ClassNotFound exception Fix: #3059 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? build spark connector jar and run with spark --- spark-connector/spark-connector-runtime/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-connector/spark-connector-runtime/build.gradle.kts b/spark-connector/spark-connector-runtime/build.gradle.kts index 3920380d9d8..5da442ec62b 100644 --- a/spark-connector/spark-connector-runtime/build.gradle.kts +++ b/spark-connector/spark-connector-runtime/build.gradle.kts @@ -28,7 +28,7 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("com.google", "com.datastrato.gravitino.shaded.com.google") relocate("google", "com.datastrato.gravitino.shaded.google") - relocate("org.apache", "com.datastrato.gravitino.shaded.org.apache") + relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc") } tasks.jar { From f299064ad9bfeb8057a98c8975a890de61a99dff Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 22 Apr 2024 09:56:12 +0800 Subject: [PATCH 087/106] [#3067] feat(doc): Add PR review and merge policy doc (#3068) ### What changes were proposed in this pull request? Add PR review and merge policy into the doc to make it more clear. ### Why are the changes needed? Fix: #3067 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? No --- MAINTAINERS.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 365621ce3c4..174f3fa9616 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -19,3 +19,65 @@ This document lists the maintainers of the Project. Maintainers may be added onc | Clearvive | Clearvive | Datastrato | | Cheyne | ch3yne | Datastrato | | Jerry Shao | jerryshao | Datastrato | + +## Review process + +All contributions to the project must be reviewed by **AT LEAST** one maintainer before merging. +For larger changes, it is recommended to have more than one reviewer. In particular, if you are +working on a area you are not familiar with, it is recommended to ask for review from that area by +using `git log --format=full ` to see who committed the most. + +## When to commit/merge a pull request + +PRs shall not be merged during active, ongoing discussions, unless they address issues such as +critical security fixes or public vulnerabilities. Time should be given prior to merging for +those involved with conversations to be resolved. + +## How to merge a pull request + +Changes pushed to the main branch on Gravitino cannot be **removed**, that is, we can't do +force-push to it. So please don't add any test commits or anything like that. If the PR is +merged by mistake, please using `git revert` to revert the commit, not `git reset` or anything +like this. + +Please use the "Squash and merge" option when merging a PR. This will keep the commit history +clean and meaningful. + +Gravitino use "issue" to track the project progress. So make sure each PR has a related issue, if +not, please ask the PR author to create one. Unless this PR is a trivial one, like fixing a typo or +something like that, all PRs should have related issues. + +1. Before merging, make sure the PR is approved by **at least** one maintainer. +2. Check and modify the PR title and description if necessary to follow the project's guidelines. +3. Make sure the PR has a related issue, if not, please ask the PR author to create one; if wrong, + please correct the PR title and description. +4. Assign the "Assignees" to the PR author. +5. If this PR needs to be backported to other branches, please add the corresponding labels to the + PR, like "branch-0.5", "branch-0.6", etc. GitHub Actions will automatically create a backport + PR for you. +6. After PR is merged, please check the related issue: + - If the issue is not closed, please close it as fixed manually. + - Assign the issue "Assignees" to the PR author. + - Starting from 0.6.0, we will use the "labels" to manage the release versions, so please add + the corresponding labels to the issue. For example, if the issue is fixed in 0.6.0, please + add the label "0.6.0". If the issue is fixed both in 0.6.0 and 0.5.1, please add both labels. + +## Policy on backporting bug fixes + +Gravitino maintains several branches for different versions, so backporting bug fixes is a +common and necessary operation for maintenance release. The decision point is when you have a +bug fix, and it's not clear whether it is worth backporting or not. Here is the general policy: + +For those bug fixes we should backport: + +1. Both the bug and fix are well understood and isolated, that is, the fix is not a large change + that could introduce new bugs. Code being modified is well tested. +2. The bug being addressed is high priority or critical to the community. +3. The backported fix does not vary widely from the main branch fix. + +For those bug fixes we should not backport: + +1. The bug or fix is not well understood. The code is not well tested. The fix is a large change + that could introduce new bugs. +2. The bug being addressed is low priority or not critical to the community. +3. The backported fix varies widely from the main branch fix. From 72d2bb0915ecea739ad49cc6ba9034875892309a Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 22 Apr 2024 13:47:55 +0800 Subject: [PATCH 088/106] [#3048] feat(web): Add web UI support for Doris catalog (#3049) ### What changes were proposed in this pull request? Add web UI support for Doris catalog image image ### Why are the changes needed? Fix: #3048 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Mannul --- web/src/lib/utils/initial.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/web/src/lib/utils/initial.js b/web/src/lib/utils/initial.js index 7679b566387..63c7f878969 100644 --- a/web/src/lib/utils/initial.js +++ b/web/src/lib/utils/initial.js @@ -143,5 +143,33 @@ export const providers = [ required: true } ] + }, + { + label: 'doris', + value: 'jdbc-doris', + defaultProps: [ + { + key: 'jdbc-driver', + value: '', + required: true, + description: 'e.g. com.mysql.jdbc.Driver' + }, + { + key: 'jdbc-url', + value: '', + required: true, + description: 'e.g. jdbc:mysql://localhost:9030' + }, + { + key: 'jdbc-user', + value: '', + required: true + }, + { + key: 'jdbc-password', + value: '', + required: true + } + ] } ] From c7f4cb81dfbbda89b5f63fdaea7d8f6dde8f8ff8 Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 22 Apr 2024 14:20:18 +0800 Subject: [PATCH 089/106] [#2696] fix(core): list catalog info handle hidden properties (#3076) ### What changes were proposed in this pull request? Filter out hidden properties before returning the result in `listCatalogInfo`. ### Why are the changes needed? Fix: #2696 ### Does this PR introduce _any_ user-facing change? yes, hidden properties will not show in result of `listCatalogInfo` ### How was this patch tested? tests added --- .../gravitino/catalog/CatalogManager.java | 77 +++++++++++++++---- .../gravitino/meta/CatalogEntity.java | 9 +++ .../integration/test/client/CatalogIT.java | 6 +- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 4180e4a3c80..fb5f36ecae8 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -23,7 +23,10 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.connector.BaseCatalog; +import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.PropertyEntry; import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; @@ -48,6 +51,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Multimaps; import com.google.common.collect.Streams; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.Closeable; @@ -63,6 +67,7 @@ import java.util.Optional; import java.util.Properties; import java.util.ServiceLoader; +import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -251,8 +256,17 @@ public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeExce checkMetalakeExists(metalakeIdent); try { - return store.list(namespace, CatalogEntity.class, EntityType.CATALOG).stream() - .map(CatalogEntity::toCatalogInfo) + List catalogEntities = + store.list(namespace, CatalogEntity.class, EntityType.CATALOG); + + // Using provider as key to avoid loading the same type catalog instance multiple times + Map> hiddenProps = new HashMap<>(); + Multimaps.index(catalogEntities, CatalogEntity::getProvider) + .asMap() + .forEach((p, e) -> hiddenProps.put(p, getHiddenPropertyNames(e.iterator().next()))); + + return catalogEntities.stream() + .map(e -> e.toCatalogInfoWithoutHiddenProps(hiddenProps.get(e.getProvider()))) .toArray(Catalog[]::new); } catch (IOException ioe) { LOG.error("Failed to list catalogs in metalake {}", metalakeIdent, ioe); @@ -547,21 +561,8 @@ private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { Map conf = entity.getProperties(); String provider = entity.getProvider(); - IsolatedClassLoader classLoader; - if (config.get(Configs.CATALOG_LOAD_ISOLATED)) { - String pkgPath = buildPkgPath(conf, provider); - String confPath = buildConfPath(conf, provider); - classLoader = IsolatedClassLoader.buildClassLoader(Lists.newArrayList(pkgPath, confPath)); - } else { - // This will use the current class loader, it is mainly used for test. - classLoader = - new IsolatedClassLoader( - Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - } - - // Load Catalog class instance - BaseCatalog catalog = createCatalogInstance(classLoader, provider); - catalog.withCatalogConf(conf).withCatalogEntity(entity); + IsolatedClassLoader classLoader = createClassLoader(provider, conf); + BaseCatalog catalog = createBaseCatalog(classLoader, entity); CatalogWrapper wrapper = new CatalogWrapper(catalog, classLoader); // Validate catalog properties and initialize the config @@ -585,6 +586,48 @@ private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { return wrapper; } + private Set getHiddenPropertyNames(CatalogEntity entity) { + Map conf = entity.getProperties(); + String provider = entity.getProvider(); + + try (IsolatedClassLoader classLoader = createClassLoader(provider, conf)) { + BaseCatalog catalog = createBaseCatalog(classLoader, entity); + return classLoader.withClassLoader( + cl -> { + try (CatalogOperations ops = catalog.ops()) { + PropertiesMetadata catalogPropertiesMetadata = ops.catalogPropertiesMetadata(); + return catalogPropertiesMetadata.propertyEntries().values().stream() + .filter(PropertyEntry::isHidden) + .map(PropertyEntry::getName) + .collect(Collectors.toSet()); + } catch (Exception e) { + LOG.error("Failed to get hidden property names", e); + throw e; + } + }, + RuntimeException.class); + } + } + + private BaseCatalog createBaseCatalog(IsolatedClassLoader classLoader, CatalogEntity entity) { + // Load Catalog class instance + BaseCatalog catalog = createCatalogInstance(classLoader, entity.getProvider()); + catalog.withCatalogConf(entity.getProperties()).withCatalogEntity(entity); + return catalog; + } + + private IsolatedClassLoader createClassLoader(String provider, Map conf) { + if (config.get(Configs.CATALOG_LOAD_ISOLATED)) { + String pkgPath = buildPkgPath(conf, provider); + String confPath = buildConfPath(conf, provider); + return IsolatedClassLoader.buildClassLoader(Lists.newArrayList(pkgPath, confPath)); + } else { + // This will use the current class loader, it is mainly used for test. + return new IsolatedClassLoader( + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + } + private BaseCatalog createCatalogInstance(IsolatedClassLoader classLoader, String provider) { BaseCatalog catalog; try { diff --git a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java index e1ce59ccace..f745e6fddc7 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; import javax.annotation.Nullable; import lombok.Getter; import lombok.ToString; @@ -123,6 +124,14 @@ public CatalogInfo toCatalogInfo() { return new CatalogInfo(id, name, type, provider, comment, properties, auditInfo, namespace); } + public CatalogInfo toCatalogInfoWithoutHiddenProps(Set hiddenKeys) { + Map filteredProperties = + properties == null ? new HashMap<>() : new HashMap<>(properties); + filteredProperties.keySet().removeAll(hiddenKeys); + return new CatalogInfo( + id, name, type, provider, comment, filteredProperties, auditInfo, namespace); + } + /** Builder class for creating instances of {@link CatalogEntity}. */ public static class Builder { diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java index 212d86c3ae5..670091a8414 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java @@ -16,6 +16,7 @@ import java.io.File; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang.ArrayUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -123,9 +124,8 @@ public void testListCatalogsInfo() { assertCatalogEquals(fileCatalog, catalog); } } - // TODO: uncomment this after fixing hidden properties - // Assertions.assertTrue(ArrayUtils.contains(catalogs, relCatalog)); - // Assertions.assertTrue(ArrayUtils.contains(catalogs, fileCatalog)); + Assertions.assertTrue(ArrayUtils.contains(catalogs, relCatalog)); + Assertions.assertTrue(ArrayUtils.contains(catalogs, fileCatalog)); } private void assertCatalogEquals(Catalog catalog1, Catalog catalog2) { From 0fdbb5889b6a125ef83eb3ec7c28d8b9ad2b69e4 Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 22 Apr 2024 14:25:27 +0800 Subject: [PATCH 090/106] [#2917] fix: Use * instead of {0,} in regular expressions (#3052) ### What changes were proposed in this pull request? Use * instead of {0,} in regular expressions ### Why are the changes needed? Fix: #2917 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? UT --- .../gravitino/catalog/doris/utils/DorisUtils.java | 2 +- .../gravitino/catalog/doris/utils/TestDorisUtils.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java index 23980689b58..936d57a9c4b 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java @@ -32,7 +32,7 @@ public static Map extractPropertiesFromSql(String createTableSql String[] lines = createTableSql.split("\n"); boolean isProperties = false; - final String sProperties = "\"(.*)\"\\s{0,}=\\s{0,}\"(.*)\",?"; + final String sProperties = "\"(.*)\"\\s*=\\s*\"(.*)\",?"; final Pattern patternProperties = Pattern.compile(sProperties); for (String line : lines) { diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java index 290a496f165..35923bbfd36 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java @@ -59,5 +59,12 @@ public void testExtractTablePropertiesFromSql() { result = DorisUtils.extractPropertiesFromSql(createTableSql); Assertions.assertEquals("test_value1", result.get("test_property1")); Assertions.assertEquals("test_value2", result.get("test_property2")); + + // test when properties has blank + createTableSql = + "CREATE DATABASE `test`\nPROPERTIES (\n\"property1\" = \"value1\",\n\"comment\"= \"comment\"\n)"; + result = DorisUtils.extractPropertiesFromSql(createTableSql); + Assertions.assertEquals("value1", result.get("property1")); + Assertions.assertEquals("comment", result.get("comment")); } } From 494c2df9a3268146c7acec92ac84c3b3b8ee8be7 Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 22 Apr 2024 15:57:15 +0800 Subject: [PATCH 091/106] [#2879] feat(docs): Add doc for Doris catalog (#3051) ### What changes were proposed in this pull request? Add doc for Doris catalog ### Why are the changes needed? Fix: #2879 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --------- Co-authored-by: Jerry Shao --- MAINTAINERS.md | 8 +- docs/assets/webui/props-doris.png | Bin 0 -> 153222 bytes docs/gravitino-server-config.md | 1 + docs/index.md | 2 + docs/jdbc-doris-catalog.md | 177 ++++++++++++++++++ docs/jdbc-mysql-catalog.md | 2 +- docs/jdbc-postgresql-catalog.md | 2 +- docs/lakehouse-iceberg-catalog.md | 2 +- ...age-relational-metadata-using-gravitino.md | 4 + docs/webui.md | 13 ++ 10 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 docs/assets/webui/props-doris.png create mode 100644 docs/jdbc-doris-catalog.md diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 174f3fa9616..9a22ffd567d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -24,7 +24,7 @@ This document lists the maintainers of the Project. Maintainers may be added onc All contributions to the project must be reviewed by **AT LEAST** one maintainer before merging. For larger changes, it is recommended to have more than one reviewer. In particular, if you are -working on a area you are not familiar with, it is recommended to ask for review from that area by +working on an area you are not familiar with, it is recommended to ask for review from that area by using `git log --format=full ` to see who committed the most. ## When to commit/merge a pull request @@ -38,12 +38,12 @@ those involved with conversations to be resolved. Changes pushed to the main branch on Gravitino cannot be **removed**, that is, we can't do force-push to it. So please don't add any test commits or anything like that. If the PR is merged by mistake, please using `git revert` to revert the commit, not `git reset` or anything -like this. +like that. Please use the "Squash and merge" option when merging a PR. This will keep the commit history clean and meaningful. -Gravitino use "issue" to track the project progress. So make sure each PR has a related issue, if +Gravitino uses "issue" to track the project progresses. So make sure each PR has a related issue, if not, please ask the PR author to create one. Unless this PR is a trivial one, like fixing a typo or something like that, all PRs should have related issues. @@ -65,7 +65,7 @@ something like that, all PRs should have related issues. ## Policy on backporting bug fixes Gravitino maintains several branches for different versions, so backporting bug fixes is a -common and necessary operation for maintenance release. The decision point is when you have a +common and necessary operation for maintenance releases. The decision point is when you have a bug fix, and it's not clear whether it is worth backporting or not. Here is the general policy: For those bug fixes we should backport: diff --git a/docs/assets/webui/props-doris.png b/docs/assets/webui/props-doris.png new file mode 100644 index 0000000000000000000000000000000000000000..d412b96d6fba5dbf379afef52918e17cdf95e925 GIT binary patch literal 153222 zcmeFZbySpH7dK3a#Ek(WN(f3flG32k-5n}5q;!|uN(l%^NDWd$Hw-llC`jkfX;8xe zgTOEX0^f!4Jol}x@4s)YZ@ursTFeZbxz2T+bN1e6@8ACI6RD}LKt@7Mf`fxYrlfdZ z8wZD=5(kISi--{TPxD0!4d4yeU0dNUPWd1$68OW%+EB?xO$~Y?(8eT zuM{rLe?H6Ovf4Ex1f3T-Fq-A#yn;A3cjy_U-((p2oRYRXYnpgUG{lS930B2_l8xcbY z4&E;>)|XO6U8?%ipdDIo^{6r#P3mor%qFvgnjV3#8NP5Be30%taJ%Bep|Yal9h-$f zh49l(4^}r27j@}{QfX6b7frHIW=^M9{Mt=!&wyz2rIQ6qYt8f|0ctw`=K&n z23hyWJ9ZQe(d^W1Ie(BQqQ4ch^MDQo&N{qLWO;1W&ib!O`!-BOkb)7@t!C8TcI$%rtyq=Jfo+{NL{@=JG@ z`(<5lLfxpgFUTYb`YeMS`PR3`C}YsXtZI?(o*``h@GNNr9%2s|5QNcU5X&(w-Q)Ob z=cB=SeVtZnQCX&&p|pnaqql4Ee|w;ZCfTL!oa4mubR&iRSKiod2+chIebzeSad1oT ztUIbx{?@^z2zgvQ!;vY6+UtL4V+7h<$EGtle$`h%zGLtRDs!qP?F9bNrU0~kEUXdL z`@=}7&k-@IrxCA*` zAvpSi`_xYIze)ry!AC`)jn-2b@(0nrp9?I~{}%9n3;0i+iC_%vxFcrpU2z#M5bW&6 zb+<5cT8&hXS9=G?Zb-l26g_{T_wy&#Jac2Zo>BvYbx67aOt7aI%o9`1|54_t&hsm{ zFqUA`N?rZ(X}j`)unEpC(_Ez_@-l~=u{z)NL&v%7*0C}(0qwLK0-JZ-_6p}t-Xmgn zeX&UpCibF)N1~)Z_8bO&Ks)8+rH4t*o-mZ3gv+p6`&rD1`PB+T?sMn5%p7JL{VTZ4 zTW4&|5}&zHyXLq1TN2x8u4igI z>wDf9t*p2FY+!b&4l9*%`4MYh1-il9Sj0uU*l()H2bb=gi2K6BX_zq-YCY6f!V1B-@fS5m%8NHL^UX>Ma(w^Z zki$26)C%T##G;-U#=`dAOfY!DqQ9^_B`aW9`EUysJH$*ie1BQYlP|0Habom<#kMlO zej1|K02({WSCbVc7gmT8c2076r-q>S)$KU;ClevF%(|plIyw z1KgjJc8M#Lq`f%8@4G0oUBu1g`LaF>ZisDJpKDF}isFsyp%XE`xoINePG%0GL9CA@ zq*t^mo*b>~xb-yR(+&)Zrg8@REju}oynMt=dWr3><)`G$o%R$)Gw86xCT2C5)xbhx zO2{W)9Mo3ln@G6RmMPLJ&U-py+BHDuS!M_`KRMo7ja}yYk9~V+1k4o|mS^?Z*<59} zDOkiN(20AC52`BEN?^7tN|C{_NkLb2)s+*(otw0IsOJ`|I#eV(-wi5KUQhnAs{I<_V{7wE*R{<9ZFXaRfr?U|~K ztH9h`ipCSe3~_gcpnRJ2s89)x(zOpF6=e$6WH=OVw8kCbkAP9t7gjXn21mquX+^W{j7y{1xO zVv;80{GK{ud!g%Sw|>K{+Im)fL_*s)d^g`ux!Ud{y7c*8_R6X@V%{Iy*P4M}?R|x##Io&)}7N~)!XbD(e!7|WzZO;F8-h)$gQbI8hv z5>UBZW^pM(HtcwIB*>)MHF`WaynpdrAW_JT63<X^uraKl59!ZN3CH`)mqX20Lbd=j)^6u1=pw(E^t?en>qt*Re)_gcCKizUSS#bLk;mH=?_pJ%sY3ck?l-`TO5=wZt>S) z`{EakCR2x=t@MG_$o(Xf;W_XyCB$m?9fS>fF)z?Ru&I#HWv{7aWz@aweSBcLX29ek z##_gx7mHYlFm5<+e?hza#q=+K~ez<-&C)SM+T>l=Z*Kbtt;Y+Vb#0m^|u-M zHkM~phH7{@Fj$Lux4Ke3SzL4uW7b?Ru$#%@%Ph#@<4-s5cT)_(ZD9Tty8(lJeb@kx z7LBai1`7v^kEg+({yLlzS+H{RKsxMU5lgF+<{V+?_o`h!Vw)En;U_0l{DTA)bIyNn zimw;akqx(eeHYH(PSCt{av(GjL3iZEx}4|f?quP7>KtQOC3&m`?HLevT&&&p9}G41 z-8>#rGq09=+V?eZIB0ltFTZhQtLdPaJSJe*z(uRicF?n4MIkiFYd!neP^h0R{XDuw zX+z^sT>CzsGHUodxx~?Wa4ADHOdX1*zGK|-Fy`w!0x6HNN$yG04Qjj5O~Du1rXwKu zVKr4SF8#bZKHMc(Le=A?nX4RXWO2@x@tu)|Q_ImrNxnn4aOb9e0fzd1;HNNN>%RSs zho}+Ddcu!;&_J)n8#^kyA(1c12{S}o`jX+V78k?>){6^sG<1*mn~PSOEVwQ%2G)2u zBJNH&Pkr=B$R|u={~km9%jNEP9TLk>mGXSAX>^u=?TMbpmE=}s6Aya?eOBO%A@sGt1bkOaR1X_8gy!|!zCGhvelZc-dszspm1+GI(Dwa zxw+b-R*Z5+>d@KvucbU@2SIq&fJ({z`zWbcs0@k#PUIw7rvBgent z<8nNcMNJy5U!KkRZ8Ezhy&|WL+L?c8v_kv>EGn;m)td{Gac5J($f>*&Lhm{RTG8(+ z_Do+w!eOA8bq^DCy=pLT@Zqe<&gA0aYJHmZkNGK+K~~SpZz4w@f0J7~O2cJx2D{P9 zpyA!Vmz}Hb0wh-9*U7X;b3*Zg+c^ioxd%aFbq*jmi+rQTthx2mfDWJQhnoaYy{CP9 zEXEM2Lz>RK+<=){yIZhk@7gCiVnPGl!Ex$k0SHxkPWvlrA7uN!$SNF4oTLOFl9?j~ z1vA$RLKQO0m-Pi->>jP2IPFj@HUAD8nB{sW zrDUXsNnmeYpm8PSLZ*EQoIcYTRk4Op^Gqv?ea>0FMl{4~5iswvtsDmlgdnUoL$%2Y zq<4E~mv?BmwfpJwlpgc-S2Cf5 zPdq##SQMdD1L)mB*0-C{Wgru25#Idu9YM%q@SGMGc+31y%rq1Wa$eGu5jF$O`D~m#mZ&!GXw`$V&!&h`iK{i?1 zBTI};sFe~)i!v-T=VSy64VH&G;@TV>ioJz$*FjwH9~t49t?{0vN>ASxcI%2UP}V>8 z0DFoZYeN!2+;KsCEkWIn7 zFlgJ>A2jSQYuaA1L;vnKqp>)U?6*w!1;)2}L&6=Lcf##&bceQt3Ez9oOByj$oxv-=(diU|>RB3*TKv-<=Q5v~ZCjuS4?;sKKW<|NS8MD(~0v4WmQ&$o_=F9~vy_cVcZA6!$ z(`r=R&B-=q&4REg!3R^Z2SH7yPUIv0*hmLvW>X)&Nna;>$`4IaLXWZY7X|2SUq^Ph z2uwbkEfkw?P+0XEop^55y!sh_NFZG+MaS%eEt|YDQ51>T3~cZ@uvd8FNw>Tl7;{D6 z@+ez0XQO8sa~Ql*<$piS+F35~)1ogTNgJw9<~u|X)e>zkLj8U(_;gRI07nt zdEnft*_f8Qv6|L`R!-06B#@1X-MyTn$^Uib8kHYJ^ z3jLCHH?F>B9Nmx`!mO0=R?CON9PzB2+j)U9 z8!pK=@|K`QyGyH|kDQ@^CmNd2tE42=GbF@dMQ>Cf(uqC&=r{R3$$OENMgOILa8TK! z_2Fv*JMs#_Q_(!>w{2esR@r?#Eank;yHAD&BEZ(180+Km$l1;1_2{9`XFcxkkPdo1??7N?s{<9-{LnfH zL()8kEMj3N5UDnJh8ae<;%1sJdF7^D4P~t4k%GeakR$BbHxB$Z zn7X`<`(JX>)m>H`fxMdcSbn-QbQ)yb&=~vhy7LpkeVVB}<;b0sF0_De_J!V{L{jBl zS-rv+?*+I^E6GzXijir%WWyl&pu0W;LU{pGkSCGNr~89c?kSzZmr|Q@E+#WD6ljzux-i2-ctUyZvm8YvcRh> zO;ngzpRZf1$|xH*-9Pw5z}@Ud?M&&UT`Y0Ky2q4E6k&`Ic5Gr(JWzO2xH&U=NFF!f z`fcZgN1T~Wep%oVo&6Z0sZw}XcuMO=jWUFCC<{K;E4m813ob6y@!niMRyWA|jwN5h;6W5D&+WUk7fPF`e{fiJuLo!8?}TuP+|keyeLFTSdJ&1E^~ zF6Hkq3lMCansCad-n~W#lj8&wEwth4@Y29SPn6`f*FK}Ekgd@c?8eN^Y^0Adl*Olh zwRLpatp7@5!WA;%yxD?cP^GMbLQ?%FSWH3}RI2xPoi1mP6J7dmpgo5*HNF*vJ| zvOj2<8Ax-4z2KCQJ6-La{8cG_-wy^y3EuV}!Ler0NUK2`EG*Vl2r1^~b_)_Wj8DE} z1;8i`9gk*Bsm_sA%g%ri2NMPB^;$!<>eOU@Q zUVh>NPba!EG!WX)o z=c*;yAhmCw=Zrv(8!07Y3B#Vny<*~2EiY2!+Rx6YcU?Tem(u9sbZQ=FNd+Qvp~-cx z%>`EOw3kly84jJyhK21}XF+6yYa$&&T18W>(YaVw4<-yr55KThM?Ls)Uwm9EE}z|5 zmW_;LokAampXrRtb;Q;u6ARXFPNuv@3Vp!fWKHv3Jfp}h3G}tZO7ETNWyGCO7BYcO zDSP%&c}wP_>ye+HA(Ow%IyH~-AP7dB<-i5|bMOT+)@YO~&ax_T&ZJ(oa0H%`z0*{?L1;L-xT3_%P_FA1RTMTWv(8c1ZS_ z$TNheF@!}nYyZ<-`lm7JP$QAB_9i7nhA8|91>dW(d)lR>tegPgUwmslB_+D$rasR# z10JWdKyu4YzKqi3ME1`|Ein0K2^+p8uo8UWo#X5Gy48Rm?M}txPH=gCogKxWnU$|o zd{@ZrzPnCQC`ndr9TB-5d7^*mgweE?O<1#q8 zI8pG8(VAtU_(x2^>HXcEPk9dVdS_0GdEh!^wMhj_X~!GwoMjOEBqs3sp6t2(j?c~5 z|WTc%yo`J6j-~OM!8b? ztm~&B=snc*c#{^7TA_R2iHc`d2UFw z`6;IbN~6Qjw?#VkwH_~^!Z1d&FO0dT;|Y!ppr+ib;CT_Y@Xv#ySDurY2svy3NkYCy zS#xff&K$^$pz6Xu_;?j^n>TNj`3=S{t{ z+2uxNd3*#LqFpW69J#+l1)jAa+*@B!S(Vk7$f)o#w6X@ zE4oltj&+xcJmx7KYZbqM`tKM4gB_k&`0y1xLkR5`)-&W%=5NwkhiP4VT@6rE$xHmwJB)$BO|`64d;aR*dHHVq!naTRThyeD)fpueNfCBUB8*MrtdXhee5o(=^1U%Q^K785p;8APJY(;awtuLt64sZ9I zioxMVG2_Z1X5(XxD%?(dgV9CM2Lt#3iC5L4hkEjJ5gx`SzGrr-`?+W|Z8&PNIbC9> zBppAE5L$AD@g?nf4z$3cINi#1`u2HF4Dl%h~B0qYx?Ri^uN*&8V>q!g; zN1Nj!>l7o-CT90Lau~WpuE<9`hY6OJ+{tT+?RWiw*FFtdD80C~*jKd*pe|tv0t%4e zX~;aXJ7Ozy=1%PO=jWVE1U<{-!Q@m?Ik}-5atfaas9Q#nR(=#_=J{pBhBJM}6l)P< zIifShjnBzxcaJ?dmAs8+CGFn`cJ0*wokRuh#jfbrXjqRu0{mM&+;bHkvq>o7XO4#j z6E3{c{T}A&7}R*u`K^YV??coIZ^#d~>bWb>#L`PYI9d5ZNU^!Pv~LD3wBw~ShF0>D zLoXYqK?lk#=)}S1*&3JWZhKY3$GLv2GRIj0)EYWs2N*O4sdIzkC2^l@w3($2-Coj4 z>K_J3`ndzxG`+ay5q6y;Uny&&u-iH`w#6Fq{xXJhV>PciMPOw!Ofn3N!)>2EgN?g-cWcdYdTz2w)-yVxWMV7a8!Qb+g8T{ zHNV-lRbGW4gE5|b^HpJ(mV!VazlTQ=YK+b{)!|{o^&V?p+vevN=8|E3b>OBXhD*(l zksL>_?;g%?&TNd;=RdMafXlD#O$nhyj!eu7mc^cGh9&N6k2)w;%(B<4&X&ZhZyJjw zTifI*$@`ggxxonM-0Vb!1xYz^{RzGU{i&6h2b z>1zhZz5sB+vH&?TWn~Lkvu8pt%Rvid&Cg#(WfD}qCKIK;e1vTVpQ{GR%w}vWC!k%z zfz?-v>Iqvm^Dv(C2M2$G6?PACPCCC(T2TtPK@~U|&;B~?s#y1PJNNV)NMniAb^RuW ziQ+n%WfvAA+l!`olV#kt9!}^xry2N-vDRs&Qp2F4G>L#IJKM^BEl>#Q#-`V-8f0K5 zdX|mpNchkU6^z7ORL0DDmkc?*nq}+EXUT5%>7<4Zn56^?Kg=NJ%Gx$cF|CXGix0uH zDR+v097Q@<&zmgQr{70f0kha>R0r2+F7URaJ6svA{%}}b@eV`s6A3DBm0Q?d`ZkC0 zcp?^o@m6``u8so1JtV$gic+jU?~KQKI+O0bGf=4ix<4Hp5o=tPaun3BBCM#e~eFP#jqaxq5 z3w@p2HdpR?)S8y@wJ{im5_{6uS~iOv-S-Sk8IccFr#jy5bFq%~6|-UM6Fv5CV)+<6 zv1jX=JHChRPp1!JUOeXO8|jE9#Cm@{k)%Q1o+sZ#VNQ?WC$nklYjmY!?^6j|ErL%; zeYe2qs95FPA9E{?N4%m-p7&9wpkq+UmF1hqSjF?g&g}uH{B=>yib*c0Wyru~jYa)f z>ngVe^ZEYOUY5tXIyi|5>ho)8Az3mtg7O20lY>Q#<#bo+JM!kBz)1_&R#Uff_q-Hr zmHhK`t~8hK<_ueT$MGUcA3QHI8N1=PSs5mf6V=G5oO~FrK-48t%v)ptsq$901?O0z!zP2ieNqw4Y7LA2coN> zHp`hf(xqh)X*X4oF{|l|bH0xv54#tAk4(x#-;CLhtA?~+*9BlIh@CnY*?W+tl7c)E zW7>0PZH=Fd0cm%aOuJ?8Fr208;BJ>)dK4vRH~o!+=vY1sp<1aSXLrJJPXXY$)XJg%G(ozc=>ttQby_m4SoEay!(1@3#B16=f9+*Kg-Mgd2hFgSeWWAQBDm2s=%npUy*F}rh9H(BRq6y3mDvQ_ zMq`(cK<@X$sxhPXCX=Tu6ig;?Q1o@ry1A|r!A)*Q860}SO9i+s6j<;0=HsRuVbVh}$@16^pOqx2`nh>Wg<&nxLyq#by)xDtU-B2zAFA<{hmxS% zA}5M+UF6kL?O{=3&lcKHL^lRRbQ5wA^W`mTvLk6aNii8*x@_>2=OFD$Kjq&nB<-RUI=xU*{l;%ZyW(}nwHr{da zzHlE}-!xjn9%q3q9?5@nbR)aFJ0+^f6N)<2J4DZ7uFb zQ)>c4>08>23m2_NWS2;8+b=4*f07JZ+|=irXyr*=bb+7nk9>7nOdUzt&uJH-J6@8+ z>urq*(h!n0BI9age&#*f8i=yl`APv7W^9z=sCOPZF~;U%7pSX?s-3NivbnRS$TJ^y2W@KTW`zO`6jxm@D2EIC}5!?A(+9mN|0jITYen1G^U>yfj$ z?jWzaeWVMyS&2qj+PI2ocB*6Q66p8EKt$!d z!%XQBS~K`m#LP3kWp62LG&@MrU+C$g=Q_`K)?#QO`JJSmonmOf$=Q`29VOe*K{_`s4~VJikEYMF1#do_)?xtC8;^ z{!IPz#$-u!(h%tFij#A?Vc#Bf_O+)w3OZV0YvAxImWQ(<2lrFZI~1a#D^)6U)!PpY zk)$3*FjU5FkJYCXtOYf}TRn^mR`UC9Yq&_fOmR8n8t+YlbbT(jpl`E0rO9b`8%UJ@ z9^*2k5$suj4_$@{cImIQ_FT_Fa@ld7`Di$JJa}oG%JlAVk<}2-L`Z1bBb&ol3#O=u z48oOsMC@GF#}#ce!>dTOV)f2Z)@abtZa=n(<-8ug0QEfugEi5$0XtCiN}jQ`8wiO& z)~8Pfxj#>{q0V8+)2AVe1-)k{k3NA!6W&Qq#pwa~Ff3Z0CSEiNmJ zNQrvKq2zItuo3Y_oy)j6zJ8X2&XK-57`awrc|~ZHRo(BJC^MCe>abyTok)Td+q5QD~0BCDT4ByKCl; z7jc^npcRk!Y2_+v{DY1R!1o*;9vPhOMGWz`{_wMk@MblnVMgGUQl&X!OF4Uu5uWdg z{%x~VX6FNVQU{1WmxGNm#!U|~z8`XKcd7DxWhwzZ8BCt$zYNch^6!D2Ih`F!8Zd}m zk3JtQMawW;ZHZLwD!?F6V#z=FFrjvri3X?k>_knzVKm>LpNr#0c-)7pok99&+Ku@8DVo%bmqT|=1Ylq^I~AmoS98Iq zUxrR;Bablb5rIQ&#|W8dNj{n%gXYKOd@8PS^RHLDl>eZaYLeiptygG>jj=xSOYxSb zyNMEUN^C#&S1PfI$&2!>E=f_FdY(iQF>Vb{|do3n81a@unD)fLJlmA?-*?GDlJ(Q@(Na#(c2MUSz~c z@3d{!O($c)&(O5xZe~P0+l$(=o$Mm1hWF&~!){3R2Ji_zFi0duS>pPxk|3(!A2rP08AM)5oMh*Wp%@V{Lb1;Kvv1x1+kin~DhUbi{`A3%w zpyrH#yG7zbRsRj+lVQjq@bGI%)(DSFBsiwWEsa>Grw;qJBYb-axyOLj%zBU?Sik5$`uxc))&j_1oKfO&1^-y;pKYHRfVLZcuiWo{%QyV%CC3n8ie2%(q4pg> z_*dIUS)ff~kAL+KH3;_sDKBN)l1Tf$fBtvMfKMG^K%38+_2A{Rz2RS7UW#}KP{g$6 zd~}f*p`zdJDRYGsAYy(T%tij8O@at$3pZb({jX8@IgC&?Aa17Y z%v3=9q0Q$K(DuIt{B1@r{r_tL6`y~sNBluva6+$LEqrOH#dK2^;9|VZ>@T|cQ^o!} zq^dJzyXqJ3{)^zB{rT`r+5Yn8`OaMY_q82Q0HrZplN-eSgVF$2#qjoAV+Qpfgr5`; z{)exrM9=?0z@Gua-}!je&Hlg7_wy4#O8|uUsreG^PtpDFeSvWTg8DYI?|%dQWpe%x zS-}_ANLkxO?);K1FUdhiz_k}RVvmmGTqpB{I{5%*~rh# zfQYvqHk%H5?bg%n^i-`4;XgjzQUSb}_;LE%<7p%S zgk*h>uvZ-XSIP(lL@BJ_FCoFSu4A5XEy)A`3@>NuidDi6F?pW5?_!(u z%VzD!yXlhwNOigj+TLN0S{$~+a~&O-~_4kFINX#YGL%oWgl zj#B(VQ~lNyI}Ow^g-#r+sGYB@Jw&jYCRoS|^0XJ7X~+9bmx;0f^YGqCJ26@Yh+(X= zmHJQSFV-y0e%4vCP03uXPCB|ZkcpX|d}O^_@92O}My*%;l8ri#+=y#Hm2Vw@d6PL- zkWaJt-M1b(9NDvI3VV%uaHb|%?T<8hn62)YEq!c1w*FA9$3I%DfI?(Xeg4#Cz5gWN z&1Q&)zf!|n!*il}WhYJWQMu}=>uZczocGrb;*tx2qaPdb{glqCD)v8+{$`vAGyr?I z%~jMBCsR>QXN74}tG0P-^9t(o)B`p3ZYa4mIy@%L?p$t~VE@a|K!3$Rfsab~^5-wy zNC2{1w*VS`d{ND6-}@6-fVbe{v$zxUCU0j>d_V6Rd6msMm<9lqv5}^fCon3+o{?IC z@NFCkgvi>;G# zkSebkD69l%#gQy&lLpZ&WXt^mVkW4Ra#IHwO4@64n$?|(J7q?wqz_||VR5Zx)DyEl zk&NP@xmCp8!~baT7CYOW^5`GI!aw|$OIE;6AxoGq<{i3T#LmA|Dy45;43w=zs`qgg zkMZN?0$C>6trn#62r^_pCzP9x*mF#ktY7k_tp&H4_#Fjbp+c~EL#D|I*T6`=_0Yoc z)7b%_aMs?zf?`g&)UfJt*!qFH|$vM3N>IP5-r zH1o$Af$M_kC|C3)B9Y#k2f&lnX9iAxTz1%q{_<}(wWChv;lrGyRYUZ&eQLN9G8>l! zj{~qA7b@_{OgHE14EE{o7;f1Hw`U7jM=SM~<8o!z8^F!Y;7-Fb2%sjY6!e6;fkUZC z^JY7Gg_@SMc4OY#zUdQ?ZMfRpcarfDB7fC{_(vS}(|)>gW#eX|sn3b)i3gig;WN2Ejb`Z4PXX=z+Ndc~<3+L0{ux*v*P1Pp7bQ@w)V z1Oa)R0H*@-lbvUrrg855U)x8Wgq`MZ#?XvGv-S+SBt|rIx5bhwo`BX?7oS5Xn)NT` zJ*D0pl6LFxb9IP&ehz;1?SHM@rFOhDkUYMj@4;^})Knua0O4{S!yYn8#aKFwwCPDwbe8KDedP_ky3!MzmTP~KK*|4L-LZsw z-+35v&VIXCEtjj%KZ0=6x(})K@I;;+iNM8fa2gG=AKk_~$ny1z`qW!TbL8Y-^CI+Gzj0fIFR__+ZC;|uufU5m}_7x z!ypS3cKEj9kv!$oW+uI2rj!fUEep6x)74tJ{73%Y5(})9P)2Xj$w4Gf z8Z2KqE;i1(Dv~*#0U65tg}I%-)~vTv4*MHwTv9W=Obdr-^_V&UM8_6V?U+rFq7} zPD}t^Wi?&w`Sj@TkN1*#la^a*{Q4qXrMr*7!Ho0b^>rtY?eEG0q-ez|5=xHlB&rJu z$}Hj6eyLFgtf3#T|CB_iIfP3%lat4xTU${PgS?dE=I8f<0IPYIp6z4ghbJ^4H z<}V}74j`y^oCTe$TF)Q9`P%>BMffILmY|eHOa{XWiz(py*O#+_@#%ZT9qHJFgc<`b zTwBVu8j_ zwTr>F!@qb8q5Q-?TTYVLChOwonYqfqHmvJufTVFKi2iMti?GMta>{Kw>K*gJGqnEr z$o4ni1EA??0?{x@9UtbG+5filyi@|ZfAwYR4@KTG3}Zlaa+h26_+Ps9V^XYv3TZtS zDX%{iGhZUA14gMysw4d``}EzG%Dg&b_C&^I{vKlE65QnmG<59E*mv6eFVoKg7_k2h z@qerMyNLgPC8*?mwIbq-mzYAWOD2>6^iN5toWsdjm`uA&_a8Xp4>A3L9o!}y{BV~0 z1!^Mm%81P+Pdh3YKRyTeKsOhGe16|#Y0B>4I$*c!Vj{doh<+W*M>RklTQGN4X)4?L zeWRD*;2a>(QO_1Q16#o#bjD(AkF`jfp_W&t>lZn?3G_+^GQvNl5&w@N0R z66$E6ihxTxTC0_8>Z2}=9BDL*v^?{O5W%KwC}F2~fK1j4gthrfxvHRrhENaCqH&Xl z-6pivH`<{4O@2u!M*4J`dEy8Q6z>%h>sa^II~F;U*NWJ`xCw7QrP6-=rf6!b>z+d2 zNN!giz>OY-8yHMBVK=2282b?sQS*BD7mcaoTZ#YOrXUo zhqw$0-|EM&>Q6Nn9$X)*fH>f&Ou~N0KrAoI<+ep7`o~;JpA~SPN80Hq~mzwhqNqEH- zLDLrfScg(gfLP}caA4TZ!hGPA5*si9{K&WBjHuaSJ*X8OTQ8QY;tyIZer07nk$5b6 z{(hLjS-Cg$HUAHb=$`1{08MIZYafI5-?uR$89;7v6dkqQLYG-0mWa(g?GSfwz)!uD z%Jy}#l_8h15O7G#@n_8gwjY$T`v&RR z=mTzRw;p;ud1c<9Cv{!e7^q%cf8(@4Z|;vKEqeIG0)7k!t3YG(X|7{7|7O>(17g2E zP7P^C(o}0B%!RdiwHrsf(n|o;a@~Grzb#W_CjEN>=f8YmHhI9)?Zl~S^1!1HIw-5l zB?KtD!8w3FGmAacHLV@QY zQ1K|eY}VVj&m-{R!;#zy331!J$)u+?x?{jO6s7F`V!eIl{AwTftCKl-X_>nEDK1N2 z7k-z9SL6%b%h#dv_poQ37l$SS;cP zD~`?B*;s{z+1^i-TIL6CnFOADljqNN28^q9y9#sS1t z+X0J*nPG(6f}yZhkHaOSK)(pWX2D9i?`h?Kq-;VH0MC!3@w)#N#usVY12Z21uxM;@SX~U4ZVZFmsaTNr9e2`ov5Dhh1sURdUwpta)zi= zn?4nAmZhkJGoV&a&yM%Fnl51pc)h)Vjk6xz6W!?ugTEZyG)^jX7XIBMyiC6Hn3BMZ zff1(u0H}=E;{9qm7M|IRZ7iRd(=wKboHG-t=I&qcwI_L$X)F|_%HT4~&1OMkWBm=( zOT4(qOnMm7)3&@j;W8 zH+}#QP&tVsX*t3Dd^f)Lf?b)E-(G6JS;L%@PV(K`8`Z%#i2AN(BEe|OYS08S<8k1SJW$QA{TIaBJZ<1(pJFBr^1d%Fx=7xMck z2uIPpV)v!`Yu%v?%WV+a(RD;-V9qwKQ(`oGi~*n4t&3#@=RIzI^5hf0|B1Hm64JnGGAl5rXExaNcv-vZ z;TjJ*FHhNR%LrB`d=_B_08mL%C9G0wB`oXH8|xB)Z75ZH)xm!zz3opMGfSHek;+Qv z1TKl)34|)^OP$0K*1L304;>_SSL(~dJi)J?BEMcHiAgP0REs=^ zFTFg3v30NO{PICCoYg_Z@p2goHq;vbj73_1oI|hlIF|B?VY6K}DpcJ0r8C{EkHO`H zA$ZFN689VFR^v@-6WtXRt(go!n0|&3kFeg{;*UgPmucA~>YXbX-~I{_0HcOgPQhk7wuRd5nz%1@TODGqsg?hH`pko-oVx z(f8w6n-0WH3Wgc00(!qFDP)CCbMj6w)PnVJuYHFtGLmw6C~lsU5rAOdb_MVPaI72I zn>x_6E{}9@_TJ`5qeNQg?VqmQ&+RHgG?YMK0BO~qNy)0qRtc06qujgdxk*x@V;HioIru*HACF zKQ}sx8cXm4c}C4uR?w%e2#eRJw3OGfhw7*?LQpX zC&MsG0F-8eB=7qF>FNLgbvbzahv)#f?I_SDXB;5 z1lXodY?D~bY*1G$?!(8MI<8IiYq-43^S%*^nL{P}=I{rZwP#X|7+CXLg|I z`K77b1(%cK#1wvUr-?9Li`uXbj_$Ue_+s5Rrxok!}wkZl-}3HHn<1P*)M z<7iY?yBe?X0&%BLZsFoxMuZUtQMNl(|Ak>ha3n0O;(yK72^`73> zJs9>Ci(l{u_YIPkK*rr1 zSt~o#T>@C}`)DWj&+IaLw_*<^Vb0vZ8Afa-P;~7H&KiBFu~4y!y?r&4LD9nrX63tg zX+t0XdPL)8#>YE(si9Qo5>mjthqbaA@0Do^87qj)Y=N+1UG-{0^RdFLyYD&f9x{{vKkU6{R8v_SHmoRPK}AN2 zf`E#Obde?<8&yE*ML?;BP(*qc9Ym=rQlu+Q2oO3*fY3vcV(2w=2rZOAXi2^uXP#%k zd47NITJJkQTuVL4Is5E=*XzFSJ2AHo@ciy^@jj>xMXK@8|M(xV#pyTq4<_6``-ylP z_N(|&^imDP*y9>|+{1r%@${RgK!)jtrG@{;AXJdfGWPzW06+gJcbCFlnqB(r@XPXl z{_}8d>2LpQ5tj@=ZgJO@He~-n{`t`hFX~198AyU5Uq66zyj14-^!%?oO?lQq-kkqo zqYn7Jq0mcy4BwgfU+Xyf*x@Aj?mzB#w-AW^nCoSVM}*Zp_r{@l$AUnul1 zCql=~|7Dn{^d24hUt2^+!H~Zo0vi4iq=QF;qhkK|sD7T@zen|7L4AAr!s+{d<+WSS z@w0iq>*^ONTL2DCM%_-{p?^qnT_up@KHS7}>Dnft>E`l!9%ZlZ6bdffj#aswUGl#- z{f}(QEl?DiD8t_?WCO^i6hP(X^t7#{IjT+i%RCq;_&lp!5k8^U@JkhlqI_~d0UTHl zYbSyEj9&yiKLH?(p~|{9Gs9`IqoFbe_n&9{tyC{e4t^-zNqs{7kA;J-;mb#_Qvqw| zC))V-45xWDT08P?)JAs7ZE8Z_+_vhVxk=kW%6w1?TS0qXRC(!h?U$YNk~$Xv5F4Rt z=-T+Xy!K1y@3F#-AHN3+BWn~w&qqY0g|hY|uFQ;H%rHz(18s%x0Wad4nm^u&V$`&8_!{9@=q=67qC$=qiD>%WRoWy5`%H4=O1tz5^h2 z){AJnXng_IoU+aH83`J#dF(X*Fjq-lV0R~gu)NnvJ zW_>#<%Ocp8sei~Y=CbiCqZnbI&1js(2lpG}$JQ!*;W_>a?c(ova6Y29CsP^$H6#K6 zAWEjD_n`DqGWQ%PsK`h<$0Qij`Cd6<^Aml|$}g3{1Af9UN_9}$hp1}#Cc)c=n}VYR!k$mMrLvs?o$$@!|hMvsTg>4 zKC0Oo8YHJnRdcuNfch#R#>A#P*SG@d&HrTOFt&5fXh3PIS;-$G&F*CIc&2kJPs(}v z{C9?UV?7R$dV`t%o*iN^|6+Dh&jNKzIn&xdi-|QCs)oK^Pf}>U7{^hqXv7h8`f`P31Pt)i&K&F~ImwHvU2XL8MwUCiHn%m5lmh^T| zat9dn`Td6TQh%&j_CSRday>`$BwCYObrnCiM*+{5Mr$VObN{voc>E9IqP6WnF9eCxTc8*YyshF zdHr-M&4eGCMyfAbvbz-56K#gW-xwttf3?7oG;hH`(@aohAkR*4xXjACU+8V84GyyE zy@v^eqpx`OvSsK?Yoj}@&omzA^VI56JHW$T#^n|=1WhO{ao%aGIS%dq%Ab+Z^x;uz z;HMi=PZJt+OSx^>%babX)VYkj`Vi_|Jj4%-=u#o#%Raoo7( z)kPfv3M+ZdAx(3#gv=B%S?U(UVs zlk#_#l?M%^MeZ30L$1Mk?a^>_YP1}hGJ#^u?5Yw}h_B zhfvpLHGh}&aK&9?#-r0&tThOyvJImlLE09?Y1@2{;j2({odS7%G%z3S{J;M2!5>O; zI15^K8Bu9(%4Z|!4_(zptH$rQT`FTROmU_~_{iav%VvqWq-;ONAA)g*?hMjIr~lA3`u2DSkyCmN4HL0gA2_)+29bE8()@8D<0W}4oIS8>T3Eni zDEj>mqzUG#x+>Jx$g!W=VLesItTWQqy(5b4M@)jv`wgyr4}Ean4<9Q^ELOd!vpsBe zi!hE?wd%V2qWfI4ky{x0($p(JmZ~u*B=tZJ5_FACPK$_Cn0kov<%(F$zFT<9&<{(! zl3gTXU6;Gtjdnb*@Sk`2`BL!$?|cT+4$LoN%GRw0oP2DD@`R{Q2x0Fp@|X6Gc<-zs zPR;hhX_iJd4AVGimxfCO+9cK|6@zkk_AOB^2|rkjH;htTKYBE`%-y87r)QckD{#$$zgs_(nyQ_3vZ;-L(Isk|w9BU8unUlOOfh zotd&_m9^Bme_l4tdcdgeb&dhzVo%oKJ6+ppajSb_Y|?>xs?PhA#qap((~q^~I5SLJ z4xeI|IZr0yDr4Apt7G9oCZvv5KFPd8VG?f34~Et{QKqDtD-N~5@llI+Yp{7v`)`hf zv44;CL?KDGU0wLh$TTSDpO*A54Jj-DbI-dua-U%XH)YA&zGlRRR#nmM{;27? zUZkt8)LIR61xXs4&oi6Jxf)gkTGICE-_u8OFS<*E)Q_Fl9DQqL5{MzyU>e%leECmx zvDvKlElRooD(6>-W_tMSX0<+oC zTPr^X?SdVW6^bW?QNJ1k3#y(f?%G_XqzdZa@Hi&l&DcacTuT|-{`-IBKFE=qnTZgI z{AJY_9=KD3w6kOQKx6a-$HB)yEF2jTfzIc>wMUY1M_b9xWTVgpYH@rwC%9B z5!SIrXC@kgzcWc;jEY}u#Rf$-e3(k_rpZN<1iaTJR~~IlR<45H*G%3HeTJ30bz$so z!lY5h#OB-aN>c%aZk{bYi!1)ft#rLI)AfflypQrP(X{GN;(@cYvg>P`?TOn@gU^Y4 zSt?C8aT76ib2c8{K{wn9lQtaqPBwO1>hd_)Q$Z`uhH$~X1tG)NrUUbGwzj)1u1mo? z>^_+zMWzaq>f&A-_2MhFiMzZ$L?#Ton6WgBB&aZw>e22CRbu-%?AD#NnMj_Q*MA^} z9LX~*S!}#r=fCpQ8j~)|81-EWF+xJePB?HI)tc;+`3oJ!z^`n4aEh0w-eA3X+GIDt z{_gSg?e%#UtGMT;Czx+u07`w9SU^b4cUbG$oIvhA&jOlGz83OAnWF^#<}=@5@r+-m z5v@#2fH-+dfF$EHQ;_&ZpD9jLqT-%!r?U^)mvkq~xa6V-ewHrEk|QSP8@}eOq{rR! z5>D58axnC?GTZmrw#&D-Uv9z#e1%Bc>OMTOqQj4Y8L4j^t6lZXZkD%8y>Kl;IMZ5= zOxkkUM!gxkGhb|&pJ&#)v^gQ*v$gn+Uvg}rKrM)-J)SXLUHsz4!Y~sJBVWs%A}ySN z@5Dwl^09*#NeJ0DnMzn3WcS@JA; zE17`#uA#Q5)~1*wjnh}5{a63%ish0@soE`H9ZB$n*der*KP-%CGbPGa80Ab|I?R0L z^kb2<@dB4KrM?@QTU|v#ZLw@f0=uvH_6D|dwg|aWfmmJ6bX8T|^%1R^d&%p>C4eP{ z^gfCSc;cAPJ#|-5h;(U;LIy7+AKBYhzbudY!Yb z@p*{y#G2O*jn-;~`R=ll`6^fIF=1ppjNLZ3 zgfu9e2`kj|h$-%UKc&#G2nR9ulni_2`fIb1OkH@wO0{85dze1b!BL9zT}746cRoZ{ z>>E5SO_z~7;XTeI%7~M%$C&$`Gk16UUQHe_Ny@&cW!MfSGnSf)P=d(MQHb1-bStQF z{7uNNAoPL31LAj0^-%g8VcoWy>nXeM3hjD|StVm7r7(#-<;v;;_o3Dz1bl^4%~uYn zQEED;i#b=3_Z*fJ^+#T-`knGip4MhDmYPVO->5_gk|o@hbCaPO26PqmNf9fD<+A=X$1hXx*a7Eq&94mO9z zBUabbV_;L$`BfH++aU^kw?~GyAmscA=y?6Y+m|f)r+NLg3=617G@~7_8-AA#c{5OI z0rmNWzgr*54Vl!0{MgP*Rw$Ykh0ymzKdfC_)5$T$%!+d;2qWKk87LyP&|h$>5tnX? zXp1GD`8(k6sHEchsGRyuB<~O1#rY*`KA^ls-= z(0tA#2@cIld@@(~$`cp`eTl92_+-+PPYsnjdsbF?m-!q|H85xhX8v?Uk5#-PPf?A} z+2Q;i(6pwvku=mj?gmemB>ol+BX!I^jc=_uH{^R7P3T$C%N~TBx570Uy6wg*Q}#n_ z?d6*TlANwpItJ0ws{JYlM{$v!8qS)2MJ2-i;61vu7yTU}?FZ5pGA?c#egPSvx6f{~XG3}lHGL6B*hxKe79d#s+rO+utR#xj(a591!fz_YEu_M)UdMCMT}_3*={g4W!3_#zigA4r7*A6k z`NX2a{Cm#U5F+cIgjH_m*vH#h!UfagYw0Bzf1|f@EQh81b5A~1gP8bXt>R-$u1mki zh4`z)R%~L3Yb6uDyF}zPO=ZdDa%Co7BQa`AD+_f=DIIsx)VKaGOOm0^iikyo`Sf7W zWxJ-H+8E;GTB|WZLH1KTd-%W)I`WZ}uWGF|H}_=Bey8ZJuV&uSGb&`Dk}R-ra#r0yh){}mCWxM8IscYzJfgP+a_Lwle$Let|toKBVw^RQOXAP z)}9-MwQ0&8b0Zj1XGzR}5y3r%q{|H%Ffwcj(QOY5zpCD-PDZ-QoaX7xg05=F`cAF8 zO=6i=iDHjLc+}#AE1S#)%aSYN2MlOu_8bs!c7D%6K>O|WB0rsC(0elah+Q)-z9s{+ zvnJ53O2@1H6gT4Suwz8qpi5djvZa7+dR6ksR~zD5;G3Qxle!cQ-0n`E&%|^Y0Wtx? z887luz_=@UL8brpQpNagL3?Hiax6B_Ouon4NkNp`nq;VA+FBN};(nn zl9Z__Q4H4NWEMA(iaxs^S2>ODEAv)hyhIdcL5|1|kF8+t{26DV8hU+tsfUcZ$O5`+jhYCj_wljLlbVE^gwLucUtKwbJ}ucEyj- zko)p3)Q(B$>v=wc7n<-TZ{g}?eTbeb1LY;FbWf&e zn$n9+9Q*xOVC>!|DbHXWDxMO~=F`y%m7g(QWIWW*w}42H@uHI%R-)s!z;vFl%} z&|v2 z%IY=rj~S5dp|IDadKrEh&OIJN?uA_S;oUYqy#S{N&dk5*p=9GyUvban@8A8OYW?BG zffm9Z1^>(RCv|}M`nGFW^M2)`Ts@lWb!4B&{`~fFz6({{PK2*}eaD}V0}pWh_W@w% z{@tQI^7sFP6lG3IKHGd2A*>~|56Mg#JiciQhMyPrzdn{5b0u@(khQh->Tg{=JvuIT zy1;OPJ$jdE{3ozmGIuKzG+lJJg3tyd<4)s>8y;*m%yi%LLI4~erdCGUsvw;ao}NWG z)`JIoBpHXFLq}hb+z~DdeZzc%=@0jP;ej+&DyBFUQ`X)2oR8tL^BS%Sgf6R$cEIwT zI)6PRH`cXedanY4rKeyDu_d!0WNUu!_xXKj7)`D-@9t}JFUhnmmK=UF9^+Q1UZNdlw<=^axW(Hk&VdlSEqgyI80a25AM@-T7ADdpvWb<+ zsN|K{VIDG>I~#=zecF4Q!$*br-!a;#LybR)_JQH>-rQB0X(KY|9=rf)@~zn#`KZlc zr4X33dd8Em*dD&bJ=p=u`+LmfwKsAxpmE7Q(QC9G9P(jZYHZXctvfHY|1h5YB9U@| zN0ld7QNVxyGf!W5aQ~qCMP5tfSB5?B^yhu6=nfu?gj4JaB2V?K7ODh)72QcV3p^p>i1hj?-m2JC9vxId#m;K)boQVr zzHyBDYXtXxQ{guUa0YFYL|20#g)n9|u}98I;RH`xlA#0cj(Xh!XF`~!S{!|Cz)8qz zUrwLH5?;u3a3fyObo0TOwyx6)&wUQ&fFB1H6|adu|CyUJS)G?uJ3ZIesq-$gENT@V zfHo%iIkcSAb@>#=vM;CwT4zrK0*5nYT93y|1>Ij1Idn#rd6Wy!gv^HXt;ztT$c&Is zAxO@?l4rh^c~-*h!6S#JtslVZmYJ1+%v$}!ri=g6k>cMn#jwUoQOdBoSToTC})(sV_d!03$VPu914%SqQIsYo(GXe}q0*FRv zqkMY2u={PBfqXu=rhyqx_C*)0zfd!63Fvm-Ys|}1fuA;Xfldi$Zj4r!=T}&JO!;b- zI+APE>)%x$GSgy|Ck`$W?qZ{zs;FnwvT1SwuD~1na?@>LxI`Af*6)22WK^z5Z{d1a z;LWw^cp29497#KT<0c+uufoo%Z7I9umEsJy^CWa{RStV@OwOd-n`Zemzv*;d|2=YHkaGYh1I32mcNPd&} z-uy8tY*EpX34lPb-U&Dns4vvsl+I)~T*eGH%Dkgc#4o zhYTn~b)+Nx6&*+P&XPBXo)bceZoXakxrmV9n6s~roO)B#ub54LJmkDgRzh2-5yGs$ zS>%Tp%dm4p250wRZ(ppW&+0RoRi2Q*GVDOh>UQcfpV34PZxT1{XXTl6B8D{Py``A@ zX(oY-Fa5iErKeWUd?+abY~)*y-fV7u-gqYW(N@g=o%NNYM^6!Je~IrzDXf2 zB4<%x?};8gJ=;bljc}YO%r2!ma;mFgqP>zHn2DApdH%HDLc`~JwS&LBWGJZYIirmE6l2lM>KMd)wMS9YjW2zlLR<#QY!wz{pIz|i z?W*JJHPILAhVPDo;?Xr0hwhTyWG_!jRc|LW-u+p3mxm@KJ4^WZJf#k5!4w~xC}uVX zS@})_uOG58g260Xvs{P2+4FzLBnqwLJrcz>rYY53_Vo8pxV)_O8Z0~MJDDGG#qkKV zvmNWqiF5YZLOBmYRAGf1ItJ`KTPIi~O6^QkMeA<>^y1>D+xwL-Qx(KSw$jEuBZZ3Y zG0#h&gaogUuLCwl>1oaV3i1LXMaaDNPG=?oL+7rXLYc1Uha9Hkn?D?s7^y^xjkS(j zt2Lz&MRd?LGa2`rdcIj_rf?n&Wfqh|k1s(y9>>T;r=3i@H_`cJ9`cazb^P|}Ohha5MTu~;7c+~+mPe6pQA!C$9;G^a1u zU8UjxbZGWrh=hm~GK1hajb!Z*DZn9a68&Es$uBD1-VpaV{zL-4 z+w>OeopI_;{^NO2VpC>kEf9WWUOD67a^FzCgSFtL%U%zb6iN8iR50_% z>|!Ug&Xh&tsNdH3GJ>$s#Pp;jQu|xzzAe2glXQtF&pP1wAVr@KD>h$dv+=+~PM7)3 zd$!r`F8k{~!zM{9V;sZCVeQ@;;Z^RF$om$zi`zO=j+t@aC|6#W74*FHgj)bfvLtm? z?yA1>H=d(md8#GnZ~P?_5;)$ZG^Iwd=V85P8ethGaDrb(W!0_xXp3CZ7G& zTUG^$ZZc>Cy^}4;>d1Gp-q-omFwahAzeah0iGTmKYkt;ripVD_fPiKj>odXK&H2E6 z4V4KG3q{mJsyDRIT_wu2aOJbTChkgryC4EetZE50>0B>f9XZ)}C{=fcukp|ql{Q&P z3UckFJd6=$o}<)#(-VBJpueUEwZ%um{j^=>gjleUvk+nysGmzzuGc2keRphYCY#0d z3>g$US+E5g`TY2{V(vm*y&=vb?ou6`u~GwD8;4>lIV>#)Gi1?cPHRR-CcKUwSRR>& zUz2SGZDW7iflT8IM3;dAD-)Uc>%<0CbsybK6$+%WBYCAWHGMjDt9Cdwg-$DRyKiUQ zPa&aA{vqF65Eb@{@cGNJmv5EbocMTVUItn!IHC9=LPpkV0DIC^Y==~iwVGD1EhSi~ zHx%f<(c#W8E~(g=JA32l=);3L64MPZtNJWU#1K@1d!CblXfHw`F1C9jriDLP z_|pBoa&W+Jo5C+NpyeSptee+ti*_&PpvJPa?%aVDg{1lqcFw|tT}ve+L7ZLx+NOCV zXF(8VCLi+V)}bR#))&xSOLRs2?Y=crK9N*#IJjz# z7ed%~s6X%XGy)t05m1RXKhf!Bn>t?q=iRM9FeqU-MEuV}{$85Fl?zqY3{kK4B{)CT zQ^F94aR2zyo+8;_tLoqe_7_%q{{9{{{_~j~{Q%o;V|10>cUTm>!%yef(OiT2smK4k z(#8nNjK!rpd*Qf0?_8)1mcY$_zR%e>5dMkyJdFTL(0+7k4-Mq6{g|}@`(qV^04=S* z+)xe%N}Ji^&;Or4B9Ux~f$GB&ijPNL*5`S)kDL_->7V;R6RQ(b)RjH%i*ktE4}sFX zt>`&9zYE6@9mJ*`(aAEp1`V1bxRol#XTU3$Dc#9DgVfu@^iA zcj>+g2)4y+y(5e?Qak9l&27R`bC@siTq_|0M92v?;oSNe$#=iV%v1J*(8on#$uOro zeiX7_(Oi|^KwpHagVuip=dbhiYonCPE5AwVys6SIuxzNGNo2iqgf=j2#KkO_MLPes zbKyG1O%D<@mss*-m*QsE9)@}dP`&nT?0F($R z^RUM^)TFNLwm42?*SDDc$dC9@qK><{^~YO>P6U4#jpcn?hVQG|&Q`BbQcS)C} zP`cWHy4m-&6TqD{c;-~_qM!2r3da7;f2gAH-pH%N>?pv-n``U)#$yWj>lA0a`V7#> zHsyvZ&1|JMXFFGl>@nxSuTFk-)jOVwo)6t&YrB)D?z>w&YvhR6^&EK#=J^w8XJ&j> zYsFBqzViwWAiM82g1Qzf?hKra9bX=+bxus(#tl?(=2Yx{kw27GNM^wxBu{~AMSQp@ z;J8I2jbjNqyQ2vK;{?(Krx~E1bxSs|{G!uI7E}u4X8V5k&`TXzRlSwo3Pa)4QW?SQ zlH56n@o}%qos_71>;uEfQ_*cMYDEyYJl!xL@4zr`0}{90eTM2Bk|&x~EE}-tac7A; z>mk%Zk@aivayCbdUUXQR$3*s(l^OE7tYt52-=s6U=X8D9GPe=#WB64R;6<6WZin=X zOUFpTy-*aK2ySV6|4K-4+42xXa1qZX=up$wS7z&(vOEXkj!^nwVs{ksoG+0R$-sI4 zyspo&y)>uOvp4)^jAK~ZD}~dImVA{q4rB$jCywyPGWC@JV=3of74Tu5k6I}up}b4*KMM3NI_84ex6 z!(kI$yuxMc!@4LWd30p+hq|RzCyj2sE3RBtX6G$!4{PVM2_DSuEqqH2oV=0fbEQod zVat*1b7SZ2ZayU(5g_*CgpYo|I^A2b`eAJXNvbEmmG&gmv+57akUyE&=rQ2-da{ly zRH;>1Hwv*N%E_dAxa-oAcJEYt=cqk#nR6>dvFd6pEA7+PqoqiKO5 z5m)>Ht1Pw_CVb_&6643z8|#JPEPv?YhMbj)MSCL(+S+B^#){n6(>N;~Kl1GrBc~9# z<9Qkfx_d5_Tf^ek3_vr0b$0YbEQ2CP656R!%dTuGaJIKZJ$xo*6q{T!(l<#G?zT57 z=GW$CFWcIl?S|=wqoz?q&y8H}w|li_3p2wqo^RcosV@{t6D~sZw`*>SIt)ecmg-LP zH8R=T&!wn)<9jxnnS5m{HvlLtLj*CDtk~#(tYF2Z{z-+wTyq@0`vosiqOTZbFjl9Qpnk&TEjhGb4q&)g58kUsmZ;s<8W z(1j63+V*yC$a&HfI;nz<@N7XCs{sGa_wf0B*kR>cTE|0QpJW~S;@H`kfp(wb{3tJ+ zn!a097|betNme-9u?Up^q|QU#mKw)ek3dhhLl%e%`PeY*+cyMDbwjVEsc-Hq8ZV+4 zbJgL(-m97VOm~kXG;&k-Lc>XoR9tluaYO;>&$)R(IPTY5#a{#g$%!d$1Zwowp*2Q! z8LykZ+aoW`q@Lv7EdV?bv}FYnLboZ>&xs%iS0YEM6-L01+;C!)4-O>dbFxRW4>l%c ztqx}os8x5t$>^Om-58lI^pMzi#5enfI2+d7ik%JAt1ubXFd5?Q-5>TugYVHE9$$0Gvh?6QBvNEc$K+Np;N`X`X=y_a*vb zp?MpPFY$}dg_!pGKe;2vgipYjD{)DKWxRe@L0G4X?)#zRw|W5DB*>^CVB`y^X%tRB zdeFm{Q`JqfN|_@!GW)(+UZ~M2HYs|0euGGMVTav8^S=WGXK(es{e z!tIBi6C+l(+ILMWJ%M{$8HcU0NOoE6jpeSz$#E5O6ZsxdYk$oIGNIi)9N*a|q$bXV z?m}u&TxNk-1>rMijMVg(wX|O8&&vS>QdZN1vLu`5Mtz@~c~@`NrfO?#a zw1mUtEfSKEXKPt$v8{F4Y5XgU_oZEv|Tb+1Vue-B1z6V`6-7R1q6q$a* zzIxnkbt@{i+|@v64wxkC=j3DkAIk}kJRz0jw=>3}Rwi+slX;M80&aJ<(#tod*>$4Z zKCyT57%IFWHx>2RvFJ+g4|}W z$rhn>jTFG4`k5!9uxcj=-Wo<;W%6YYIK>(WLJRIii8&k!#&RD{(u`~KpJr4j6=@QwYM`Oh^2iyt4AY=#%i*M;PKUMVElB&xBzM1zOF*n^ODbQ6-S;oDo6fKsgSy)IHgA6 zfTb3aNH$oywLhUNi*YR~c_v!O_erJqhJJdFcMkFuk&UZz*-bnnDD z;p=E-4S*;YM;5V&PCa>eyk6t9sC!N6srVaxkEA>CjfKDIdPqDmxa(5|2pjK2Ehm=& z-QV0XeQ+-j1muhUS~G5lQK$EbExlfWkD1Rj?Gj19QtKU^p6wq_2J?r}Z(xNNJUxmO z&PDm(#%h23kajx()O$$W%JE&l^zJw@`Gp!zl_8Eo!0$!v)m%z}Fbd@TWgjuEdqagk zla3e)Tc6Wh_TTUPy9fU-$pLcB&S&%4%y3Km{>)ZR;V58q(Pc(Bs22Li-={oAH5zs~lGU8CjsNqyV>)NWI(7G&~pPB8~#SXT=z_mpO z)!wfs0BrK}8RF06$h$+Vk}f4%%xahS!vXhBUn3Ut5qtP<$9H{p1g7kN!snypwpC+6KeEPlw&x=zr|0)TLc^VjFfi$_^!7l36m9KH zFQ9jA)7<>UkkEq$WY8krpo(;km;*48_ATGAbJCaWEtE6GZ2NVn`Fif|d#717Dr|`j z1g{M!VXoGR#lix>w2yop4^wQy*R7bXx?~w6ZoSc7kE0ea)6WJ1=U}Os>ZOlsULn-g13uk*CAk)2LlYt(A!|3#B^jsG7a*rYD}hM$jGguP{x z&OCZobDup-x=s~9t021kMbrg14AApesiWN5*F(yGH)cqKm8D#w%Hn^5D~nC@9qXKs z@3rFE0KJfNFSBxB?ePDOU4se(F{JuLDF+)8yPxf# zYXd-mO>vb(ceIiJ`I!B{A~l_&5cn;n#cGL} zsGqTs-)(@a9v_zG{wGEX9>DtV0|4&(?-uQmn}4O~Pml3`wW5g}&D4F2Asx!I-D#t6 z!YG^}YbX@Pbf;y%fo-7%^yn|ZJqiMU2Ta=o{9-4fWj=+f9<;?XB(qBtF>;j}AfGte z3|A&=;^r;)o25}bD#+vdHvB-5Bw!}fgU*Oh!DNM_`Jw(gS9U;3ivE#mX*W_Pt&^>9 z;=xiIaOy2+o~#3F97Mi1cd zX5WIG7E9XH3p^+5C?3sB{-piN`uVN7591?jvDx7 ziDd>C{PjVZ`YnmL(=b=WvWh7LxlM50+*~O3%Z}&qCrs6@)3%*4x`5+c18e)}DzMKt z-!%3UKEUxOiCPuu5b=|CcPkN)%ruoce+wzke0i#OpEq|B^1~a764O*u@hl~=A`cy7 zKvW}gNDGV5ykX=q@YrBY`Iq!F;@SgBpfSPhul-iq&C8kbF>9j9So=t^x0NyKqn2TA z#qx0K%Y6sg@dt=a48a$+vj^)w%PhSHTWr^tF{L^Hgooq;NZ3>CgTGXn7&!kbV>+CC zFeVDT(7=7&bDUs^iY$B_hG!=WO zwRN*>@h==XZneLHxpC&C8e34JEBcg^ueR_bIe!jTyl?$-NuOU}icj9loT)ef13%ZT zvQNq>t6<;)#g|4qO~}5}pk!i{p)Q%}@S>0V)(T>p2k*fy1TKh1{PN<5k7|L2{eK_x z@235qm9z(%?~8P>fs7P8Ht4z(XoMitjnxL2ZRTDSbQ^sUXcykRKMFob#mWhg+na{E z{AUzzbyH?lY|%v4p+QIUX!R=#PYz1!?n92(_4PBVVHxS_ASCzb8pQ4Grl)A#g9@M* zbBjA_&@Ev9#(xgN2#Jj?pYc0XL?Pj9t{e($(3JHR$w97;Q)+;!Z1xHk$5Spi0<4Qt zBXz@^)YZ1+=!)5=aS}$L(vF?%cCNMPe+nDKZ9~p~&2D8$kk?lgcQ6fR7vgTPfRl*Z z4Is!jTLT5T@wlx#0K50)LR7p2BxlS@hR*_y&;z5#!{tS^QmalB=I~kb=BHvfNbBC( zx!exGUIuzQyPFS-+7BBGl*fQNm0P<=lsw$HqB%5>!UMj4UQD7*X1+@=-%ERUmb1VN zIMbEd1stcWlOyObFCSSM(aMoCApZ)ZWiu?(*YpI6SJqR7Nw1m8ta4{ttuOJGmI1Uf z^f;W^m{*l$cQ#vW8fv;YSR#vl!@95L0Jit59Wac}1mA!npT*azf&f%h5{ zg&@aEeA~q7np9^$om!fR1iNU)K$8;S+`;se*C&|IQW|m``sd>^(x>KaM5F{N)MsRN zC)?MIIMtZh(-Wk`HI_vQA^v$z^)Y?lpsx3sWv{9=Zp8bn9uu^g9W84C6YhCN8t00f z>gV)6`P)F4&12@=ethe^MExuz0+f6$H81PVAaW}4FBg=8=mgrmg%U+Sy(|MGT^f#^ zL0P~Td^FX*5-v9Y`g+|53z(^r3sMX21lO8T5s0P}5m`?+2Zap3~D-YKSvwGqa>J_}@I0FR4!(r3f&2g(xlUwB6q zuY^=GgV+Ej5ZAUvi`=cg8ApWMV=6-ukY+PWrE7i)rl3Q7eZ#&njV;Z8RP*%uE|22` z(}<%g<6Tx(?$iSm(rZ_IY|SV3p|yfC>Y2_^F($qilQ3zbDP?klB~6EVAi#Vr)m6MM zZ&(tzB0zc3qPn9;s;g|X9J`Xk6MD)l((0fF)+fsNCdZQd?1~$7WfgtJ6wu@IT!?9I zq^aJCLf-41zF_}Qh?x_Q~I%;(52X*@mDx+UCQVeFV|rg1C@oPBkV!cLGy`7 zewsNn&-6X1C~AN9@~JYZj!emwMtS#mc=_b+!m;=8x-;z+k8ru&^P+b>DoE4gQV$T; zJzMJ=vgGFZ_X#)jjnO+F?qMAd>L}IhodN-vQr7_faBZCEK$B8Gs4|tJWa}F^`o`?A z38b!j(UmMr>y56Uv>tom(a}FROrfIO*A_ubxESCH8?9usk3?lxi||lpbz~SQ5RHnh zRd;v0-M415!90-2uu)}>_z{DAsp;<~XF)B#@it6h;*OXJlWdv2i;Tn^;Gs1#CbFM_ zJXv*S5vvyxD31&<;EGmC&TYFnRhPV^xjni=kop)lBw@dO4vbO=Po}g0&Timo;PS_( z(cX!qEtoqIkiuUUoa=v@FRaW{gj=_{+g)>o$5h4AyIq|WBjbv^@5g=UF41ah4b7ImLB%7U&Z)MMlkpa-@4cUn zcA?{UU^rAhp6}%LqtE=Pyj6%mTcLKuLX<%nfOBm*^Qz~Uu^qc|?ly_V$+8wT(1wy` z**T8Zq}6OJ0%-z;PHXwOc~a^?rev-S2N>!y$F4%Yp-=l*SiK@B)|idwtNwh#+v4l{vbmO{jmybmGHaC&PKfZ zCVp}aKFN`b-hKk1+tpA?ZqnFXE8t8B0`>k_biRmS^8r$0$x#Q z?51rzR;bp+DYH4kTZR1;_CcBBO$;|AJ0~^=Q4X$3Y}B0VH0`K5xHU+hVC1l}wO#{5 zJ3ySes1+Dt>Yw$QiQQUCe(LVvlQ|}$M$dbwXD01CE70Ea)bgo^3Xl-nJmC*q5mT^v z_2ft2ERBXQZAY01C*N|5Nc=!f_C;1wnq#Kby@w+BBFU>mVk7b_^L)n zua<)&F;o~W_Tk`Hn-Ctdid2Sb&xP_K zbUkWJ4639f3hi_a!ZW{Vuob6rB^}x(NX~s+eYW&9zW)fZ`DF|7WP(Ir--^uMzy*X$ zd<@`>wVECH7_ZUhq75EQpn|PezO}OIk^ZRtj5>Qv+b6y-RLh`o7X);9?(>od^|l0u zAqcON`9IBeQJPJE8HaC~DYVUjcfXsPRRrckVNM5I6hP}~Wv&&|9LdOQukvl?Gs>(x z<|LSFkh>|jL0_TOtl(JWUsna8QmvI_aTn!D!>oQ4@J zP}(iCU+8K_j6YIhFTNYfnnFyx#*s+tf{e)H`?u>F7_9(PtoQ+b0Xtz=Y_2hUBf)*8 zsISx=8e~`k+SA;?G`MM0qE}6wjd1Uv`holGDbE)0nrhz(72lrf+YCHkm`|g71b9xI zvNA?iLwy^DVLZsEx5y6TDGu(fX!h}3Kk~XY zXgA|1T=Lz-_94kt4^baw-0+{6%v40nu0=ovTH2t~89u3N+(g_IQ@n18ao?RV+1TuZ zE?=9jR;+Hn)GOLp^dD)D_mK}r6`?{$DliE}{Q`cuI2_BLBS%yw#Vd$AaUPhGIlB1$=K>?FN&`*n;iJvz1cs<_6PUy(YcI z8b%k6y*#O6U^0)Uv7BU2ubRjBJhRWjl%|e6k#t!MmBjg8PLR3r0gC=3PjE3i79}7&n;^QIt(4gf+4KUK#vD$cGH*Vex#dx z-?nvRyPlSj_RC>ERj#_Uq}sGuZge;&#@Xk@EJlpYecNBEE@r3Uiy~ahV}0&ig~vu9 zeagou!Q|H{&zx_(XiQo&15;v@fEKAyQ(*F9BZZlUpC1&H<>Xtc1jX+%O)!(@ik(OA zlrs)yF;Q+KtbP1Qq8BMb58m2W>IA0e^h|h8h?*#fDu52s~ zGJ$5*Y)M;63+Gn-OoMx8d)7`?j3`>TjUErOG>sv#ohGT-!B5l%SZ!|N_C^!qkgTj_ zf};+8p?~Gj4b2xDO|+Qv1&O=v#)Sygpz&V_ZjOLW-KWsWFH1&Y>|D)&RD+h>D>3Z@6$P-|9_9i@BVM5(QAKQ z%kz4^uFK&;_{Wb<;aTSY1t5V_6ix-q=wEn33-$uojNZZlE?ZrzV+bIHr9wUoEf~oq zd#krtR?YB&U+;GLPO)q$Ar%=rz?z(bpp7&6?XVPF4U9||1wZ7N|7)&-U#{~VKmbl) z?uYz$$n1~cVjHkW0Db8%8~5w7(*r)cpmBTw%U=Z$1-hyQF2C`ozq|wa)v&?@5R?pA zIL-R^xD!9X5{|VW{2Qdj!k+>(<^s*S`(NwwF9Ty5@b_IA|H9M!Bd!zzxJIliBPV%u$p-!Tgyg@FdpSPe+~(r@nW zpXs3f@;K^`u-Hc+xJr-b6}qU~F3d+noPTZO&m-J^6Xm_dCmn zhi_@5k9d@AjfM~2XAPVGJYVfl+Z@N$@^CRG{Y=a2_j^mt8kg}ktscf#CCf30yBkkl z+ry7=C48UsqPrl-9@e{u zF#V;Ivuw5h&IRyKWqyD>`AWCFBL7p5;MWz$1cYiA@MEUxPQYy@D<)5ni zUGxAdV?6PEP{8unssO)cq5uXO1qaRi(^$%Za)PYbdv`;ATfJXjvw%jkIAm1*X)LLf ziZWcM1%v-wtp6-63s5MRwSQdlpT@EX)Gqr>daHHmpO&ovXtdy1*WU_%00a2XcK)ZI zKR@07-xmIF3l)AprvJNzz%BebIb+>r^|Q0w{KboAnoz$VRK1zv_WeehWI1llS4zDV(r`dw_-aJB8yR&l}XV|@~j^lz0EyS->WIulla7nqBZzN$b6 z@<*4}hIeFTylT2NQnNVC<9M7rnEx`WuZBnV^ZK2QVpohL>^h{XGlQIe6*hs&S6KY}<|UB+3pyN>+J?$5b!%9URo}t~8wWw5oK4)7 z;lHic(AA8wip&kY?TBP`a!Ugkq(D{dnte<@^m1Rl&lGuou+~>ZvE*35{h_}MQ6b~! zaegm=iY2(~sJ2l2#ceU$H|Y$#X>E~p7I&#jRaJukWOLaM*!$iRj!3po?5~sL30#z& zU;Jxzmu?r@m^D@6v3u>sPEcupdZxpXPt_By>^`!Fb+1ZEv8uL!1Rhd%$1AjKddDRb zz6@8{O^>#hUsLBebsQV^R}sy^8CYrEal4Mc+=5RdL7DSpppc%HFi_j@^A9cP+c1_7 zn2U?-PT9N6r4z~w#%IY|=GvFv!Y1sR<9?UR4}>#BoCv2&LL)nf>+C*RBU5|2v`TK={T3C9bpwSZ&f_bGhSd+E6G=c$_JOb(i| zNtd%AEv~a)bYvmkKcLb+tNdQJin1*i2Pj$+?p5%A z)DD#U$J%`5l-hv{pYBcnlZrfU_VfJT1P{>HD->|79`kSu{vWjiyZ*W-e`qp=5=zOQ z-4!j#zb@1tM~eEf-#?a7d_t)mI1n1zcIi)jc=69qKei9)AhI8@#`y1+4jl*>fz5`n<-@;OPHBsf0GfvmJQe{>)++} z|F-HchSUGH>dyo6zgzYH<69MY;W$U=hkAMVht3qqWDo1)nW@JnPDb|W_|Dypc_1fP zMAeUu9j_c(wY4vEefbE3N-Kde_*{f!HLS%6`*igV=WPeAHt&R2e-phk4BgZVe(y_C zTn9cQJ23&h`>8UpHlSd|Q%-SbG=41ASsp%q)2Py1P2UjrSp)I7vWXNhb>67n->t3R zc?{LU27+4pZt+w>#*&g=l4;Ac=Bn{?xvW!Xj;V;KnZg$gUSfb2}2Lg&O1pM zp&V>Y>t;X%XxZ4Y^(6yO&LV3DEu|WiDm}<33gG)YeWD* zO+w}JbCf+{7~{F*AUm@9X=GP7wru|Z?qk?Rf;zTuLd64q(o2ua#4P%ce9hF?aUF)C zWaioBwT%~f@J=K1sq0yZ#dzF_?0xtBO80&#=}C;$rrW~e^gv<}I3H;)yCC6M8EY>g z_5HH6f7alVHRnRN9R5qhxhM6$9t@Q~e?M>vL4bwQKh9*Z4}RQqKGq6}((21o-*cPx z^0DZow(rTRRlgwmk=xH9PIHem=6<*+IA{$q6~BGzgOuBkH_I=!59r!IUOy-|8&foz zrdD9g~M9 zr|liK#}7POB7gwdwcM6lQape2?O$O5L#c)*lL6ca4O^}09hz=dWnK50*zkm#*$;`6 zhWB7gBRyf^n!D3h*y-E7lSe@;TSHOAq~mo;kgV~cHW(-ThTUMsf&WfcwhFYkEK)AM zUmK56v)oJxX|NpJ*1iYlx-D#%iD!*X!fgN*p4OU*M?4(;+G`qQv1EOFl}N72+Y9JH zqve2c6&2iUGz2P@n`O7c{TF2qNRINefW6uN$<U^bikosa=@# z{FzhkG0pp4S6`nveYQ@=bsOtc`(<~7^e#cOiGP2C)YE~E)Ctdhb|rN1gMa;*wP3z> zeFXZ_QSxvqazpF*ae+%dqvO^7sX$K3Xl7-B2X1S(<3m4}G{bEDEH{1Syl%sL?<9M- z<}~}-o$r?`O=Uda-9lTMhLT14PSUmO^RD{yTDfah@zB2^!pX3P1&j6i*|jaU>uX1@ z!z3>NB6c02nvTyw(q}jl%L&fkBRP@R@p@W_<~*p_7f)WlAJaqCUs26TlJL`#HG;C2 zzx^gGStO2BJZxEdg?k34hnfh$gw+mow?ttJzJMg($jK@?wSJwb&DM!g*r>S!{$1H%rhw(NzlMPPo4!_y}*Ypf9}Crlxk( zhTfg)j@O-^n{`U(-X4KNJCSZs0o{&vvES|XNqz-}k~`e7QYC%u zNIjivN{DP^qbmmGzJt_>ZO_YNqPHEDC2x{bbmx8USnd^o!};>mQ|}DVlGgHFeRfXrLkhf%xqPbK&7 z!=6_IOm_IgmZ#G69#szB{#r%e8g65gs>@poeV^zQ1&drIB!E=;6^9D8YW&kr?fNi! zn4vv)n47M|r}z09L;m;|f|Ad3FBr$hc7kkjsp4E+LQWf66#AMxDB>Pf^LHTRp?FZ5 zS3uY3%ES!Kq47xr8y0A&HM8ta7AxlB08rfO$uJLX`7tagx=nMS7nYdwyJyk;7VwpB zPh~?QFs8nr#W?2;dHZw3oYTG=3FjKM9Nf!2QW9pUp%kwtCpb;ixu_BtZ}U(BV^-wN zRI7bj4~GC-hta zeIL)Q6N_|P%ewK{kC@+8Z-FSFd11=Q^+J&znrf(c)o5ha)8)jN$@hHw4yv7L#u~M! z=!AqgVzrcnYgKvN{Kl2DnKhnC)?hHFj*1FolFi*muMJ$?tr$p0yEG}#CfXT7P!&2l z@O@TpNjpyTVi{1#dhn$%f%skHP+|x+Q|Ixe%5aQM|8JTF3OHy*_7loA2c2AM*nKX+ z>qYL%n(1=yA6!QwpSMnDEO0zF(8f!~YH+RXWm7wFK~1iY@%F&9FUb2Oq;t0f%x59( zPucETZ#0oCfbzETtseNIJU{x{6GyM{)G*=V5V|{cX4Ne!Nf42J||Upz!&-SsO%Rai2^EoCxwJTa&ysNcveL2jXx6kh^by4Ib)^1 zCs9dqca->udRPA^K5bjtaBo?Wx*2;6rUErDS( zIQ#p#Ju4Sn;et@R3E#l25=V_}CSuG0p8x-0(z1_FIpE|{u=S@ z3(n~Mg%t08#ZW94Vh8Pz>Om`?Qye20UI%KY?r z2q?$z>_as$Y?rlO?6kcO-oM_*tC3;{E=uOIwGu|lu^$GUhkx*SE)VeTIGX$(i(v6R z;%fH!faUf#dOMnr($1JRtNPVE1hJ-{eHi z)ik5x!hYa%y48+kJ^ZemXAJ71?E&lwggCjLw%VmtQ2uhd@-#zkw%Uq+$D2a&rypvO zp%e8*avIOB?4`N{qFC)EhRxZ+#d8QeY=+ z2k}AZcV#A&nz2x2N{Sl-*VfU@@cUVRa=p75R$k=On|5xQMK3)6x$-9=7a}cltu}3J zm9cPUlDPU<&_+PIS7&uC&7x?&X2sn57G4k4g*z?<-RNucZ{xi?1>xUNDRGeET#!>v zbKOYw>y0mS57p<3#uJqTu1mS%maUPT@5h7y)*FrLW4+Uu%m~hEqTZmF3?5=5mP!#34O-*CEdFZ!IdEq@BHc z?o-g+*nb-4A7FQr=h@1%Cgqjes5jTyP~r zi}#;e1vq(Sodzo>{{gYP_>uy6dTAA${`;Ce{~!ryt(ogyFaPEbAHT?Q5BTk-3_tk9 zKeRfYqR?5JFLN)$gGM7=(&F=HK7>JE-av(AI6cJY$D)Dsv#i0czASII_EuZ{iM_hl zzdBHW)1PJILWz><2C5zH?bF^5S@vaqyj)pnRl787SZURdg*+}>&OnUS_4{niTETr` zrCm@fqkk`z!gbmrPd(OlXr|7yQ+Hfe%IY^ajg2Z?9YgebnO}H^$C16+g-xLob^q$_ zLkNqPMW-Kjxv6ETo>onCcyDx{ZmEHe8*M7T7n2H;uq`t3U|+117=@Z~qDciL!N-3> zpfXU+Te3f znS9{~=gp^7GpIMs34bozSK#@vzLmx|y>Mz?-u$Kd5P(`AcyA!v2$FouziQd(ad@(s z-XC+sg?)G~Ckusq5cgPd0i*#`YfdV@RDjVvLYTo%4?`vT9~0>5RRHq6a?PqpI8c|D z7vzO%X1XBd0Qd2zd}yKa7;q-->^Dm2er;W#A*)Sq&g%$n@xfBjDx@kp5MVDGnfIKc z_^7*Y(SlrO`u=Ut1lpZ=8W&MzSZsBPri~#5z(8hLBo{#>n2NM}JHI?+2htLmUIuX3 z37H?SG0SY6)%xt{t`J8U?Kp*- zOGW)LQSj9p9#bryYnFuwV{8wy)~^WrGkW)6tfIo{rNztmV|CH^TH6H;qek0eR&MPl z+m!=_7HOKjat>OJQwDtv2WCTd>5u%sy}}W?CEZ#`^#jE!4KO$8dbZNhl9HjC&j1_1 zIuz=<3t$S5Jv~P+NR^eUiFaKT)E4@8%w1uR5`|t)cxo8|47o}Ybj0Z$)`sU+MyY_r z153+%2328`hA*ndzVV3b2Aqw!sxc1wNJ8ZV+ zlg;0`Uit&}5%W1h6r1ZogfF=U^6={Mj5Kz6U(44_FV|ZT-pjH}nD*==7~eX({cD7IjBSd${PDa;=awM-rJl@2 zGbB4_PkAO~rt@AV=;rtNst#zZ5b+!BM;|OQ{9h(m(d7*NPxNMmoaRMT1Aw9WXhs?q z1!4ba<2@gOhp!Yh+Qa=muRk2P(pcGeM0eJyLFRCnQ2ngk!`FKBGP`>c2ef1$jmHC# zz~ki*m&kec=Wb%3BLK$8HSGy_I)THHLvCz?K~bF*joa%gerAXP@o!=OmcD_8XCDxc zHLyv1DB?gr$51thI+rt>pjR#t4u`vCDPp{6CQcgi2q?*v4et`GkK7VOr|!m-9a!=o(?o z!v$8}8{hRWnHqr5F>zNjf)sMuG||w2kPX+F%x<4MB4`A0K5u$TtWpX2`581;zCpAL z4?G+KRfijs9y)3&AwJ}3Tv&Qx$8ONmV*yG)6(Ng0`22YdIVe{A%}YWL_+0kxm)A=N zl+TsBwQPcFa%f2Hk6w=m3APcRjnFpuz`D>Lj)a^P*w^R1B+g`GNTe2yUd<8!#LRUj zwO_;ZKFYk7@Otrw{5?iZuP`@~?UsPaQ3*GsD<#zQ)4f!#QGisrFTS-7An!yG(&4L2 zV;ds7X=(QL0Q9zk2#cz@sN={!oeIW(*{of17%9jUfXI9A@tbaWhePuG(qI31*e=ij zS}`=+C2&52!4eiaICDB#NFvtNoxf4J$h93Ms-uo{NdhIo;=NaE-Zgy2jVQGv%*Ku+ z&v1)0*f>lwxv#2%*<%4wL5?$OU0y&%cVM_fkfSr8%{6Qd4T%)`Ve}}SU-$K^ZuniC zju4*qlN8J*-414(-{CXeQMLT@7@*_2-_qb1u|W7YZejHT?Aehs)9$LC?=vnfL8_2a zQH9tMZM@}vFoR$$*yYnzC{y9=ZghGv=L83yS`C3vRymC-^*ACtZGU0z1>{yFTdk0F zY?vYS!bkx!sOY=xmSGT*Y)L^MxAmnnMcX}KHWJDvaB_NgT5Dg z5Ov7v(h&$oGV4A`3KH@GoJG#yMBLER)_RbAt|-~;PD^f5DJYNY4w$3Lu+cktuC&6n zo4E3jyS}W#(u)UBEHzrx80}~G14vsD%H~UNG3X?BtPk2~_F^v|jkQmgh8Ly=CnJeIm%DY38&SI0V8&v#3pLF;y7*)N#; z6|KBeKh6OFGwfTtnm@S?Zzsk!X|_Ky8*C39w_oTa=26A@mpKB!J9@M2>Y@H>TXEm& z^?%KN@*)fM<4`8{9Yx{l+U1ADuUcrwX-B&PXbE$T_J+rOr?2h~>DINijwR~F`bZ;V zjmbSN&Xx>=$~^J52=`(|Wqpis#Jo>_+y^Oi(QU$qhWbvz8_Wec@dZ}-?N@I~0d)oQ zm{Fl^bT@IB*D#~_9|4DTA@=5X+u4No43E{mW9oxldgFABW;&~{SGH6`sM>wui4<@+ zwe*hcAJd{*7<_$3AME~(i5s*oaSOMW6^~WyYlThZU3{+K0Z=8{klzg^M)~DKNj1g% zF1P($0s{DIKZe-ts6!uUx)JHdo+*}ekY1QR913NeH+2%qR11H39@4u}bQV$S>(AcD z|I*Gp19`qpJko|znH;0 zJAi>RyIfPOL2nFm^CZS5)PcfZh|`-1n^rc_mFXji77S>jqQzTE8H#a5i8fQF`G24ad0bn7Mn%ttIwRTrc)8Qn3{@jf}C4|dp^y}da)J| zYO$TVd~d3)K=N^}ule7y3ft`#XXzWxwf%6EUK)aoReToeyLH_V<32yDGS(ke z>rP~kCC?M&;obfR1iq~BRSHATMvI!>YR;I&>G{Hi%w=18U=@J;(j|+Makg9c;UD`Q zwUp$+$8u&T-+D|)`P|FCWy|kOR5z)%Lo9{S6yuh(*hlD~67lLFtyp8Rq6q+g-sf5d z2MP!mgIKWin`Eb3K1d$UU+On-jn-Y`xZ?FxA@|N#zS^C|-HuxFV6k3DcpS@{Bw6b~ z<}BFqaIJCtjtvy2u4QRtAopd~m;O#6*-TIMt7fjgS?hGFcR0NL`>cB|U4oo<3g?OK znEKs;k+o5X+VqSmoWPfve#6JhUHg{-#i*k&6U@@wTYGorBxPQq3dHKlE~e~bjqIv} z$PvDI@pGZ$dQ^H4`F)#S=DZM4Ay)a?N%_0$-^YnN;qbJQeLAKco@4YLOCFv&n+9YP`Uv7w*SoeJ)JK@}n>btch@UNbeX%(BdRY_5 z8e$*t{=pnA?v~mPHYPv%*QpMA-pSZP{ED2{C~#9wHz8J-qX+3|IO?f{%PRJ+WJu3t zl3%PkHzQ_08nG{9j>^kQDWK*mrtrTATC zZ|tb7nx>9wWcvxB)YBBxJkX7L;zFjIa*C1poCfCfdJ+FvYY4yh((vwwchy83EEIB3 z?5@<*l#ImScw?Nc+=$+!iF}g^zBG7;xYLk%yhfs#=ZIHsfCzE5wGTVvxmjB^T;()< zRIjUXvs!k>P-`v$KL1G~_kyAp;)9$UzC#=c0T32eUjj#fyon2U9(*DpI9aI6>oJ}C zehF!WR8RaQ1JSW&WgmxXj+IB~)@Mi0-lR z8BA#Nk{vwPn2#}+Un|EC!WR|}$HxJ&r-6d`e5eoz1=KQ?@W+MR?CUuS8$utBoAc0s zJ0C{$=3cx&5qw{jfmGWJenKTqB;V*@(vVm1P`(5E@xqsB2^&d`J<9AKugg!8+fIL) zpRCSg~AFD#TL&Z;U?JP3ewjv-afu zpAStD;s`pBD=4kbBT zZ4n3NLze*^_g0BIMW|xdtU~Dcog+@Bt2Y5(IyEQnk=ePG5~+g4lYhk_6tdYfX!WxX zk5l~?1fqOxHUeDdx+rGUKk$ZtO9rs^4o^{0K%0NTtpEP>3Jv_`#lvIw57aBbJEMLa zad3qSX#YR%^dFyCoPpnd%E{sW!~y=(FmMC_v~XXAit2CODO?-{ew&2*6>$B%&F)!B zlwvoEit2COJy)osxM$>Tt-r%i3K>0s3t+d-bA;;m*-_YUz;9t&KQlf4q0MY|AhbLy zPfhjv?0^;h$@cZN=LRs$Th#E`na7MXUj+_#y>M=kl5I zEH{x|J+m=XUi`s>b#bkfFY>&Tda^5zOyH+8zYed@DF6-^`4-`JCAoU3s-DbjSW4O< zmD)$JFtM@?0e->06sWPauexmI15fB@Z@KF)>r)Ar&uW~T3$e?X-y z{ZQvS(`4-_A(Eq^%tnnSvez5c^6g5)Y5(O+jr#?5kPPR>0#?)xfj8UJU5+1TCuX|9 ztuf4Thzf{Rqw_twO;mDevU!JFQGj>ucvKr8w;ZF zGjuN&W9^CJwj*EZ7d|Kd+UyzNq)Ckh3!pA$fC744d+U+mSc4wNl{ybv_iuM4moR5z zVp6U>XI1c`2pB*r9HVa0JWF^OFMcuRUWm#^UGDcZa(kJ@2)CBi!fWpJ7z!Js7W;t? zgUdS1QB(W5u2#7Om>x@G2M1jU%~H}s`=Z_UE1DRw5JIF%OO>i>v&fW>kl?S$usoQ+D$1%_6pExO`o6ub39GJ+i{x$dr1%)Gt6mUJw zcS9H-QkD*dokx`Ra&4YxH+{PAv-M!BI=#Vs&4c{YnY$bim){!4v5x{q}!EIqduwXSerLv#gH+du^7JaUv`qq1g%XV zY$8f=V8D!+hmaGwXMz>?2gkch7o|`cd6GP?%&_kEtVOc31K=Fu(I*M!Swg#EwN08+ zPgNhoCf77kb}Xhm6J4PYcspf%)pxEkVYOK_YcHHZ5DUZ{ ziKfo@dZY?EDWW8ydX=><^J)fmi-ZnxifQ_!q%=@sD~fh@L^Qh1>$ma;4yShzJao#3-YtTu1=5CVc4nA&AQxM@d%|2x~Nkw?nsbSGy&x8sMu zAPA0Y13)6|h&ZG+H#++J0q zzNhwETIGj0q_H)b=(+dx(orC^st6=Gwpl(anB!!)Ts!r>sJ#f4?00(AzutGa18?GV z<>kral{+e_g!F(VuEUX1p&j-AO?b5JRnr|X*gdlEfhXoi&y0B^Aw4r_w-zo8dy6>z`tV=~zh{R`_AcKjnO+F> zgd+1$oM?=)@Nuia(+7zTYC!&*1(?yLn$<_%~{dXn&n`} zgTnw*yyHrjoOk}5wi3CjkgSzO_#FLwnz&wIyN?zN<8DHf7JcFU22J?KSQ# zrDN8t{XeRZp@btPA?^B!)G&IR90s#F(XBo*4-iL6n?pxbMDvUT%wmp?8h`XaPvqdDMU%D)eFw`vx3Am>8f zq}vKmge6OrS1Ezr8!&!Ig|7977DVf3uNlNj1<-}{WTrIfJd05BLi_T;B6|9Vh5D^^p`3ci20y!X;Q;t{ z0fUq5?t1$)k`*Sq6hs1f6>eD;QaQR7cFOUV2H^qaqX1>dk{?+2cEImf0Ad`+omx=26jAog4R?ib1JT+s{?#@Gn#rq};b(VHNlp z3j{voj=v&B<_X%p%ZXDFq#u|i6&gCDT!fn^KGsUy=woK71rOL| z0IZA#*4X*%5f;1KxtJUwLOV#+%{3|39+jssviDiYFZ`A;q41tB55KONmgi-Z6SRQK zIZ!!HhU&X6*B~4UOwp)-XbO*a2}kkukzUZe@MImuGt| zsUfOyOhiVJa6ilv?c2XBqx_*`3#2|DsiaJt`}XpMF^ALG}R4R_!?OR5@wV z`Ec<}k6w%~lBe}>_O$wf^xAjynAw$=Oe+$mIF6}8_OTTSPXDdY8F)LAjwjiY0Ckzo zD;7b8k1(EtAKI^@3`82|f|Tt}ZN6o=0Y7-}8ADLkGr4Y5-^q7&d9m{o+Qc&iYmVzf zbEGYI#+PEpU%z-b=BZd*y5EsLj1Jg+E;@X<2-4H+44LB{NjEO z0>-k`VKxILg-}t8q`5o%HJNeOSX0@5)WkuD)7nt)6vj140b-}>bj<8uFbLYp+HDk@ zr75TDs`2j69{vb8Y-?C64Ol?ZwV|nW#-~ji*W7p>^?B zAL2scTp*nMEue0e|HRR>q97*^=pwb~u%{=LL&wXG&Ano0^3*iu%hw$KjO?c~c0>Ms zdht8@)|fM>P9p#N)X0cPw9~fj)kb|#O`P%B6wFpW^R7kCMLZ(}`ovDSfGXYal?V*6 zYT^fvY;4_uZY9=kr5|FdI$MQ`KH=?ZbEx7DoacS+BR;(kmkV!%*HzJGMb|Hg7IS9o z?uohK{wwm0Yr?DY0E%L+HzPaXp5AG6=?ukXH$44y5$`1I_@@6N3#*p> zW!h4lOu$}&`7|cZwj_S9Gu2PxV{LdG?+f9c>G$b8(*7e7@%jsb(EBeH9zBKxu)pf% zGJ>8J*v`^E%_{af?k+pHFC-=Gh5q}ND3E7QkwOH9iR7Yq9c3alNi8Qd;$Lt8({s<4 zq$zdZGLU!A3KZOT(P0pvt;Ok;dLLs~qxcnySrb8Y+dS%e2&{7730SX^28Jq*`pNu2 zR&TLU!{N_U)w$y_n0rMh-H5_3nHEbIX76jW=rrA<9TpROi(#8h$R+5jZwZu}r%|+s z0L^+T>kA0I_Vez!~K5d)V@cZcg6VhPUN{=<2- zaS{_LNH$wd8FJAW`}DOU00UyXD@ZSAOC!aK8i9W&(jL@ebobZB{6OVS>bPfM_U>|| z&=j0mhUCmtFY5eGDTp>{8Nw_-9R$Q$)+7RUe>U%Kq(ugG_RrtlavnS68B=}OCjw5K z#yl)SrG?2ErDMs}uH=%8o^NyS;r6#^)kOu8G@&xn?1ql^wG&Rm5fBr*#yujBAHzLI z2m!+I!A4un9(7#TMLqCSz`;V-j^!sACk3T3Rz1bZM$dF}sHWdc!`QG$BU!{e&c_>U zU#PWtc2V>EhhA$#1miummKy z^}I}@Fx{?rI=O;82Evpy(%b}=&j8c!^M!(OdIl(`N>kJe{l06FAoe0>^i*?^t^sNEj<=M>*bD44_V)D_;)UV zAc(o#Qv7OOuyk76&^si~J-rtZKL=>N=QOyFcgHq_lO#NeALUE|Rxzs%dxlWk3tl%U z!xP$wyV}L6@y0^OV}+pI4~XA&QcH34KNd_6?q)6Y9F3CuZln{kaaG=bFOsv3w)7); zHxhC~S!pBTmN~d~TrWYBy#8z}TjCqq2X0$-rOq@~WvMIP7=4qrKf1jG|Ls9dp&XAJ z;Uuvydu{&M7j=l~4?Y2jet$fp(Ajzk+mYjkI>K<=t!sKcu9))dthnBJll-g^ zne;Z(H365?FTCu8F9>6vv0`F?gmw$2IZUje6ItcDR8mclFL_- zERZ9;U#?*N$i{>$54OKCcR`q|arb)9UY4DD9I)59$mCW<40tUsDKF=zC zNr;~5zg;AU7qmmMhP|j@j)w_dkZ>&kSmYZ5QRR{C`SPxn?dAA(ks<{bj7}e16>XA7 z;U(S%vI*$vD#n$@Db$clq~m*}zZB}{T~NF9{N4QiY?V#c@a+1EaZ_=bEcDKt6NFz$ zC9*cSCRg|cZZ*FmIJ4eZ;~MlNe`{@Y5nf5#ZDXHKO=M+vYi_z4@=VvA7U17v-fltZ zZY4@~$Xq&60D2N@dR^}Xk&aNyV=vLOnYlJv}krLP-EicW(93e2V;s)b_jD3c^rrA*2>= zU%(KV?)^HTa81+Z-D$XKmuY~t|KyKe7;^d{e^0W2bA98DXCV4z6`sQd`v6OM>{j~e zBQYzZq9R?JnlOt;C%y;?VhZ*9)*O`s{8Z;PEw*};8f&4O+z)C{Bvl_9iFr_!*8yXUg(QTnR?qa7u zgvsEKzlvog#_EpJ?JEU|#tK7Xw^+dpyk~;^(9W2$!V>;VK9*O6^e{~EG1az1)jIow zJvJ{eHhQlQ%(@p??trt#+@aLBP5HIQT;8#bGDM-uHO-+i6pEkrk%$4dwskS+&PTEj zj^9snZ(jO9WVqsfZ8=yOBcqpib+y5vEx1`y=gM@hpp--btsW+eO)Yc1B*Q4f5FVo6 zGLJDKW`UMiV}Z!_eO92PahoybG{Vyrj_bIQvfLkWqqZP7McGfcM_nj_{#=;TtGVq0 z{?`3N@|P`magB=!{o0&1o?i-mzm!u9dPF@|`yO4cL*Ufdn?2Eutefv38)Wxn&S?Xs zru6bFpoAO>-Abs5%x5!Ff3vbqZBkPvm5Taf0)l4Py6{J zW`6f(j$*e*R#VBWNB{Kg_x3}xw_3aU1qG|#sIr#mXZcNA>AtgHKUCQ^(M~G|7Ac#+ zAmO`lcoiC6Jv^8{EAIV^%n(NA)! zs&=)rYme};cD~Zk(f0tJl{F2Hq3uia#pL?&rZWcyD;M28Q6@0dKHl@`^bMt0h4tFB=1SQ$6mBkNj^rJY53*1~s6UJ(`= zF&7p79*~l`;kBsU109p=u})B-#)nxuIK#_h5gf(S)yvnrbJkR`)rhWSw$uzx+Dx(1 zi`}{2cZEzL{dVylj>F&@hrySd>H%}A>-AH+vvpa1a;h4oY)*Fl#c?v3{DEFy%M_wEhHy0nRq1eShR!(A|uO3x+lBD4tZKuF|lp&UpLj%X&Si?}rBm-iLJ+0q$pX zp+PsD!&N3b;#PJ>V|seNJfnLT$Gikhf?D&fc@sxpiW+$~M-3O3m{U@pfz% z;Vk3&1!;Yb^&a0u2N7sJ0Pm!qGq_2VVO7TyzXE`-|Dt$5_u(h8?4bA$79l~`+M+iuN1&)TQ6nX??1I(a$9dlYUJ zs=^x2MB{H;3@dpYUg_HIs=}O7KKeM*DAQ>`UCnNvZNA{t!`PI1L0&&JuZXS@Ii}9! z7f=3AeN>^~7XL;j=5+iw>GwL5AT1#w-h5FgS}m{Y0p~qJ2v;89XosA5=>Dp!Nnu64(*;OpNO;pjDeMe>E2z-!&x$j z=2&DL@y6?hbka^w1zT}pbXLAm-pxmlhHH!ttE4?%ao&Nj*DIn23nL(li9HnK5jk}vU>-D)7$WHwU zqS-p%SG|tRwHM^=g_zU;FUL@xKR_OeL(+aiN{rweo`8)X)lPg)(@Lt)qteHl;Y|&a zctE;+Qvm8SZD*T)dg1#CVDegO)H9b)psMcb2Ps%m({~4Fs9<`6qVexqKbu-2B0oxH zv#0bh81Zs2D+#CTM3kHvEuFo&)A?9wO@Grw?D3W2Sz0rUWfV`>;tMI_s$L?J$D)n7 zqA4Otx6HYbSye#cyz3L&sb(;{O2q(B{&SLX9*-4FeriIoa7StHx|nbd9LE$=uf^)A zBJTHNl-fg&+!HiXUJgaN)C;u=$F^cB9J%)&L#WgwV}D?Raq*ANJ5xe5E@_RMVQ#Gi zpPIyPpVo`>dm=jfb~p55s54Ju|-Tr_;AwnqsxG+AZzQ z#UZ*>P%P;XwmTfsPh9F9c8k>Vcd6o5;UL3?;Te11fQjB_v!f6`7!Nqtr#2;Le!d{N z9-uQS)#KhrP+U89)r;`eKnA@l)dYT;;TYO*`u6BrNOfQ0w|uOZ`BQAzfajfCt7c*f zXBn-O-sk5V#Vd3BZT6%0Ol=xcokr(1q6P|A9@u{D3mcebG+ubVd;O9k83-A|C61)) zW?r9psciQs_IiWk(8@ghqat2_p1L-x5`1ur(MP{pW0aDA>K5zlsfYdzuok2$yRJkP zL7r@RxRpl!c|MH#KoAHGsJq`|SkCnUJ0-t0HkQ{Bn5O*_c`o{RR`63&COIRzRfs>< zIHt<*$`HZ?EwF=VwukwW!!-k%BW^q%PEu+yw>Oyd=*a1LrcbES|&m(w#%iMxWWj zFmZ$8<^3NM`EYwN)Yrpxs04l<(}#F;^lUIG(Hz&bQXu187+n>LjG&i$+2q!XFzve? zMcuI5_iBVf=gGJPaKP*ys0fA6b3~8NU&SncXrkYG^yzIVn=EVR?C3}hs;baaX)Poi z1hdd(57ru1P05;MMTGYiGF@BA2VIq6SZH6#9qj) zdv1(Omn3R}w;EaP#S>Ny3v=w@xok#b+oK|jrpCoZZB9M{2`sAT67H9ZaOg_GlQ~me zTf4diSj}=V@I8?PPOLQGaq7;V|b(gsgw1nx@3l+WJl%|=vW#H-9dBzZ6_Kfa!5cZ4uKWD*BurbL! z#DyC)Rfd?%)AOe{7KL&Z zsc03lj=I>~uY0?t`TAqF;=3uSgS*w z_w}gM6w&TV!tnK-&K#SZ7WAm*a{f}?wvX)%CB+e&yRL}CJ{|$bo*YsdfW|)=Ru{C{ z82{e2y9HCf3B0^^T^qpVkF<@1-{ur2j2#4wRjQ=>4>jgaZFv%19akM-yA-*$y-}>U z&wMcBDW{T5Z6vCD@Z86MM#Jv_h2F2*^{2-_@uxndh&VSxaIi#J zQv~Y==Sp74&+U}(qcNfSjJ6LG_+h?RYm$&}dPNeM znk_m|WkMv#tUuTdzJu5v((}sw+MWKqaq@76P2}u&_qqeWD$n>dKmq#TzAGv%;$gUmykGHC zk|92{T9&s&AYH}v+|>x@wTl(FE8C4c;tIBFm3Rz~PmtvkH+m?moo*O4G{qgp-RK1b z+a6cO;3qE+2{SMwC0acEV6KiDK ztoAqAF-&%LEWOjH_kBXFU~@JbPWJFvns)n1Me)^l5KcsMK0J@T2rXq#JSnIetB5ic9xFc zEFy&z} zqX(3_wY@C-m^>hrUa$3#KB?q%9?`zsA6Dp5b|W8sXhUm1U#BRMkx3I0yBlOzYiX{b zq%;b|C(pyLj;?#RKP(cCAwso20|ohGSR~GkdYFrnm0Eq?6Q1P`_LINth#Zt1@m`8$ zaFf4ll|-ps<|VU(Qoe}f`^V*BJvQ*i?|}N6kg%KqCw;jbl68%p)ONDQPix;+;%{R- zB`l9^Wl~H4ulsIs^1i9u{M1$Ua3cTNQ<_g<0^Ea9kvhpXp0ymhWXeEJjDf5=_&D!D zvHTfySQRP|VW68lObb}{K=Gy9abNwe_toG@X;5YjuP)-LqqoekU=cY^R_hii zFXwcl(p2nL37f9)%gPmC#KR{pNiAW(pIY83kSP(>6R0a2rP)1%aAo zGu<}ul~&fiWVO@7dfg=RvJ(jDQt{hmqaq=7k(39Vvw>Rqo@1Dh zqX&A+TGHk}d=(ucfTVxni?k8%$E(s^hC-Nk+RpRc-?-LO+2;vu-hZ;DeBV|1u$bgr zkh-P2CyI)clo_F+Og*7mw)MQKNGozniC0YQ@v$9wX)jsRT~#92+DPS(+^K^sy4*lmj{db1#1n@}45q<;V zhw)G;)=CGCVgm)$P!Nv8Wdli57!zj1QC_a@+++xwr>ds9^-w7vD7z(>k@hQNgJ=e<9{rWOFLc8H!cWs0@BRKv)Tg$Kqgg5gK7y38&t{)owJ3daco&m zO={|CIznBc>8>&YL*K zsQhpoJb?<)6sLt)q^Lo2(-dxp7SFZW@Ez~YamB3FFm0*e_C#nq#XoR5KxU1HDt+k) zl`$0}VNEOB-AJ}_gfXWVwcX?@ra)_iD{$`tosGQdBagTggS*W_^f6Oh^|S%J$OxSb*n2uqiI$Y@RSEMh_-ycC1?d zeD9Bh-c30X))sy5!so@8DpRwN64oD^Wx0+GIo#mmI2q%41fPlN1YiK&KO#9H(5_Ffux~C;ji4p>^jWn?O|{6i*&SK9h}TA*icLsg6#p1yw;k{uvx~` zudpj;c>FhQq)#d~P=OahHJHh`S3sVto|o>gWIR6As!(pTHq?vpzeh5l4~U&yQp5I3 z6~Uiy%!xdwOvr6@)Xqx1j<%$%%E>S3!dVL0%IYEh`+u~fN)s?Md8nbd6(BR&4o^VMWWwOLy?hvx%~QxKDZQ?1eh$w%O!;|Z;94MISQ0Mp{g(*DHX z@y^%HKT>jk&BmV}77PH9>N_u%i2q2){-cg8dE(#f6u?8V$5DmVLzfy4|LEJ_^Q5yQ zfz0X&j%TI+#avyK0%UHco>e~o-MU{*{PRPE5=eM0F*Z;`fvKpr|NTP|#qz>^Xl(xH z^#9ohRCJ#YAPH6NZU0R5U(I<3hjidR*NzQQM9Js9fJf~9v&ylQ% z_t%|*IKL_l#>d{3)}#Qn6{lxW884LbSCb#0`jWl>`|IaIXeamo)}D>;m3~=u?z`8A z3&!Zb$4TjTSJvHaT^E4TW&MQQTw~i}qT^rtFSz)3Yu}WBE04it;;#+W1e1nmj@^ zjSzbsD2xe^dExxr7y6Eju+IW`s-~yQ&}^->af|!PMT-tV(U-W(3i(wEps4HfA#(Jk z*ji)*>FOFlfU#9tZ?@~xpcd!$i#k=u;Me!G!;O^e&xq5cvR|PrD8-;sw**SQiY#;S zr%+8;jic1DTF2X83qzAR+um=OkWYO*oe&WVtBotqg@4`^c6kAx$oKkn$1zK`z1VwC zlHmu(I+#2_D;#Nn^?p}zxlRW9nwsAzxO@^|w8Js#-npSf4Rq-T`fVV5YVQ|1Ok!)} zNs{cq*rKkvwUhxt5Ov*t_A5|@eQNJJjK{>MK6)OCkBL4*tH?(RaKYfa$4X}F?@~D( ztUT4O^@>C0*cI&s<5ADUwCXKV-r+F(+C(PRAMNyt=M4JXxsCR2j5X%w#LH72eEzgj zvt#QQzaBKOzuY}JRQuIpAzd~t(C$t#BQUj!Y3Kv4`8IwztE%y&*oB4bM6|E^;`)m7#`~8qhpc+R=tw^aVJ*J8k?Q$*p2-4LW`tfOe|bRTFIS47 zj{(hD%bu50W*aD?{QZ!NqC|C!@~VyZ)$G{ux1ZaR)xg_?@;PuQUb=mqw3#=WgJKH* zY5`DE^TZHhr^oGY<3~c7Z#31>&<=GJR?$%TX4U*_fkXprdeh)^bqDgV2RDb{+z)+p zF)uD#g*djjVbg&()bd^)t?d?}YeG)8t39#sHNVB=6;R*k;u>)_0)~Vp8V}vVJRkQQ zV0|l-?Y#?-JS5$sZjcdT`AXBTOe$9is@1Hpabd<$sYPSH1qIgk8PyzKZ?t z!s@r|5+OzH`wqVWw*TM;o9hbh#~nKJ&B!VvyCc4@XDo>u?~xa3rv|&O#Q>XX*^|@>_M@ zNalD`sVzCTIooLb_@m>VVony1-FiFj!}ipRFP!eq*Rk;z-#=?AZ;j`);8#W5=WIVx zhjJqPR#9;!a)3un*SWiULKkoFpKHz>(8>Avi+Dgi9Sb@H_hb$L*u0$Ucjk2gJ_@mC zo6A}`K$YrCrHR;BBK{w~R#rk8-!kBY54FF}H#SABhet`mNOVbxB%nj6%;~1ER z_IUh#=_=9#3P5$q=-sc30QOc>tZ7Su z|1=P}s0D=kEWk}u=4r#fl?a=;b-Zi5Gg||$iq`7=tOe;GH(KfYW{l!E!CN02Yigwa zky@~GZ~$HiMIN2O@UiG}-2?K+ahKYLfU|v0fxg(>iq0 zz(*c{`9sEf5(BX4a0wtLvw|1*V zOL48jVocs-ld5c6l(E#^N`S@i8xv#Z)^xe!mjqq5`iaF3+R18d^{w4VPk@PuxYx@7 zrUyU$q$gCFdpi?IW?f)3$@+2dB41_&JTe88cwe0y*v-uXY?e^794Smo0Yu{VamwgH znO5E2F%?gdegO2i9D0i>JXthogh z*mRoCm9U4r-^PY*`|JnNBd(}R?8Z-lAC63h2t#CbvZOo*2#g0wPF64JYHq`sBc zq)foLAH@|1t$NAhc4~k`N;=NfI0hkG8?9!=cQcyrpsGz!BKp@y_9Dt~aOwU0YW!5( zwh~2YH&@{bDFb;-52d|!GyY(X|Q z2e*@P@3kz(O6z;hjBDW=w*H(7@Y$Rp6oj&h5=NLkLW+*)F zy9%u7gD-5DukL*weV%~E)^4}u$aQ%gbavc{?4E)hF$_-0+d^t$(ln}d>xFNB_qocU!>&v%tcWx0II zD$nr-(zO#lk{GOOV~C6VbseOC#P}=#^J$#+q++6WKR2!Kq?+F!ZTQ+uvpMU>zeM6o znmcMd(z+yMJ0+)wEmMyd%#D2mo02y0$lh4;?tX`o^uw#!xn0%d_ax`O?d^UXnMU7_SLbuo!k3)JX&n>CSD zPG7s*wD3$uIS`Z7f^&Xe1$+!(R8ttNya(F0>|(BCbQu=lHjE`JCn)ZFIZR?l;D^rleY%^`~UNl2_CPH;tE-wKX8r@BHKlomABFF`8v% z2L5!mkf10@n=qj%e`IIgD^oXN+_`wp%RhOUdD1_hU;aWG1 zfDLEm)3LR)ATM!z0h^8|<}Mv~YJym8Xjm z)Z^H6;ig$xz*EA9A^SH3%?IA>HXZ^Ty?a!u!Rq(%C+d^eNHPV(9e6u2VqC<+A1P*d9KxCr(lcq=Jw*8J$1 zp?{uX{0wCGW4V38YML)@4^Q;v#ZuU<^JR22`sr}ju$K27t&CEt(_Yh$BlVoS>4j9d z;>*`T9g>AZsk;G%3aN|~h8YyN8p_14g`C##ewM{wPA@mm(d{qg-1HzHHzt>M6PoRM zG207a)&RDpV*TURP9YpxPRThsx|3z+h4p$&pJIh5HV#8hEw(ptce3o`3B>*TiXOeI z&f0e%S7b4;WgF6L_-Ze4Khmv$&?{6rY>mG81uQo(TYKmPcOleoTOiN-64dS|%Ygov z=5LqceGEDSk{++G#bxcrn2_z*Frt&NONk7YEWEQyB3HLA_1lYLNvYX!eS2_hD*li} zGky>d(_Vml(k}-u#L>12p=0PhSrNI7e=e(DRy(gAMzDLnV*J zwjXB^4)#TmfPjtdq~C<>To_|w5gQd*X)k4nSzHwho^9l84kAeD8wqeB0=cL>YNqFk zOdzyf5Y5?^K73+K{rI$lfO)WzY_>EQbVBH~vcCUz2#}ufi!bLRhfx z_9|q)sd=&39I8Y1dWFM|M*?p#=$W)YsV}2epP~56j!o<=chIotptY*Qor?5lEfsQ% zZ(bN7<6yxGkMrc`=zI-x4`znvjCghjHqBCJv8yhvA^GP&Q@-=T=vcXyH7xY?@ zT?-_p7bj$^NSsBLlp2V$zLL?FgX2T(X^I>TfpDn{ijMEGWu-Dzn5N14F!pYI#+#hDI z5zg^A)s2A#S86!uch_fGWHA!?<{lGrzogn*$;$vl%k(6MA5*J zkQXPuR+?q05--gNNc!yKhjU$Rh!U(7w_Q7*_!YA(+};R=$T$y6}F^Y5T-5!=Jf_eBn)fedBk5n&H`Xr#Z^TF11b9DDmE~|pI`5l4ONbf0($QqN;1;4*u3>jk&#yXN&SB}*D>Z3dm6Cb zI~Lm3Jh%2fJ@Ot>lI4?+1yLCc8oQ|N8H6$3(bIj@%+flv_k&WnjioK^In#mAOM{nX zu@7Ev?mspM8NFG%Ypq+o^ljP~OSa0*JyftZmFMZsV(e#m@JMf1U`4rU=~Sa-uig@H z0L3aKyk3=wmiqqosm@Ec25QIsh+-~i7;)irpNDg@)Z3fhm(g892|E#oJ+D=H=1fy4 zaKFehE6o)rDhXMUXIg16Y=sG*cy!-B72~{?OLU-8YShIHT;Uv`K_@lz!`28(A`8He zRcEmXewS`Pl*&_eSn>p+?a){t@i&tM7yu?-U^SP_U$k}9Q++9M9Q-L{dzgxsMqC%w z`xQj(`u*y33jYFY(m&ua*Xw(c_3{oib$v#=-zq)VGS@%b$pxj$$g-xmbEcVv)Wcjs z@>b7l|y)5dsUn@dV*IGs`T6f4Jpp z45(jjlD6;lu#6okYPv<&=TZcj{8563RRMX{Pn80`U0^>jI%<6#XVxXoF3DjW-?U%e zw%p<(qf$k^GVl5`pLbq1uArgXY@~->{v3jp_fX4dZ)nabUPfM?UUcRDgBTB?74MGO zx%WOU>5tT!4PRy~;pMCzI~QEcEOTO)g_mdTfL_Er5zN3+tlME=R(R{)C3~`CxV%tq zHucufV5Rr5gVnbx)kut;eaQ}opUaM9q8lB3aUgkSPnH-us)8Apv&CXWl9|3O*BX0> zzGnw4{JL*FngaK5nP@`Ur&*r@xBm{u;Y2;`aE_9-)2S-z84$8iOEivZnz2#Wpk5=$ zLtb5vjGP)LeY&u+m(Z5}dEH?Na`$o0j-Yw~!cKP47*1^)x1MX{_o^f-Nh;G?>lPa3 z{l@X2zmH!%Nq?5#BR#0|l-%82+ylD(M^eMD$ukJ{mjH9o?UhvveQ4w?(P6k|t6NLf z#1uxt`?PO%AqC}>`~xv>+<_0I)gqLr zW}IDKUVYoV1BZ<@JEyuT@L(v2ccHaF=V}D6iGRPrkY9<-%Y`Qq4 zvC^-x0n9CjISCt%$Yq0GIT10d)#R?}O!A%x9)^5`c9C-Cu23C5AvsQpiFYUcg2hzH zD!B2|i1-sDc{v$Waj>jsJvYd@Oa2M;9uf?U5`0#js`}Uygy{wTEVNrj2^xd&)*(X#NHa0H{gd9g$ZMrw{ zY#vF#{d0NhX;0;B_XwR2|S}c7B`Pk$A9Uo6aUB^_x>hueITKZo_Jb!g? zAREX)Qt{#%-g3o<*p>yQnk{_QaCzFhVfs_^#k-a0w}PyLl}R=br(NjoDF&@T++pA$ zR^3gOH@us84+d1ndDjfsjqxE=x_t3`h)%b?8?-AQI=cGvn^*RJEF*9CCjDDZ9S{~7 zJ!0EcZylN>%~C*L@$BF#wt>#xwWaqDqTjrkQtEt7pOw zl91=HwwIFCmpw^B2}kz@4E4O{hH&xsL$6ZhP9N(j%(3jM`kfy)yTreST1B>)%#qab z!&-4bE6sky=uXW)GC*^A*P(y2z5q+iMCv2cQ5_1PXrQZASMc2Y$2=_-;qc%rpiF0NseZgo7bdqc5 zKF)kUVL;Ki@GBP$5B_kKnOsd`WQcy^wle`I0oQVn4}M z=P~?jWz5s7m%GD+Ux=%lG`d(-)jH}qb^dSlnki!E92Babq;=1SlW^VUk(5g-0fV&f zUeRJgXOYvzU5cZ|*oUdkVcVC)G|$5?O+A~OLn4ZtuiceSxY_h7d)8De=Zjv+pp|Z0 zD$4z1-=X%P&jCR+mYVB&G2Hc$2)Hyx?EIo@1$kIlsLL+99U(PUx!eoT%kX<76Snp0 z@?AC`Nijc_`{;i)9gViefa9{kt@f2dQm)HBoRFC#lIvfh8W?pEIG_d-+%LohWkd|- zlL>`VPQISWT8UR3FVKdGFNAOQ^m!SxQ7v_r5vfdkwtsVDw_b~sYN}9*EK37PQ4Ur) zT%o7^;;d;$$8kWJ6HW)&h2|Z)%DL%Vh&>nkr^ZG3(ofOoZ52W#aqUuW^p`!5U{h_) z)q%I2|I4fU?cqhIe~fKlG|QuR75}(0$@zbV1~obZuS8zYGxvH;s{LyT{bA3GECA+g zLOn#DlI(8+?JwbmdH}lG>h#F0%zr%}jj&G}CHtJFikJW4cmB@@ReXsp-Cja*p{f1! zZ*J-TzNSHUl?7g~0Oxbm`9Gua%T8m$Lc`HWFc!9C{@23#!*-w}G zvxlod*-5G0wwZPQKP~w`x6}$N@B&xJ*0}9|_ppcpaG_NruRfRE`|lxYrdJdMxq~Cu zSbVt5Mw(E0Jr=5^O0*g2$*wkRMwuU#bnW9iF$crRWs=yQ_+4#_hE08(0LMU zdAvDc%UxpTG*<(F<7*4f;8#~}V6ED!7Uuaz_blMtwmeH%LBiHF`1owzq)hg&k+O>w znSqzs*(r|~>pZG(hN1DA58SCY*cgb{UGg#b+1%Z_-P}E>oH!x+N9P&{Fc#dL6EAjX z6`Bz5*uTC%27-h3ZJwvE5pCX{!!00o^8t%FfPV=jm$n1MF5XTus3`pPzT2TFk9aoY zteV4YINf$^jm-`?QOl7vjn|P3pYq9W>E(>)W-)4@RUV?>JU`l-YjQbD4X=9{zcrQ@ z2DW@asQl=!VeShDp7!-cu?i;93(xa#z{h=D@4o9bRiKK?YxKdJ+D5~tR8jelg$10z z-Q@7zGUuZswDl$yG;xQ2tgg{TB4(7kLgQ3M`xWNjof*!@G-%AUZ9xNU07ut+5ew2?mc-M+ zS}l_ae0})W>dStE!8s_YRZyV%8bgNYh3jVg$RoRU8@Ov8We2BBBGcK{dgZcBFIdn9 z`K2?mqEixY3&Ifbu<559dD4>x>s~hJrIn{sB@$ffg-&HegB?*~K~YV>p-PY@M~jP! z?q~Y2KYAP;+TPwOxV<_kTJNx`<$P>PyVEVZgzydv?Yx6^#75M-G^bNDOsNKS1P%mpR@kCS)<*7)xWwR^t6vEx3q;c+D2mb2`h%;O&LkfmW$*U0$BHksWn((hGB z|FAo(_2X_@ga3zsj|syBCGXfZHaDm7PWD=_ntMrj=bHuf>fAEs-Wxt--0?B!ioR|w zmfx9cP(4v|uNI^@+WA(iqOsJkTnKe?XVx9^uQ(xXdP4kLcKBFf|KJY)C~@Bq0AYs$ z~dMNQta9S-(S6Tyt6_4HLJ>!mP!}Y#y=-`aO#3^&T1IMwIr!C z;->p^R%y^lq-iSgJutx{a;i4y{x{?^Uc$oExp%C&cuzLAw`ZztLITXA$+u#|ZZ(h& zrJmMzkC&Q)3b7Ddt74dhpH8R+R?=v#q7`vE?$z(k>ROP{E#yP%%Ct{M!`@>4PqDov zSf7eZqE&?saFbJXC^QFLYV0qlyCWz+SiZ^bSl$2}mnJ7C%G(Q_c;w^uKq1fOL}{bz zPg@W0Hh=|qW|dRAhmcz|GHOx>l1=x8ou?J?j8g)PXkDL8*Gh0L)c=x819E^CX(TN1 zF~NwW6kaREf>pjrF1!{cC+9BOy>+~mQgOEEcX8G_lHT@w(QfE3 z72=~q1Gs@Y)3{_s`14PQ?^ezcrNyO%Qc$i;Nt@K`*t_P=JYBOL`>OqO(eK_+_K*5h@{aCDjFNq}$>3YCXZCv8_L|3%Fko^Q-{wS{ryc`-LQ`S6tH+)i zU;?iRDpb2k+zHsZ@kNcF4tv_J<-eP1{nEf^;vb)FYjImP+)7+4YQ$k!agaHQrOB#e zo>d1M`wvo9MjtOTFlET^chG6xEq?F5Kju#eeumG|Ty_pn&$IKs3iF8Ylxz029seOe z%ydV=036|e_j@nbNt!E)B6$$VAW>SpFhOKqwOTi)?B9}oH?)1;&tE#>h0y}fIVYT~Z zRY#FLd05;}Czg}ZP>@&~N}OkN68yl#%)LuZ#n@nz)SHZtD19H;S zF)%W5-tXBxNJ!LZUv|4gg~el3lKG8EuxPlEbh1H14?UAJ+s_@SKAl#XUR(eEYqN{( z0ENr-fQ`-fji^PW^8~%Q=*8FGOwj#+THDTBVAHSyqAIrWyWiX-eozoce{_sQThlrO zPHnZvO)3j@)pC44H;bPm*P$sXEJODs!I4~Rx=8L-b1t4FA>C9@{DZ^!lMD#-@m8;h zEFC%M5wBGJ@Q!X=Al@=r(s{&~#^@KmnRS0=xzkzTC@rz}B_Hd%V>!~bElt&ip)H=`xn+~UzZVB`990GhqD;>2vA?2#}QXr9BrbOFkL z!Q20oS#?j;<`a9}-j`wqqq2H(uh=TY=44}+mZ&stXLy{0AbDQPBx@SKBu7{0$I&}> zanTB4CnhLWtEh>y=Zn7kNXQ@G4o?GsPXN9qC@T;K4T?NoPxXqQ)>*2zM+hN%GK3!K z)K=BQ=@^8Q0^j?6taa&r$Zo)@+t|~_vH|K)%S{qNRk$qOYu;_J)#(5Onc@PAm!~N; zQNf8!Nh|V@6NxhOxLmA;>XT2s((S)Y_9yqyo7;bSd*wYm_RJIpqw#+wCwmk|M(%G` zZ>4t+=k1fZ&pUt}FORb0`w83U7HRbGDw-+9&JldOOR6Heum!#-itjMh>Pw9kqE~&P z55aBs^|bn}t_sTF-63(%OZt2%>&^L`7bzEs)?U+gvwi9#z9w_!XTN%`hmNhOgWJGh zphMfBwi34JD$#)5GZ=Ydt(|GVKTuw(iaRwgP*j(?_Rv$qQ`pxEgMi6)I|zk2&m_IB_fP)Naf^dWrHKyPR$A+fg4IBUA@Ja*^+B#F zGqf&PLt^?7@B9VJSJMm})%~8LKp-G>3~DCEQ68tqFakI~h28|3Q}O(oQ5y7ooDXu? zk4dL)gC8{7tTf*?^b;t0_I>B9hDNgG3U0-oTWj;}vv{9J4ZA;%QCjFBhh);D7lcB| z5doJfEMt}T4gVGxhde%0Y*mqoynjJu?IL#ButUrylJP~|6h_?!Dw zqAo9zc`RE;;%d!yy^sbUr)!}LpSFQBi85Z=PX@ISaK`rW)q}PYH8ij!jhuhmDk661 zI^-vF>z@9^c0&#CMY2``>$K`p-KoF?|!L!{Y1nvtGC`jnrd3>zO&n)ceo!)KEsrGfP4EjHm z-{=0ym<+44Pii}3)QPSTJ$N7UA}OXiR`a4xg6wJR@}_S>c_k)+tjUxsOSaS|V^*)o&HVd4-Y)`0(`M9dhdutZ*h`ov{Sc573eJSE zfTp|p$DsD*&8t7>ih>K}Gn~}*_~xOWgWNdwF5Tt&zjO(WKDNIl#?&2G!_`4OWnl># zZXc$fDaOmZeYX=tq~{|MB#rV1--eSL908Y4R?B^WyGVO#qB8BoLGmj<{%z~+vXi}6 zo?|3acE8;t+Gisec+C~(4!cWs+&S)lKB1c+w9nb`vI~cL_ANxM%+B#D+|_o`$8l8e zlAt4bW8G)|WM{^?J?!I3x=B2n_={ib=H6qUA#G(q`8mA zPhVe4q={nVdPMEA$=?|qvTl9YzW~H_WVSOmc>L!z0lav)CUhVKg@oiYw%4Mgg;`4B zC#BI61&XiWJsck@y^^q7jE>8H%%?ZhT+;WTpOKuFBC9!i051iPJAnk)9qQ-+^As&h zlamPo$*Etjc}40Y%8;#Y6VU8zJtr!yx6Fc>?a|YLX;BzaB<>;+RzSwkOqX?5ybB1Z zgd~v$HGfMfw%8Y5OP_YtptBn-w)U0C&A4ACTKgF-0hnZQ*7kbxGdIx2bX)+87{;8S zu;A))=en(ATIon$e<}y4r&_xZn-8Pxbc^p{udksOs6$4#CGn;BIo}-@*jUFZU7A|ISTrC<3C!5 zT17U1)srxHA=UDSNA%k-`roIIi~`WWd|?cXU+B-@oA&!}J5-EvenEIM5WXn$`sFcSz!uW3H6*2`h~qn#q?K*xwgzOnqH-+*>T zp0#<~hb5NTIWIVOQL6~8QK`~=>n@K)GyZgCqCJVsaH!zXka1W!t3%G ze5)4YMR34uzn4}XazHq3$lI&|__;b)ZjOUf$riuSu6UXcKYke2TR%swcJs`^9uEEqHbt~^V9B7h!rm)e$j(rS;K)SE(Oe6p3pHP9V-h0tx&)76Dc zvR2vfs9v$oIX42Q?Va%i-Q8X;Q7Mjfyg1V^Jli~m@5SVSTcNZ-`fq+0!}=wwkC-rO zIA(9OZ`4T9O%wlq(7qJjxWryz{*8f>JEgUgj*3&zk9m1qxX~o3x+Hlfjqi$S6`RR z6m*|#;czSf9hPvAnL(YJ&Uc%6=LD}WEnA^x$XzlpkhP*&PD{-!aJ~{|(;vzWM&;NQ z!dxrHw(lG6)xCDk|EQ_+q2(l zExxg2@x1aZ^T+tfs<+s{SgH(3Bo{RLFo`jsExbiiA}UY(L&I&S%vpCzeg{h5c`LEB z`~siCE`pr>&$LOL4*?te- z`y72v!ahfIVps|s65MOBqs3hfw{hJvYY|oL(?kze&pMmMpC?|{uSa@QUkbATOy=7c zi|o`)vZ@n+yH&J;f;$gx{?M`ZN=qEl(H^W80W$?!PLe0uOK9!<#%|K_(E+&c2OA|v z7dJ2yP={mF$UTb`>=jNl@u`%LD9#-ZT@%o?7HP#g;Zw{V{SaQqJ8GxR$;dsAzAB)F*E31BX+1_?N6O#cv- zB5)$?MQ?Te#1;gJw(7!pqpYMLBpL&NI+x^U*B5`=zi0R_I;H6wjHq%lTOxJJcz}hT8l|(E#4Vll$Y-4Xl*UYsg32QnDXuyrCZZ~-KiL0ti`)X&t7S3Vlrnkn>5O9ac6 zZU^PERr=oX(nuw#cbL%9uX2r(G$+0Vs$Kky(9|zY!J?pD#xT}KF_n8e*}!@xstvPH zLIZW|RsB#S6Dg_k6y7=%8r19WcK&;boPKN{BW6fJv@1RR&Jd=17Y?E+42cl|Dx zDlk@7g{;)U?WY{8a#eN);=UfIhIxU<%NC!IFU=M?5#K*Oy`GiK?6*l2Zw#W3rFK<9 zhPPPO?M&~WfEKm#xfS3v<88r5(&DgLp`@sXoA+1iWfGV+_=(kh(+#2o_H(oNvlX0q ztrYd^Cue*8%oe|^+C)X%o3EErpK5dFF;gsB8MJ2KL1qPHK;4gvRRC-!pJOH~}xw|=SnapW*<;3aQ<5Rxe%=EI21^t|5 zX0x>(>AdO$kgE9});Zl;i-~!Xyr9BafB!*Xiz?+EtwA+$=Vv3*c21|4f1H+XzJa%hkQ<08C+nG-jzV)C^ z3u&Ee@+_Qd0ky%9pV|zLwt-wUHsX5mEfd|bHI{?k(_GWiOHX=444o))k9Eo<{@u;; zeBOfLgEbOFefYspiz6T-tVBZOC#MWIqEs*1IBX9bU`aq_Koh{UeeNzCiGXMDH)w-f zn#VnV0y+r#`Aqie9gtAZQs)Xlo9lHpFv<(Tt`psY3s7f4?!`9^Q#q6t6d?t+zA2il zL^W;27n>)5odXA{#NlZ4%6(4d09A%Op^tlIm-DmQaFB=hyNweUw+#{LOn0JTaUM`t zC*=NddJQ&Q&_P$W{km35L`EW-rdF<#(NDCv>7Lr2O7WO;toBiBp z3g$I}I!iC&{&)7`kF{nnG(>ruTi^AX4coUHQ{qD9+HZGuJ&w8P6Z6ksEu_GZXAckn zNGx+tY;W;!dl1w#;&CJf?s~R$c#uerMYnMhh&#_% zPw5jfqmKbCQD?8{en+a{fA!q`%Ue{giWBnunBCsWbZC=``xhiTO*RMQvNGcd-);C^BKIl zt3n4kTQc!D!qXtw)fqCrN!{e-W4bKog|uZ<&a&smwR`g^#b)B_g>#h9s$Ol(MQ)wd z1Rs05Tnfh&fRskLedBEs;FFulx9l$?(%FEA~ z=chDEitq~VWAQ#4Du!ojmpf9|DLnI^`LG@6>M>I7ZX)!>VhXG&+P)-^YK}mJt4z1b zl`4Mqw#Sc1kYGWO97_RK$y?&SZ6=LkLi6Pj5;P1Zn|FkA&&0t%{3WGc#nXA-#?6(h z=3OTPvY-esj~+N02ejCVH(M;!RJIxL)YZ?|)FRsr-HA2nifrvWhd4n}#1Hgd52}|V z!cYk!F7BZSHMv6!{&&36X+wkNX|C@mpYO7<4I8tyK3N@7%XQT0a}A232y=Cus3Hsj z47hcd(~luG3~Vcz-6m%3b9=}MUzDm-)2&EEeJ6l!YkDX<# zk{DEP#35#cymO-LAub4WogRg0I<<%syzsj8zX%9XgIx~Xgu}cKE85`mK;jWt9Wi{j z@)pNVO47pK6dhH%J7RrXuv3F969ky0jx$)B*;XY9dKP@KS2vt#KDt!UBT*f}z z@Ve^lHC`ggWE6X&vLtjf2Wup;m}?DUeq!kU;OP#^)*8q5O1o^m&T!?T-kbXc`x#ofTK=8gK~wfK46lWoo%vM%~t`f3?W& z1>98dNw11nem?h1lCTCRpq3SAz5b0z39Gf}$5~9kwrK1y~wI$y>oKpEdO1?GtfxeCg#XiIfZU6KXxVTv|isG-yQA>KtU_u``q&aQK_>n zqnh=eYgC9-oC$j}*29ijAcZgZVg`8;#0jImg6U@=mL_VA*=@yk^3Av)BTy@JR`f@6 zUt3nZFRwWeL0O*si?~YuZOF5+F(Fqs-LM-p6!-9jAK%_H(ylU!Ae%eA9&q;ByoRX@ zn^9@fVSXJ~V15Z~3}ovtThl1+H}3gU=f-UFv@|CIo%^Q5#e=gv8yR z-e1;xW14zc;hBb?%lsnBw=%00I1qyG8dO0|b-jv^4K8Bc_P*I;_X{5|L51M!x>Q;_ zuvzNDSeHoPXAP0d@|58;ZHYsv=g&F-Grk)msY2izFLno_Xq& zJZAw4iErxfPXbPU{AKc!;RqZhnYFkCBz;Yy1`c7kjLoUEajJaU48o=rzjrpKOl&k5 zE4N({v{A@nMYzl4A4V`tI{TOcPl1u@T>tt1WA814;%c_G;Xp`m7$8{CK@&VAK!Cw5 zfnXs)f?FWCy95~AA-DySAi>=R8{FO9A-Fs9ZIb&w=iKLgo~rN9`|qvVMGZAIyL$+Ak3T}#^fa;~!!M-%yfRf0$=q46jN_M+df`bwwg-y01RFfRf}-E|+5OdQrN<$er~6Io z+3jHPV_id__}0OqtM>Jp{Plevg zQ>mR|6mjJ_&7cAuo?FamVdcFLCy+GnfGF7WYCrn+#{>{%1X7jK7$UegC%o)hcOWA7 zRWDbWZBxoi|D<&SWK4JyAnop**=+!Lpkbq8`yDL{aEFq##f1ErN)J2tVHN|>BEA^( zrY&xl9UMkOKIby4>M2OAVdHrU9E7dYWT-C%uFo;GU~sY>fb!Se8mx3j1JTl$KOZct zYrmnQbZrZchIRn-W*PyM_F)x4+rnB-HdvJFE0`ap^P$99-U;enO%72(M|YPQwimj5 zk0)n;wy}F3uIM|e%Whk(6^NOZzkL16!CBA#e0pUsqeq6X%0=Sm5m&#AZe&Ik0vteX zl2aE4-9nJH)}s0Nmshz|Zmg#b-X)eFHo)!oPrrFyWDplzhc5{MfB_C&v+q~yRJUEO zLV_Q)qC^@W>mh@wuyipjg+p#pb-Vimt`~0w1C*evG&52N?K1gDWxAmdhTn4t>^0(b zgPv=giS{M!9+C3qI3f|NFXman$Ou{ zPaa8b+}+y(mGYhESs=fR@1b}%^b>ruX6pXQx}b+tAnH(2vgzE}?SU;$KUN58zmw|- zWlOc>Xr5v-d{R(J6c!Xqp}#afq1R;Y=S#K}iXY;?P7Pf+5U)=$XgfIzKJ%=CIuUlw zxt**QpA6p$RBEb-!Z23%GjPC>kQd8}1X$suqUX;JFX&1gSqq8gv`BC-n)Zj#r`zlW z2X90bQxs3(KdTG5yCKkww&2{0V3l*VvzbIU?b1E8EsU+;ipb65JED&J_q1_wSjXE@ zT#dGbDq`{ecdN^B!0a;K8`&I=B{PR1(7X3McaG>=auveNuUck#+EF4^=G?~D_b|FI z)*jI7k+c=#v1?sCIQ3sXGWGOKLv-Hsg;WQ_X$ZXgR4H?43SmGFG#0;5?}7P}7C(0< zagZ;}^j3<qsx3H*V>LUU z+ks>*&PkJ=v*H90$~q0M5|TP+%LTvmcPn+W)qGL(>K^7!WF;ov+uvv@L-1DwrD1v| z-zq~R8FvUz94zOJVLodDKhI`@>We+4jKcsGUH1UvDhpKFXY9MfBs}OjaQr93eR`MG z_tJ{V9>Hl3Tdku}jqD~ii-EAOhvK=G1w?{P~(U=eg` zd(D^8u!uyQ$8zVBQ|(|ne}}t|YVv6A5Ohh!wad@jAHBb=$5=!K3TO$O9jf0lULk-; zc?cU5muy(^+*)8qF}b)Q1{mdLO$xDewn;2xFQ4NwNV+|3mA6p*UYYi_E9bK3gLc%m z?t$z`Q+6kQupjC(t`7`+q4#>b-*0yw6iN|ShwTK1IYV{u#~Djyq_7UmUasNgcbiq7S*glE6o$&%5M z)3kWwG#uTRQLy44CXSNV@q(5*^SNYgwe<`M0v!IXo}}|d_n{=Wc7^=S?`SNh$h~4T zS~?~t3%V`TIpUDExW4^Fnr)60aKHmvT$Q3wyhTQ)CFttGdkZr zt%Gj(9*(&TgBi|Ivt#f_i3>}efu9$s>6*IE=WAdd66eee$qoFI;@9Y~qWvk1MF$nt zYyxth1JRJ3mVK8C|GgXd`jepg0Sv=xsl`OnI!hh3iSZj32buWXrz5!Yw9zY=nBBwf zM>OowS7cMp%~8(!D^I!@)L?}Mo`CaN*6avUs$QpFVM!?tT?uLC7Kf^OM7KW@qYjlv zED*QcTx?c0EQRaCcfGB97V6ps&CNd145=M4NVMw&ki1e6^x#D%ykd*-U4@?GNt3YD z+&B&WcwS*{X8n?m-B_%H`lKxc*3rQ6@e&nrfO<9`4%uuDTmFU$iq~ysoOPB{&j}f1 zceUS29vQ}9AAJ+P`EF|vh?97ANeS~7G0h7$EU5Q9i2fc)LVWgyWw!aAxq(Y@z7n~K zNXkZs)Zpfu>5Nb(+Q{96Qllj!qw8AV)mv$zns{s4rx>ZziUdk|s=>cwiNfd$&)`pr zRF5(foY~5X>3TLpP+`;~-dlL^h-wgxRLP!MKHxKrR~k;s31L3Nx27Wy z8_>0rg*55(3N9qA{8E(&qJ(z#KErL*Z2iP~s_yM*!@a168$5e;dS%Z!Agjg_9alJb zr9MW%Qnbv2KK;CSqgw?=^!hXm{PX$awY$`)Op6B`vmzuYAYKapS~s>Z)C((vcZ*X^ z+qugs?P`yK^EX2W$cp0^mC$D%t}+Pgjh41LehWE_3pAv$8_n0`kIsb2@UmNMiXLN+ z4t|r!c-#&Ayd;KItC)|wM6ZN1XD#~8nFfaHddbea=^uu7!Fq*k6cTdiHJ|D*LxS^4 zMTf9L5Qb9DM5HAHWMgmQJsqI@38gR2Q!D0q{nO#_6RB<@nra9Wk82Zt+H%w_tpiLg zc^wTmLfqL$RFDH!rd5o3;Y_6LdxnW#d%|)ba*DbTS;|5-0oe@i!#B5Ga~y3wvK-be7!3#bntkGu?T! zhf$e;O+t!H_q#Hg5&y<8Tb^H`@!4ka#Dr(rR46R5+AqF_`<}S>ha2d@@!B+f^_g2x=WVxjsRplkzja+Wuja-g|0Wl6;5RlFxxq4Puks%q1i&HxsyYQTV zU-Mcg;^+3}^MnffsbH0|_q6!R3i;iJnUM8wkD2?a9T+)-)g%jh+`h8IbS_w{J zBxLqY>ME&|Ij{G*lTW_LSDme~geVJH4xa?~>-TB)?PUif<`E3O@@{1N5X-#_+hjtB zoTt=URVOuk--s8o+9;A&4i9ji*W|h29IW}Xd~1OBy`mzR1>Oo_q~1f8Wt9nYhD$0h zW+Vq1+8o7gV!u81d+s6za=*vxaP1W6|M|@&Ntd`k2F?-FO6~a~-Fx*J=p9S|xcbSLAS9#-#EwZR5lsFjROr|&jHcqz)N{HddQS0&6&92()!L5xsPFv za-c;Wxm`~2*f$C7mED#$IX7E{x8uPoPRute>e^xG(Scq5#$MvxeJ3!EA+9YiZ6Jy_ zT)^p)TrM9Sj8oWatuJ|wdlp9L$oM7pTFukN#2a^194+Q|>a_q*0sl;6nl|RUkbUsH zG8K#5TB=3q^QThAG|kvqrS~HvQMV)xbtPI^BstA4~_sx%n#ht_S0F zky5$u#D4l!!!IG?t|Pklh^4s5`JgR2+=Ug7B|hF!_%{wo3z-**`%~CSH_exa@3)qc zf^Q@t7q#*;gk-W&ZsLP0=imbeJFPq$@4%BJArex~rX`GoleIFnIoIUKBeayBF<^%q zyz~x$V@e*%{&)_XS9uifFS^Y;J5c9H%Kzg!0LMhP&c+UK*K|qSkcAxBe8*wAX5;p@V zDbO)}03U6%H>l(A3I2}tcfsP~7eLl-yd`(R(A^x6mbEjQ%6qU2 zl$4}8Ge7z%eKr(4P@3J!~&PfGgv%9lm|syMejghj~} zNX!`j2nh>(=`A3j`!<7yey#4gd)V1My)}9<)L2#SukyGEvbBI`IU)Ov0GaAS-?>Es zAm+Y~f8w_>ndM3>l(OC5;~Y$(^@+F(0OS(Os3K;VEhj77A{GP`)6kMqxJzHb43j1$ zLjYCOQL)>2Ae@(hAPD^R9Cxuo>8irRI`eF{b4r?v%5s#jf?d>8DQ5JyIRb$EHFIi! zM;bA*CVxdZwrQQ&=%uRYh&Z#hh2@ta%_^}vdj_2!hpBqdMhZKhX!C3 zpR1E&Lr@_@|0`COd^JQ(}VF@6@`ncn~~%%Vv3Fz)Wp$a>*=p(W>;Z#}>#QNGp$6 zPHTtI`w>r;>j(ex#UO9j{Q~Lw!RI-_D3WZTldMy2w!$BB6V+8pWm&9!-Jd2_j(ONC zGbTT4o2$*k{)V?=(~(B3jQ$!|UK~~^=zP{&tOwP77!Cvck4y-WZjHU`U`Hgyw5Ju! zzYST)V{g`xqet%i2C#ljA!5t3CXRugsMl}0!~eHApI`!TfPV+)jC+HFIsG<+H-%GH{C^ zUx84pt15v0pOIAw4J_!BpLdxhd}gWNC3rPNR_uSJ-~?o_=EQt%Bd>vAQZ*~f;XGvG zP`IH(($aMfCc^MR??~fWO9clf1$!EYC}y-;8l|S@_|g^v2)B$eZidZzg)SVp}N5bTjVFu}*$)-pcwgUspN5VCx4iX!OzNPDFgcqcG}IU#dU7(Z0MQ zgZRC?UDC#aUl*ts9OQl|xARhc(f5 zC9cpW$S=d8>miYg&9(S^91VL5jeMI_qC_a zK2peX$@}1s9(>o;gOWon`#3kNdsM;B)xN84M-4$Mry?Ze6Irfy#c6AT{8(5AVWAow z9@7>2m53rwpJx&9=J4<{_~jnrx@>vX2KH=97eu!lxB2^B{Z6AGFe=JQ;AQ*zYf>GU zUS#cXbgReoSAc8xAy33iUAiW9b@k#JuBkckQoX}BD!*2_v8;!&?_yBBH*JY8D%}DM zLRncr>47}vix11weN`96QjN=mEL0_@r)wrLV=wtMpw6YTs*>2I;T-i=*6R$xx`E$l z@#(kJo!4xe2b!X+ZETT<52L7)M+?=*DJpd{@d~U4*UiV2Ya?F~o2aJNXBYfgRBVu` zYHp>eDxj!DTIAa2ZQ*raKaW3Sr>cl|yg#0SCw7_O*lNDJSAY9$C3G@MZta0}gR`z1 zg1F%fnP5C=_w;amKT?0}R=I)g@zq zYl`Bokk1eM4a9v4J$Chr2Yw9T=#gf~6*ZHt1Y#NXpYV+9q2w;qHJ?TS*f?v;{ZRJ# zz1zjwh9~$5KwRB}3V=b*98s@N0I*_xN;@aG+Yu0_^fmBx*sH%VaU8SZ762&DTCmF{ z9?vD;$Kq1r4(L=(Z8pY~Y)zR6~EKOuN7%^tpzs z@U)}VHMS4R8KFlv%hTGBti|obZi3*(@yY5_7&_Dmf}cpTT|9r?u_xqlgm`MA zYtXp?A@Mvpq6+UD`i*Ec5C?+iK<)Pp4ZIEA3tmtj(X_fI8ytXmLPDxOnHerR1M$|7 zK*`O^CW#cXN22tAOx)nt9(5)XZ^GBYs2?aFKz|yvO|L)WKE!ZeGACSgo*|J4B}1KC zk&t2wVJqC&1X9eda#a^c+?0wq{ahhr^|ub+#_p6vblxA=jQ(}VSVJMnq$~{gzdg+J z{K}yF47ZzvT@3X66?L8CyjRHM*89s_+SgyMx=YzzEiqv~yce;bb=f>{pTiIH$D4+i zZ1A8LdmhXdUJBOf<2D8m9cbZ&e7y>+wY1)$NtH~;E_i-6>l_ViQ9M1aIdwn*?Wje~ z^cmmF`k8`Bk;bLVP#z7YX0cTv&vW_Rr$NoYn? z7X$VMO8(rt;hwHdu16f4y8F;YE#yeda1yPcn9<$ZErv5d0m?a?PV!Xwo?A3d~8xkkm!&a&HpmOwjt z-+P&++mR3px6D<@u8vq|sjlkD(JHFo(jF@*6je9Y16pq-2u8lEd08i=Z+s-`G0X@} z1r+$I+tWOI!y`KD%&QuF(UG}*aE>=4A%)}j&*s)#9Jf{|D+jp_G%(XPDy+3;0D+?0 z>&#j4Q*=!EJbn1tJN{jgMl{?SCo=Bra58|&BeaXx3}p9rdL>ef!>Z4@kAEv-MFEs$ zOMiUScZ|8|7tRtousZXmj$Z5f&?M2B>H~lo^E3tA6|s)a{*8eMqbbzGIfTarLLGrZ zDbHfT5Umj&UEo)`+)=E*`*n7P#$XYy)u5H~wpt{LTcHUMsZag1av>UFI##C05~up0 zP`VdzA>O@3CO)i*L_d@OnDKh|@u5`)ZyO+nf>-5Kd{OkvA}2ejLeUjZZ?y}xDm#VD zmR3dNz8c{AeIhGGhMX@GhNxKM9_fAjQfW55T!CvjYaiQwaDEVds+Z+UoR22bqvCp! z?^yR$1aWX0$WS6*DZ5SUv^`VTE}loLRYh$OL9FF;W?+K$G}r8r6x27iH_0h{KSBx% zDOmBKi0bxezE-5wxr)=A|2;3f-KpBwDrh3+F27N+^$oHPN>xF#bjkym5jF2RbQpGQ z!UI9U{(NW7Q0I#BWm5K9JHLrn=M*$pT078mns=E5Lc`^*1#z~?i=A!Vb8@GM@1?%Y znfQgvSIhmL1h+tL0*#*C4o}D#DMUr10wKRN-9I{iH#oU(M%eK>h>@?;+EO12bt1x} zP?hVqtEC6}t?nx-CjIiq<37lP_$~Oxjb0p3n6-G z2@2o*jzFx?0=(h=_PSV9cLu2@fb;Aw-%Qp4*{G;_@)s@<9erEK>PhUe8ap07O%3() z*DUh&&$g!?bxyOtTJJlMW|U=<&J6$Tv#wa7Ja;)+Aiw$-p6x3PWyR~gmiJHXpnK`A z&^OS9cXJkh3&&7vql11P=dF5Ts1Z+9eu^nfdTVu)s#6_Ol*o;STI5Ceu?aj%B=tuGjsw|C)G(#5?`o{%=8oA^h`8 zV!_Q{WBGs_rr9pZ$9e^+O=(OTR7|Yx;b!|%#qFQ(@|P}8w-bE?SbpKVue^lwm0K=1 z+(8!pfL*Cw?*e^BnU9~MeW4FD4q7XF_19NyPrL?cA8HSrn67rBzgs~I?;L-7OE3GF z$Br+ot6!1zZ?LT-c;wmNP~X3X`CtD=4Bt^Xk9QT<|7z|}qyO=jPG^(kZrJ~)tK$>U zHyf9;Y}^07@81pJ7ujL`-H+erp#Oj;ue<>K0Ny&+5#Im#?jYc6zK8!?zibi#A3b?a z;8*g0-}mqKP*^DL|8M=`y@Nc_4}X zmY~*PhL_V4^VB=!C)TS=0k!^GcXQK+QboJND6O_+G7phRnyuf8W;Olmmx{U5`6b;cNqR2eU~93DEe2av?hgGTmm ze^2U#YXY=!v;tBMew)#b{!sucFkeSed%9-AGs^yFKrr7>1GJ=%D>fGGm?5uoiM?QV ztnNYwl}WzoQd7qEfVVI;?MiEz$;XO+S;L#YvD^!wVf<$Ou&Tc}(Ol`hOm#=KsjK~! z_IvI$Ix{qugPpW}P1~Gzy_Fe}X{>Mng#tcQ1>5}BbKh}+H`UQ}@Ng8r*v1?+N9b0fJP?+55(pq+^HHxmNYcy{G`f#=ktwMs0#BZXFGvmiiALQGnAVJg7S2 zTPiRWMTc_tj?1M&R%7$E)~VhB%-L-h$Gz=eBLy+v{XYIdaX;cjveClhCOOnP*_>;& z&hH>LovKRb2PpB=x;|g_zuIh1ppSP!N=-%v@@4C{#ZHv&Mvy7$9&aHU8CI%F{)ZQ~ z8q9ahCtopPM`zzs_F;?H;RO_$vaY6eR+CJKM!1-ae?`8g>DBsNmE!&O{Ss*SUmF)j zc;nSx!(Le0%S`}=zCvQofu?dj-+siD){t<+@70tchBqIMhk&HH_(dUA{S<7Y*|JZND)&cQr#AtMd-U zN}HXBm6I0de=Wx`are-tJDp>2b$xP9htQ8W<~~`@o>4)*{EUzNgWi4g5(&Q|`bb*W z=Ry6qa_1Po#)_2Mn+icLk0{5A$j|RTIk<0POLR;DuH+rElHcw;U>Ql+<>M#0MB2M8 z&{z1LJhR0H*+}7fBG0GosWOw)Gx+;!k3~L*M$wgUa)GtFUZ>~me@F>WG?1O9@Fm_B zT2PDwN}lK{zO}jKhB04+VV{41;L0PhN(`WE0fnw_-$CS` z9{*XI9^oT}imJYHRhp*xYbXBmI{)kMgIB8UC}`zt=?|;Q_wW@%n!q`2RTq!mW5t04-wn2ndJ(YD{s8-zH=GvlR*P z|7%b5l9mJPMlW=)?2bs8gPsTQe_7Rv+eUNM8SR^ve*CY$`szYI?_pjK0N<@nx77e8 ziqpq?fJDb^@iOB~K+}yTAO+%f6&zRSfQn0H4X`{N>`lg{Yvfq7Nss}h4cpvtOS_%v zG8+a$^RBVnW?eN#ncf4{je0dp8AA23hGZ2S0mg6vKvlkdnWsp5vl?q}F<(Bt zGwIrx!skB4^=YK>j>|xM@TKNXc+TBowj=>iE2z$W`$M|M!le2ib9e>Y>)3I`x%BC; zY_xaeUYoW=)pv+dfEB-4P^IomD5QIlZ;%69T097+l_412n`#J;Griuy425tG01_ zIg`6PX3Fm9?qc_JwD=^c;G6-nd;vQbP=B7E3(7a6te?m(LM zdo$@!sYi1r@^vpeF;nkoS3NSADW>ONJ$54*+g$N-Wpmf)VHv=dwb<5h8m7J8zFRdH ztfKp4maqIw>nrZfsv7$%%@|0})(49_F+;G}O>iaoT<3F554l}|=; zcQqI>s960L8dc=#>stVj2gY}1J*i`1wE%(Wb%FxbqK|FJvL2#IOmLF#%0xJgQqb|O z9fj`W-C$WWa09{XDP2FuKH+J(Lsmmm=u|-kME6dz!Qbqw_s3Y<;0*?jIJn=lTx;r#s&! z6mT7YWb%U{D$0UX&lB@@F~+w06DCQL=Q@`-M;7qzr%KD|Ny*%9`aliw6I@G6RaYJ@ zt~LrT#g%+VquSiIkUS7lFm$UB#@ic8DVY7`bGgMbQQEp6DxkgssH>b0P}>ZE+M>JO z)EN{&-k6XzzGJMGwRgdqVN4^l8pHfUYGbx1h!dAbpzz6gxJp{ zNujO2rPiRr_npV_=~i8buZ}O8XxhYWswRKy7y$$57qCoJd_$z&Oze1^0`fv`RgW@I ziKdKP4v(vMX)86=rCSz|5S<4EvvM*WpcaOUN6!r1R$b20+a3(*qW9G8+uA zsNG)Fd3u^XO7SDzD7^C_HDFThfOktgn~Qs`y_^DM&0oyBZ}>4^2iOeqy|H}ObgIVB zNHQC>9C1S|o&==Os%tIlr|=4h%EtkA-p5|;C8J{8Nk?+b?Of6}-y=>kr1P@Cq;0Z3 z^gh#;*QNtu`K^MwyrRk@s{MfWhg%;*CSL3Ye>)ZTVFuD}*I=_W8f%gJlJ9ukh74)N z(HDB=bx-sy@Sai!b~#Y{z;o^`+Z}hk`8)9^Jl*INTHr`=YPcBH;~oDGgbPps!B-c+26t480-A zvi}Gf+|4H_FrUaL@q9bouQXi;Zh-;-r?{06l;^)_ zs_jyXZpU+#+A&4W5>z;q@=}>6?oho;Nu@xLfg@5rBTsq=%*b3j+`H|63!_SDF_y3(WvM;V~)`Q zRq(pe+FAGX@E{#f30^4zvi?h+UQUfa|fQHE$U&oAPX)k$+tk z>`$=n!LfFyIS)f6yE@7Wbq)sW>~Qoym)sS5*ILVo_I4x*z#_ zoLWJc2g%DMOE25^F=PxwACH{{<5E3kO8aT1+GgE{+QiKC3Xa}f$`x~AZ(*I9i<0;5 zd)ePHx4RPw^*B&~-~{KV=Y^EcZpNw>Y$^Q`0Ohw*>KvSJ43>YZUCy+gTlR@81u77% z$-?god~;c}_8G^KtRh(CtJLBjbLBD^Qg-g9{!HCC<8VXhF{`WUDo!Zwb-6|XZCBvb z|56gJZnLpZ-Pz2KjH~yS6;r4MlL3M?+?Mx12IeH;G>*M2*$#i;{KU5%?Wib2$O)+V zqMNQ>lxZp3!^|=Vy4|fhkZ#b5eReo$d~_>%PfUT=W;?!iRFYpmi30_#0$cZiB8CVa ztq@5!8Wkq)u!qP26=h8^_n>j5 ziDh>NV_I?ASBpO$nTZvzFLF8T68K3wUxpCRa>Z$H2j5BOcpfz%J#-VDpw{iyrL4{_ zGFX50QV?Feh1q_SP-?7Z)(3>HuI*mlHX|}sy{ZDOdZ+|H&6ES!`nraaMpn%R`{P&%)O^7Li^9$iv-YfU1+5BkD>2P|Ez`!Cy&Tb< zi#T8(mu$h{htP;D0oP!N0;nDY!joE}I>#eNKcWR>e8T~XeOv2Axr6G4(!SmSW)Q;d zJk}I;_THac;zoyijt`9p;L?+@K2B;A!x-dHBI7cPxPoWxIsn~Nu}n7gDhEK(^0FA zG@i+p4HL~z%3^+zMTS-^v!Z!koqAe%^v73rP%#58NJ!wopi98pMe=qs7S(7!3FqF} z0w#>-SVtEh)$0w~FL!uCg8St7Q?|)T4d6`93kz2c#i|sKmtT51jv*giPc1&+S@So? zO;6%7Cwr>~!VaIli0lD9INRknu3pAT;Bf0H{AM!B)R(*?JyEO&Cbh_6spwle%;^Gn z<^K+601`r+UL%h6)ibsytc>WOx25w@uW37yx%wl;MITUD>fz4EhEFom$;UX6g_Gk4 z9Vw8)Ao%7^`;@QC@-o)&e)$y_82EKG->28&3SLO=n# z^{c<$_Wsa|$;+=H;tPnp&m37S54h*SQToqAH~c1q-#WCU>o^U95?^*e+Tr&d)VtfE z3%(O4SM90gEyFu?=aO$dEAZGe7G8_L$d`cSO(vD>izjV4aJLtKI$J2_9SX5jYRayC`$UJRT^PpM7dba(Lj> zG6eoM|GA<-??041yfkP5XL$VdRq7cd9XEsJ(BAE_-Bf;4H+2jJ3(jDUC9)I~ZSh%k z#R6-rg(e!^AnLwxgD6h&28@FO`puo;o?b|{AD4;3ir13oyGAO?OC_~Wo{1|z&=tjX zbXR!rCHx^8O|+%~=o(ERb+kbJGBHT5vf73&C>8XPgg4x(dEpuUOBA;Bl=D66*p`0w zZc9;eVpjDXTPDLOb>`{ALBT~sp9dl|H7tCy*{W8U$R;n1hDL)c@G{pENJJ^~4BCyJ z!~vKnOVM96=Q?{**M6rH8$qf+ZzM=q%}YXyR#oZuhVK-*dpY|1fbs@c*B|bjbTJE3 zcgyny4?*%l9xWu#4(ztC_borfGt4PNb)!mirN(izM(D1;?7wMgT*x7bs?R#i%P`e- z&eZa8;>F?FgJ?Z`lgQazceG3H*j*lXm4B_1LWR~9$Sve#Vp!8BZf^W1@Bep-Wd#|C z5>^)xV`Sq3G6cu!&0@da$Eq-<9O%S%dHdi0hLy<)iRmj}+=0>`6<}<&W3ndv&`{nk zCmbWNdGV}Ei)~us{-qO=X?-9qnrZ-T$U`Yr#>Jq6u}UMAcK>~DC_8*vd7{G}@|A$C z>y^J#`v{9%QL4q&@J{&oN!Dd(w%<62x0xx$sAy4Rf0=3PR3^l1Ih0B&c zgE{TT>X*H-ky=!PlBmzVq+D`3hl|5NS)%!JgU=|fsR4pTBGnTp0QSX7{(P-QS(+YL4O9$SP<3W06xtJs)`Mt9a6NLyq%PEA8!2=@&xj8Cd&H*T(xQ^iQFJR8 z9*M%?KO{P*0hXlnTxuRJ9c&j+$3BJ=g#@6(lYDC&8(R~5xI zA#`207jVWsxyMZ8mmtV=c8Jkqa3xu#nG91R7=M^b>j@O0T}=32wWAJ0gny}KK&DCT z&)@_LA_e8na|;wDADh>p6!7JhN`QyralPfQBnb8IT9IAyiX?v-h_#opgGj48yBEI2 z*MBX9e{ay09feu7NlW5{O|8P9`bpSwOIV#8w_Hi*nl-olX#>5Qk>k#`#kN_7Y_a}R zIX#(J{mFOb;)k$k0A-h=VZ@^RAzt!${tUH9SA^=zkuc;My;6mOM|< zlW>)~cNNC)FwLYH0jd;8R40BQb;-Az7&aFA6Z*y>+2EMRszR2hVk=|ZUdLB;_2S}=-Ee_f9Wd!@12=>t>*! z`kSUXMV_jT;G@)_hhA7TG3@F0&=aliW3m%E(W{lFoAEnR>_1h8m6|p@{ZMaLEYL#d zff($z?CQ3JnL>s7!=Abo%hlyu`GlBG=gcfNt-R?7Q=;_}fVW@IF3@g+J_>~FPZdZ5 zLXaE%_z?R!7tg(Ti=N1=t!m=RB$V0g`A;3wzDe$R;>1CBW=iQeu`etb3G83Xwe&QG zq8X7E!5so*RDA~-DEx_O;Hws=CkZQa#-+mwlv-w0l+6ATEfeuzqiGox$zb;t;C-3r1AV0sj}@{?Ihy7md%J8FMY<4Mu+vx^an_wB{a(?(7p@&9B?8x4r94v zkZAVw+SD(FLMBTej-UEl)PDCd& z@;|H^AzVd3Qj?2*rd4!PKywfG{)`SSbhEZ*YUZJP=sx4jXSmp=i%c%Mxy#aGtyI~S zIbjy)5ZN;K0h5JVGxlns`|0xG@VH}FJ#EYyk~K>1@gl=bboi_h$}{NR%N=Nt#Hl1{ zDn=h)%4*x8OVXTb+W64|17TpM^>C2FH^0m+8&v@rYGH5)x@aJ~5B1Tq${cs@naF^M zHXbrGW*fwGaejYV5{=UkQ#usQ3nGNr|CW|OHmK2Cci$Q(Rj#Tyt&s9b(%4-6(}JIZ zmM|A9kAPJ zoIo$WoE~$Xl$`+0a~?6+Up`0w%j-*ix^c?dW8o3CaR1tl zV&^t1EI1N4RZTu!_#`9?R(M$yL<=ROEA`y<-Eqk${?+A?@bO1ew2{fN?Wo?Aj}>p0 z$U=^yyfBB@t2xe-$IZw{plAD@{#GNkm3hx8&0bnG!SraWghxIX1)3s(u@KQ$C!QDx zd5{d)O*Y2sq@Aip>h|QsF{B9gK_#1ol_^3kai3!Db^Ea=2H1wmbziREB+$6K`OJ(< z*IUT#0$c56VatkIv%UX#;)T!%xxzVB5jeKuCb*1cqWY z2aJ)IB-I_1Fw;`uN9{)Bslj8@yNNDlO|=;F1fXYih&V-@>Kf`Lgb$^9X6|z$;+2E} zX{Vqi#Aa!lQ->Qj1bY<|pBm}k7n&0Cqaye*yt;1k)z1*Jr!!x`UpgOKWG`S_AySeg zojJaU1q_HU^v!UdRwd_QMc1w$t6>w5B=JElMeddG6j1H*>jm-Ed4N8kgrlK}-P-I< zT#C8-#hP3F12S%o0+~yMf}ZtQ=7?2!!H)%)DE0v%9B8mJ z^XVsJr9D@w0XeU>!}g?|Jb3FGA}Y5RT`Neyy;m<`?M9Ea++1-el@za%optz?Pg{5y z#j2#-U;KLt0MjA<6LRT`YXgJp&Nr0->)|hv_hUc7=9pf;MvwYNDcupi(r-oUmvmYD z7)HO1;Q%Vn3$3iD1#k+teQdVPfl0#J4+{hQGCvX!p|WGlya{hz(DX4l>mZEdPCwrE3Prct2Rl~#KZCPN27Ilm7@GHM-4t1e!Rw0DvP;13L zT1$c{JVCVe&9+_b3^Ro_l^R1%F;ECk>@H^6J^P$cALi=kt@?}_oh@HxgroU4n|P&@ z2ri>Jiy_-M-G>MFFx;F9Qequ6Ow3Jwg3=hF6*^AlcmSmo{n|c~j&O@;cn^?w3^+Jx zHq~M=BLQLsVpos1tAyi^B8iHH&bgcc}G~~RS9-gYi2H=6s`YNdW9@N`vq^Ue^}3W-b_9_{3)bO zNQ^oELfgDo#l8v;=j7wW_<>S6c69)V)0gAtp<%F3>O*e|h(+5HwI7*_)6(Lz1V6fo z4z)6c=ItUAZ4NTi3!M|*v+(uHrRO+>JP>bEk|kSW z%-H=3#wB!?F&G;N8^jO*;gs`fM~44ep!b5PeEQY-Tv2SvISJ}qV-d?ZwU!`Ac~M1u zsr#Top|0~|P4DUTtD>9zI`tP==B2Zb_MP)q?<=>Z&;E+2@6V!)!br?^$}GG-D3mbQQnh^3 z1_mJ^_kuAe127EN*}K9-b#YdG99`6bL9dj{4b|D+rWB+X!(|dv6F5TA*$%&A)e=K4 zu6f+jwpqc_`#||;E*Xt8Z|Hwal|OA#bE0zux*BMW2ka5z)9DlPu7srjhw?J5KQ^j=HBaqjOaoXZv4HzZ`oHJMpC0JG`i!jn ze|G?i2S?OS?Oo8YuyCX;_hbVT1k{(7*HIvw3;F6p!^5UsJ%Pc|6BF^DSF5A{vO;FG zuxmRu>4-HeFR$W~p4ipo$kZIzMuv`yUQJK0jYnl-V1&lVbnE+f4sGdR%*@hm2=TuS zR29{D?(x35gxTziI@*iMvd%Evv(Dn^QKTd&nfHwplk6mIOcrwNs@z{Ti$_P^+vTP1 z7ewe^Y%Cl^$jv*|Nu;{(wjb6})w5LEMSX-R&kxY;?Fv5OvX5(5e-KXlufg^TxEtUM zHluU9k2*TSr<4T|z~bK7-5HfnraXMbs`sXB{D-Z9#o#j`J_AuA3$pvo4U-l{^x7DAg8CPB zG8Oi?$xbyGkqFbe$(#bS=L*EA@VTrlp85zgk9cfE+eg9bcpUpV?(i=AJszh3%CjV?*ZYm4O zmjC#r+m!EXQR_&3rxz~@vSRac^a2=W8NY5tSXlZQyrA(nnM?*#btHRIxMb zAwtsNb_9Y!`M##->KIg2RfE#h%bMu-9!oWQ={o)P+%Gz}h~{61aN8Ha0M(!LN+>qE zy1E$+`568zSjO~c0I9d5ijB1!(o!pV>Dy?Q|Ac5SA#xi>6b+`u^P!E@6>!5PPsAn{ zuE##@#ZMh;$>v66=*0;~mT8)rU^(GqAkGp~{}9OhPfPlYaEqL%FenE8EpWy5xXj4i z{QfJ4GJRK_| zmrn*)Fn;<(nYaT1gWiX=>#{pnDNkfsXH@G=q=guZB=#c09}5B!(O<)q4eyWJvd4w% zh^U>qR-d0&8(^}slZU5=2zVTpUuCQJkZ%j7#3eB*Y}QOp;_+sZ^e7wc?@ZfFU;pxO z{)R9h_~EF15x&OAqyD2gVfKn-{6`HH*oGn2F8e)hRLNt(VpeCnq*pGyNo_hda0JQ8VW!40^;tp?^PEDwxY($@ zBEHs{8i34M@R$|7e~u-IM+TDz_RCZ%M?`D+`o1ahw=_r{1$u)uZ;8a~X3@|z_LYbK7n&PL_md_vazHc=RSvo$SnV)&Sm zg9!;O4iT5#a6$xKybu*+C?S~lpHl+UI<45aAptIltULKR7D#9HR3dkPuPB6cpe&63 z9Xal};|+`l6cUS31MQrjfo72e;#==)SbG_j6RQj`Fn?0+BYuiCs}K5lB)4j`bw7PP1`Sv)K36HJuBHz|cS4wtWo4pkH0%nDBy$dFzad z<~g_~tVwQ(;O$&-(Uy*r?6-HhdD7BYn}PTeh1?;n`=i~U)vazsPso6t&aC|XQZ%IYc6$U;*AW!_IXq0wWVLHd@i+>% zi%&ErhkO=rGJU??V~L6qleT#>VMOCu18_+kQ<{x&^kWi^@B9biu0?_9dw}Lo)eA~V zclGnd(aV&sz$*R3_f$HhAsH>cO+@YVflyWvQv|yhvUHrStb1fEQia(D2j3I~6*(vC z_Et8raCB^P<#no<_79I`LT!0foP5s7QJsMAT!qx3(lXdU$#(IEK!OmY9-B9W53~L5 zgqL08bxB8Dsq|+<+QKj%`1y3tWzlrG#^JFQ`^GcLQFII@Mdt4a#8)YCvAhnCx$~V} z)sYe;?>ihN_6Te8d{7}mhRx-c6+v6sMjND&Few6Y(j8Q!lFB|#ya5m@Z5!2|(jijY zP8oKos@Z(FR zIF^p_i`O?3if5w+pbh2y7J#^3moB|f&!Ku6`ScG4cXd`y!vzI?1-vK`N%f&`8V;%Qx9?4lz7QKw~0VYak5fYl&;BG{BND zK7Q;+a}7^U*<$G_9hLH<>Ard49sMs8- z=7uK0yz7&VK{*S#U$lqP9hLgGvlKBKMTM75_>@su+x~&vd{*0mD(ekz`J)@+S#*kd z8!-o{Fn*%6v7=lxM&?ETUc(}f48?0pc0SsQh2qOp=}9E#w?8e*_5Izs2B3a9Y0@3huQku4~^SGR$C8E zz=-J(D6`x%Qpzibl&d8mx*$`!6A8eiwUUADRb^7Qww)vArn6JuCd(KZ20FHOkV@(9 zxuUKLI=yzK-c3K2kQ&$@cQQWh!ryYHPLhDrW^#sfCpLZoq87eY80_J3bJ(VnyqZW$ zdXU)JcXjQq@Nj^NwS{BgvHuFdW>yQ^rSfTmGcGU*FIf-?c>~M!p1-Lya+`|CN~G({ zX*gk0RgJ>ym}n{`uPQ|LuT3kG8Hy&=++Q>>ih0iNd-(|InGYqsX&Z{g#WMWzxc}Zi zH_%2E!#LDAIX+GL5$At?j>PF9(pZ)H{3V!$Bi-H79w2PTz{7s8rYMs zhHWRg%!VV*BXlVCoO@M0znl>;t96)d>OQO;8RdqS3-L?SCN;@$6p4@WW+^VV%oi-B zbi483KihWu!W4?Cy%B@FupaOCkN0O6u1N))dRL^$@{Vv~YpUT9w-F8oI`TU162h4w zuHQ_CEGZ1X>SVd${W)cGsZ2A9V@Lhg(ucUv{@dig4U>21{ylxjf?E zL^$xCH#j<;{VIHe54?WoVaZd0`F{WVF=bxsyHBVVBj|A^RQCV|MA24G;mb_ZCg(i*_xRA@hWs7WzrO4}IdOmbjA8*y)TkF6=wXll-s%7TJu3$|SdFFV zS}*U{+JFC1QUHA7|NH&_#qlE>pBpgTmeLOv*&h%xv$Ei}dCvaxoxFAmqMW@&zmV#+7kHLVt81LU-3Q%<$LD!7 zTVlpIjosD?>?v!G(EdMfGjWm$upabhyUt3zhoCxY;dv=}Xa_%WYEa_Iiu z>fN`sA~d_9@e}ruM`^MDeAOGSThyD}f5p37+!yOve_vdtuXaq4xu{wByX-s!;b7|j ztlZob5k0-TPw9R@L8(8FMdtIJMg+b1J5BE;st-V`^mUIhNcsJ}^?$w#T+qmRlc-}! zenbBEhyT9LjeaCpmK$yTX~JXOJ#}`JHK6JfRif6L#rW>Fw$GzF1@YrGP96?Y&Nih~ zG*nUsy5VHkte)~I3x+qlrl#g`M~?A1UseJE%1z4EypSeZAr=E=E2A+Gthsp=*<4jRNY*2QLH*qsMJ((EFq2|!l zgJWe8>e|MnCE)dD%16}5W3v!|Cd_jHoXslU_}Ya(k8eg-VXG@GRo|PWHM6Q|SXr^D z$d!tTi(3?`Or^8^><3qFE>#2{H3`3995UKFX#d)PV>K>mk!5m=uYlU=_aI80;I#hQQALl zGhoiz+r#E#`d1;juYGI5aO<9Jo&&FX_WT%R55W2+%s)i$Y5cwY^zvD5v#v zdim5?pa7)gCku0H41vLM-A{)(A)%~XncfqG#VHVV8_dfqM9H1a5$xnL^AVM~^tdF; z4l3p77|BnGv)r|-DK%4?tT5Y2$j-PbsG%WY#MJtt4%1%8{$v>~BMpeID*an3))dTj#6GoHVJv``6A0#!@eJw9%vDwloy2-jo;L@Gtl+dHVO`%3ryawZoDIS z5B+fvlXe3co^e+Ws7`)bCaJhY2w40?DpnqdjIO6)o>>PG_6{5VA>!T1T>{%+&GvD< zW-1RIA+`R^aiN$O!a;(m0g`)$m6v;ihREO_2WII>gO47yH-&qKXoODRUjrpM_V=>tVYeA?g@KHJ@7?DZ|8Ic!;enTiFIvK);ngXXpDN3n{v zB3iR`<<(U3&D8cwZs=R)z7+9@vf5QvGi)~pF@%!g5)v9pbQSlb!jJ|#Skd>k8=|@Y zIHNl8W}qXW6A>j6_zbJ=vEn!~RvhZkYw1*W=U9+@tF*I9HY6%XK|LlNJ^%%;m6O5g$T}CXz;zIX&SL_pfhS zzYWeI_Sd}CnOsf#Lip~3JmzPjub#k*$hQ)XFZml+fr{^kVZxV53I{FXt$KGy-03Q% zMSZy?Dt&$LMy6x=rWDO-$P_28NQs~8b`A-hWxi3F__lx7T?(1@MT&p9-?*!Bayu{n zGYbnBervPIxC>LE{_P3#zw{X1T1a(QB{N?}CrhNs6g6*YIJIY=6va*tG{{Xr`nAi;zeL84XD zRW;g}Z?wzfqT1Yf`6+yM$5 zQu-3+6jpA$L{bOm6+<{zb2#h@5NIKz!B_NT1l=VXDF%X?gU5w%_-iJ<(V}ae%12W7Pir0PwSNt7ME3@$fDO) z`PC{SOEdl5K(q^@m zlbs_1UC4Bbb4(37c|ciH3ggADo>zBMH>p3lSex&o_1QhTHr&5}XHg7Rc6qNO1QSh8 zOz6ofx{oO8`;Tgax0HE5%~Q=zpSan_m6t}@ptvM}VGX(%8qtTKpQkH4V|%oET{U?7 z-fW`sh^>$nF0Fs1)8*|mpqZ<|UkZfE+f|YW;i8fl-NI2rqGC4(P*KSPz(>X84r`-j zW$`hw1rwIp5TibytB(}{>wNk%iAj8o7NbXz5YF`Fv1iX`0Zg`N3=6Q?u`Drn`)4i7 zYE2!^>qjh4*5br;3xsbjTZzear*4<{k3xprJ64EKx(5Zfn6z!2lQ%*xZwH*u5+=}R z>!m$!>=iDFRw9+5t+(wqN#8+FJ@*3ABapQ?Vxj2&>~Fw_rn?WPxh{1yPjR8bE?FLJ z;K1)Z5hEg#4S3sv8P6Z#CuKNoWsF>F>pBOmDR-(d#?F zn7{hKVwI4(l+v5o2MlaY<&6~1a(AdK^nz+e*Z*khV6~FeOTFhKUSr?Jk%{m@B)$TI zqks>0+z6D!SGYwh-DL}7qeQ8cjmBdg3q5)FTgXZK%b=IBOaE}h08jv_-3v%gdIMY< zqwbIc!7Yl1FX3f$!y9aseON!}2dW$vJI>mI)Wav#Vb~JZw(HX%Q0pU+TN-Y{!=(N+ z!KeGRfXIhPSi?$;u9>(;5usii>Ro3#fb3E3inTuE*B94S8G~r|_To8}OAqI7`iBQY z%6e}){Nub+SEORF^n2Soxk_53Lw@*A#UrNK?&Rd-@@ zot8oS7e|I@sNvepKh$`=X6!i6;#+^b@aln^d6HpUxqif#@6s4q)vPL52u;Ak&CArw z6(RfTQr-~9vc0qQJbCGbH*23F>O5@SxO~arO0Lt1NFGrsUf2cvxAN%?;T>uhNkudF#| z-MUU2lSCBuxx*{*<|@SyN`wvzlt>r0Zw#$C4=Mvy5xTCa!ZIg zJmnkssQB&1uI4n;UhT(3T#<8o!LO_{-t)AF-Xx1St$*thb{~t_SI28n7O%dZB}aej zKBQ}7#$L3^Ra=wne~|w;^G$wSe2f(w<4m|>gdy)gU=s$@mL1qD)-8M((=|Ln@d~Xp z4QO4Lh`w$dC+Pl~YONThw(~qEA{997M9&ee&@Lp!QaNjTW;UumJ;nZfxiiBiOet$q z*iU7yr&vg#QgB=t8SJhumAmj6eYH+(@TovtJQJ&tCI4ap^2X=I-yn-aw<%=>A0_b$q&x z-P&0GNED$Jb!0(bq;9AHe8uTWuifjsK~0`S}Z7%9y~e%+sZJd+n}xuXz1w zrehCb$0pX*U48&t;b~7rlVT)^7jG^A9H9(W61xy|$?>m%6)w!jyt zx9P7wTawlG2Dd&w_L+$1YNc~~%7;+;KrrUg2kd^uT;0S)j6x~=V)?y?5P~haegLIx zu|D>MK0@ij$Od!5TXly{GjONA$>2mHQQ)9Qi!S9|oDQT@UhJH{$Lj9^1I!R0HJHH; zi9CG^FRZ>1#%yM8jWk)5Odm-0Qu&#xvNNwKTVp0x$MW+Ln^F=WT)TxtD4^y{XDSF) z_ucquiS2dB0dx5#5qrv#gx+fd`1T2%o!Qa5yQo)FZuuPmL**<`LrJSKUACbSq)TYQ z8=k_FQEcv*TO)r;DoBnPo?4~?eb8BC2DYs`uBEA{dL3?sf4=eb+9xMQ38Ep}n=Esc zaM>Q4$-F3=pQ`ryDx_y~ds6M(Z!9ZosiC)51giZj_7G9Or!jzKdQI3|R`s>mJ1Ff< zY5^Kgk^7cx*m^c{+G7;QR5cie$g`)aAWLg2aU~6o)~}T6OM~JQ@L0!li`B+7U z?W7u{*+(^?K zz1tJVO1Bb{vtJ(KJyVAf^UguEXNwWKp31ZiB~vfRPRAzA<;0Q56bjJzK78cNjN#e32`1plM-sMKzg`BI1l*rP%&Dw!8K7rhp!@hJPPV(n&rI@W&*Up*$|gg7GyD6T3^){) zvx$^H_uGF*1AlJ^b_s83K!PlwfP$W1PeaG!DdKF(THl)HxMj6VLUIsPmz?oUb1d#KNfqXCM6;e52k)uNmtEurMBHhu3v?||zc*%hr2+|b? z%gd*JQBf)$iBX2@5iO(dABr05>vIU?@3v7RTl@uU-XmT1_3I!&>H>h+B+ixm_9v9Rc0N=M0~S`=i~5L#vc@A z_HAWAK00X6zP|0W@lx`HoIr^0R=Z?B>ha09~5<>U@FFSpKE6d)<&fXPX}~i9$PSChDQUQjmkOYvF+_vXkuR zn}jKAYcqCTQRr7+)||`UgnkH)qqB^gTOLM*t>2>|YJajpe^FUK69Ru)klmEVW}5cf zjG7)-86Wlzc``m~s{SYmxy7{T6r$%@?d@6wr!JkvPMmJ1jGeflhNFa`_r7WioUz?3 zC_wYYn1-_D*dY`}sl=wqTA&nMh`3)+{{lWs3_;w$TLpzhw}9J`n(AwEgQdFuv!2Qa zl2`)E$CiXUbCa*Ulxi{L5k-*xy$EzqYtv^f1haMk1moWY;3)a&dbY|`;gk)-rdTnk zZ~T~pizd7Utpr2xdr0VnZ2B8h-$@P(KhprHq`*T&Km@iON#tD;Y6cQYP`{dj#y_7v zOdb>5FM13nHP@u8sgPOOr1-~N%JZ*AW{uJw!T^8K$Zw`v|06&)BvF(NYcq+R4W_l4 zV!HWtN(g#nCQ@HLM05Y1tiO5d&n{n(eBfwe+5qqRXOonU@s#Z-zlnTGKht1(zj7J# zU*z^L%J3Nw`Df^5K7Zqk+vlbwZcSBO;y(uWfT;dE8_uc=?4oX&@SY2*GGOmlx;q1E zDJp83z^2U9q29Wv|Cww2^a!5O6O|@bM9Fh6d%UOjzjpKwF)uxPf&cfx{zGa$T-b5~ zP_v{(z>?_iDfHJp06c{ICJ_D)H-^)`r}&Fl&p?7jaDR=o#qV`~I% zhRVv(1EOE7hz*4kUj7~f3G9bZO=noDLU6ia(%9I@O80uX-0$u(Q|R{~^E=h$n^__Q zAx0ixMH3)a+451*vUhUSwj8+fa3)((KMZ{W%qImmy}#l&FBC5DHWd^XbfJ+fpC zzdJiD=hsZ@1HD)KUY8oQkpKd=gd5z_Q5k0io3@Hs=?CitZiZu()Gzj)D(?*A_&M@g zwo-F?I;S+kz~8z_(h}=_0P1`m-ACnFfoIFmv3v|X+c`km69_k!@jR#Im41bMB&(&6 z`UNe7t1{|$S4I>bqL8dkbz_m~43tb`9Qy#jtd;!T>G!(mXNBRH)bF>ox9-93{yGvP zqJFP>RE@WA57VizP1Aot*I%4o+9=TS3i9+}3PiqQwoQ7o!9GOMwRElH%qWbqdo-L-u)g{qP709}KT0ZOKGf8s4 zy%R(Mri7KNja9X7>ZlbP@s z?(JlhxOeLz#fB!ruJlyL>%^?e@t&f=1VWtID!_e;guSsFl2X8CWC?*{{Dz$(s~B~T z+i}Oo^Pk3Rp0u0$sn&Iz2?>;TNHp$j@mXnLl_J^Cp zEm}!p8ZddVd_GbXpF5M}-yO@SDMaX3i!x|uL_nJ|A=uJt$sXiB0Q% z0Io?vNj+KoF{1Eq1vn5eB{Sf+(+v=x)c;>13)~#X z1fofv2N<6L4#rp@&B#Q2%|7cO7ofmi{~59O%)c1mQnL$kRQUL8K-Q1)O~Ti|GTo0X z7zu9CEP(X5fbM&AOd@%}1+M{%wu>Hc)>#oycx4sMK#~oN(;kp36}TSw-;ig~tJx9& z{>iY#u8&U=40#PZ4~ruX$~yM*wl?er3N;$NCg^%P>G-GXMQdAtB+5+=;m%S9b2V|jQe$p_ZR#oM;TXF zSGh#q_wO+Y39-rqoc}27+Z-@W_v^_0_)4g0XcP z(pv3FD2TMYL+=@>>##X%SPAZ4a zE&K4ze{7`)EW7C9oFL-r)P@%$DX{>_&rQP8+HaLZH|i~Co$+gN@;^f-wg{6}=jIS| za`z4)MT`GB%D_tZ13bEhnm!0(|aY<0pf{&?%w@^tjaQbr7#IGs!a-(`|48mNfg)3`v`n1!Z@~y(QnQ= zbAonX4jQUzoRK@8jEuadVi%MHqCt{Rn_Ryrx!Rmcds^2}(R_`VG+A(PaQc1{rwYWG zSq(y*`2SdHDKfCn7(=ha;Tn+5R}ux#eVJG+EgN_Aft{#oOoW(CP1^!`&MyYjN@Gst zsRDsq@I*5j>3dyXQ^=)W1Y*x;%E7Hm|UzYrBkMe1~rDHm-iRueG(4z}k(uAF##`yS%7Vz26Tb?~3{R-C?B77I1-DEeg z(^J^^{(ZUeEM`Guz|pH!AX_lrpN!#*g6^7--&w})b*4P*1x}ecvvyo+BlRHGCR-R_l(B94U7rq+W_Y9mXX7#bngQ0>H6WHxXp6reIUL!)q(c zr=sm^3lGDPAezqy*~zBbLf?^NP$h_9xL)kuc|SIF^g7Rgfcqp+|ER%Ed8{c~j5n)t zI{tWF|HV{?lEDXH8RY1it8F>gpe&Y~{-A;J*PO~yz$^$sV<51q*mH1T z*gA_lB}(0a;VI+mGIV1z?eLQ#DsKLfew>v$NNOtJ#TFo8`;(1L&l?nq zPm+mIgLBmvCONjAN;QSJ0@Q%qudd(F@BzibfQq$!zei2I1_Hez=LcNLq$scH78OYddT3QtlE1dq*(^I z53`T)?VKHopeovtv~=I4g+k((@Q@dHN;8PY7X#pn8sFG(yQD>_;~MutoS^>*N&H0~ z!9MLK2{iyoI}&W-@OT{eQZEdV^w?OHzE;nX_Ci1!Y**hWF&V|7o zQ-Mt}AS+n~xhPJC`14{ z6$^i|Isr9O5V9*R9e}A_ddY5_2q@GH1aeP_3qAq4J5!}Y&X(%Ci9V+#a8^Fl)HDP` zO#3&Gh5w>jP$oB;*QJ#~&Db-LzP<&$skP$#ne}b66dJr$(tC4jxmLMGxNK{k-zQOkI5Y*hV;TC$}?=5myzFy4P&u$1(y68G&5NIp82bw zV$hiO@aV1OzSXkdjx==yVknCk{hAIPJ#Hr*`aQP7_j|1WDU9qJhvkejZuJ08N+y*< zyUJ(p21enRT%`3s0uDve+cxW>TcF0JUDH-*&xpCTl8=TMA(Bb?WfY63S(C@NF_q+E zNC02iM>0Dr_qzSyQ9y7MP-9q46Me#!oNPFJjrmF694J8HJLtqF@TdR^SfT$q_^MmtqQ4RVq_w5B=9F{sk>pS zJO|+k0|`*y#}6=;al5;w_`GkTvJ{Rc}nUj4-c9^M7Yg)nKpM1~fXxw9o@e|ZZ!qtto zP=3^ov5Kyv0a?2?KN=TWPbr%*XuA&m+p#TlPbwq52GiT%%E?F^M)vURZ9}$Ewa9o% z^)4_80%$i{C8c`7VRcXfE z;OcIDHXnRRo(bPeJ@wfzeWc=fko6+Mi99+3%!8nq>@`RqmT=)>^F1BS5NWr!#a0*! zISLjbXlSqx|0CRoPePTXRN!5>gs+0HgkTbgYnI382WHAk+&8O2)NmX>4)WP(s4=r8 zk)qT~kiFwYD`{ngm|au?@}1S#TH(knTFKIB4zE4i=VyIt0~Krel-hzQ*C*+}2Rjpv z?41E)Tf(DGJfCt3ZJ>xg2-nA;A}jluO>Y;2eaW@fhOPQQt~aJajFaZVOMYD1 z3+c6(G+HGV%)N?CaxvwUR2W6qu7rmc?d<(#yc&tdCpxqJ7btoXdqp-kCr5|ecjwVZ ziVwmzpE8O^Y$%34sgHW_eD=+NgAed50q22t;-`%Q7%RL?6wmq%&N=8k)D8M{x2;fC zy_c}^JGRQ0-59wl`3_jPPK%>|tqECWK2QZsA+3OoeZIw?jwl2iT!c0umw>9fVtlXu zzBeODkGkO)-Wn7GT4g8kK?XvuU_$I-2mVy!u6!U3i-ST_)cQ0y(#1pzOap4wPQ)8u>#Q<~m9nc^XR+Fan0&lF#kQQq#aZmhD zuc*A)+CQq}F|ZondrlO7ZKIb{FA=iJ_Te#WD9?~O5a!5qH9mt*I_0pe6P}p8b}p7R zwGp$mF9R~8VraKr?ThN!yeCyk+3CV2l8yBBfB3hOz55zJz(N;|MKBH{{?_?7M)3B6 z#Us>oJQ4E8dfTrKfVR61=1D*QUDi+-r`d!-V*~gOpmWiM?BWI&Mb{9$3^|5)W)AYC zpm3G%U)r^Nufx^L!M@`@S+v=fq6JS2)Z%M5KDFM?)|-7nx{(l)hT!z4YrQ*kS5Zml zHl(t1Nc{no%(I#+>(y4XeEIqeP=RrpsAVQniMhf+jLE%l58pf_&XHk)^euml$Le6n z2eTPkzxSka2=O5E$)^Q$hM+r;KFwSp3)x-Z461O`!3r^Z+f~FH^XIdKxgQFWP|t3y z2<3b)iqOK=dxSgSF;s&EG^J8PVk@sTq=$cy1+3N)m0c?>JKZC~I=KgOWp*bZAkl&9 zR{;6KcUg7KR}~oHwtG_qPa>|>Zc&#PvC!vpkk$;}rH9)C6+GlzHe-nam{c)a?UL%g zWr&{OaT|U%uNxU}WoHJQns^yo)x@N|MO&f)7NqcSFYo$&6A&6PZdi8V@&ayD$o^!ymLM0D{ zGR$+N;Nb{ccaBBw2(o)TD|gHHD66joQYQUfL4Ehc1t5=ZQ-^^;F?^X=Xa7PhIaq*` zldiY~HyULog)wFrig!{dHYg>~{T9^oQdep{kO+k{4&dtm9SI%-SdxdsDRlShx%O4R zQ;Y}|*5W5EDm|UuWVG*ncv+904yv)4`I387r zA6@U%q(qQxFQZBQf+@U3&5!t)ACoBI#-@`0HZm{S^b#F7h){e}Jm_@N3* zHMFJx3!6Mk@fd_2sn6h$3b(HW4H52xib!P$Ew6e!+^>o38xBrM5ofqUYr~P)Qx-Y# zP;Bna(C;Ws0v<5JtIK@ld3=}* z5Mw)2fw)MXaIC9N*G2oSi@bUv%&LcXX?Nvh8=pq5iM>uVW3ERObYrmch7y;t1W97A z=Y(&7*8`M90^EoKmcY%~;iK)kVd`%~fXHz!t%7~PHglqF1f%6Y(W(SC*nS^Q!(T3M z|BMY8Of423kYb+R)F3`R&A;X*?5sXS&@ZNxHZt-H&9dz8@;)O@ z<(91#I{4OXJRDU!Kf?eNBZ|TGyhV|#Yndc!wOUD<>9Kf5hC?}3zwO<0v?LUxhKvpH zWmIPi_1}q5V9+A!lNfI^8oB#!X1L}~b$=e60hImRRfIX=DH0qjm6Xl!n46k8G0&Mq zc&qJZAx+4F5y_?*o$FrM;SWJAYFdWQV=NMVo`tC1k%|>R56NF!+xzjpvz9K`KKxy1 zsI;zVvM(?GsTComFM01{DP2c-*cBH$RF>)i`D*X1qwdBMS(CM#b96ONf}AY0f*j6H zU%eIceY(Du02NOrHDs(}?F&o(P!mr0oR#M@lCd&plzZV*Q&U4@S4}CExnEmx3=;%7 zDmI$a4%lCXc4abYG8M*ON*km-U01L)>QW@gID2ddBuLU}7H^0@Gmbbcu&HAP4(NVE z;FUMVZn-5L>aP>_>obtv2q);3*hDfUxz_?EZ&)J|i75dcMRdxi{30{#SE`cw@^u=8 z1&ve#OuM;HsGiIJh|LGWEz~UhGV4Q3(Lj!V>j2)5_%$(HVNCp{g#`q*VzocGYNIA10UHRnh zvF=A8B0*W0@bNALgLd&=jvomk%UkiIS10}vYkQyry?q(fPsu_n)7t1NN9=VTX46+c zEt7Ux!9HSiFZaGhbzz8Q(aK&?3Ie22v49*Qq36C1(7GEOq9Bk|0UHGpLq`osD`wE! z!UzTmjl2W$A?@D4X986}*6ML<`p|V<$JlH3lp zN#F$nKDmlEQD#xE^slt`lS>2(D%(b^65u84;EnaUOBv2wT!V40j&mQe&j4?*N76c0 zpA{iuaN(fSRrtg%0kAWFG=W94~wrVFQDzx>+gHc z5dV-yL2fJ?yR>kGjn?V?MTggc-S5^nnZ?h$%4P0VXUBx6rC*)>qG=8W?oF%@3A>kd zdGRx?XLT5dT)}>{4uK2vmiGk+f+YccH}34J3tsRm zi{Cls5DbXKa9UacC5^IyaT^EBf?CMN5a&WG*y{Dv2z8=Z4r~KlyJq@@oj?&@RCL2z;BpUpM3g!&6uJCNF>>l=>d9!(C8rMJ- zBrtgdQ&FOg{Zqe?N^1b zj}kwZ<`9VaI%Rm7VyRa4+y zm!n*Ys2b+aO=PXwWFy90SW9%SPulcl(Z^o{0u7uXDiLqg)k=9hfy!;f)sJw~Cgfka zNU(^2wu~9)9i)CwiJEO81q{KR8gdD1Tm@Q{wQKp=Sxr_VlPLentIg1X+LkR73@Uxz zJv7RTWhz;-Y`)NLw3`BzKbZ%EJD~X8VN<8IM^XTwFieb+Jdq=m+(9Q?3&oTdd{U({!H9VmvU zOX0hhz7==@BBpr-ZADMcBZ*tMR%kV$A$a~6QreM)mnIF!xQZ5H?L&hDRt_#(?+*6O z>lP(f=ujhQ^#aQg=3Bw*BbZWZFLCt9WV zMGTuJG;$kdA)^eDWAq@xb3^8Ju5KxG^~B`&398q6AJ=JXzSk4kvS8PQ0?W($CiVTH zLl{Px`<)naL`zw$f&8PlsBpB`ih6)|juh`53||gQZVEQpo34^S8#?qKhTpPNbFK~a z8^c#Z(A5OHSn7Z%8%_FL71W8oazZ8`gm525Hp~Z%{e$Kz9h)&!$(-6dQG~C#H^Vv+ z9972FbF>+sa#QwRc2vVYSvD4t0E zd}{JU5+PZzM_q<@8?131n_ndS<2>uq?5>UNz@{0nyvsf@VSgAZAh3G1SoQY1*a!~9 zTJ&p&W%Gz_XV5O|s&{+!O9=6KCR4~8DzRoLjkijwpiSCA9xKa4upA_e^_5;ASTI=r z6ye9M=>ao}JebUYmmYTT{VY`UQ{fv0A%>rGoc)FBYUBQpfMgYI#94{uKuq%IJ@RQ> zYt2K33+eenCYNE6X)pJi#f4uoggKL2Us{sje@flCLviyI0+5z0i4U&lDGKFRmhtal zSI7KVzGr#>=7GO{Vw*;s;eGd6dY)YS?&C*!Je1Z0TrZYgf>V-@2G5s=T5bf7QV%WC zZ|cYp$0`@RhW#}ZdJaHH*Ffe{-a@CGaXHTFn(qWBW_6JH!yJ>1dRF{LJJhxcNHaKH z7&S)lKy8)X9}%HV2eEFTG4dg>vFj4X)BZ>s6bj@+B!jrg8R%oLzDt#~GWJ#F!f)nu9%&mr6y zVXVQ@U>pN*A{kdZne4@~H9UhhX@Av9_m7$>BA)9!r7NROFH5amPs-zhHh3KHRMV02 zlpT~rtH{M%f6+~}8RiDx`kbs+O!mY?`M_&wU;Dl&1*;G&>5d?y@a0r;^k`xj#`zL=y>?5B%f?6L?_-e7uFL*&g*SA~k z&eQ208;!=82pOa47w6)KHbQq*7<+eDdY(3;uY%@JWRD@ELD)!aO1`$&rJhU5y-O4Q zlb1K7H?#g+uIVFJHxH&9x`)2-2W|sP0UZGGBb|vR2Ne2U*#6GsDHM_Io!`@H&Arj;xBaH~`KZml_oLXL ze+Li3_YP8}AyNoRKMPVnQPUR$Q*i+$L^xu?q)ABgSg_3GULSZK7AWn^@&T!dh~7x> z|6%Vf!=l`}fMG>aKtKdRQW22`k!Ao%r3GmTK{|%+8d5;OAf-`qNMYz15QIUxyJ6_= z=DSBd=lDG5{obGNujjh>G1qMN-fQo*_Ug3;`ff3I=A?xYmAroowU{vXT=v5QmkQIS z2X5Nm@V|`}5^}0V6^l?ku#{#u8?hpk-0t7%mvHd4!abGr_oR?8xE9oqMlhQ2h?>LE zRA8KR3{E%`)bgIRC2QsKUbkRX6cCSb!r1CQubVo3FE~AQv64N&s@a&e?|Zt+GCeJP zl3fHh)|lDVWRM8F95e753VbG(!-^+S>Hnmq;gIb6Zr#NN{>2UcjPr`ija0=3H#g7| z^qBu5_s%n)xv?pl_quJ`lv0gEBMQ9w;H;AJtbls(tpzIGG)86Gu}A6l_W;*K*Km{E`BBX(lq zCt@3Uv~)IE@`;xo&(jA0apYW9dz${l_xLFyrkxIzg~T;m^;HE=Zb@i8w4R!g>(Ep$ zNWolZs4XidGFjX1eBDf(XziBP5fZe+t;}kqf8MotW zt~LiR{8SpyohcU1OS>-JUM!WdA1!uWe8#H@I9vex%=EdG;xSU+{ASt))IExXySiN` zs5{y|sAz8>KI-%zB9*#tRks_{m0TAv)<(9Ivt#uzOJ*v3qqP6c+%}%-O z785N3>2{T&pXT136nMu^JEr#Z`ytw^^z*MzmcG(~%iCrqAtUgLsh^ELN_M;~#KJw?7pnqo1zfQ_~WCcS4p|1y&y%jgbRCtAA- zjhNGn*U~YAMV_W>J(P&!X=hVO#w83KBKS1@vY{4g)zXK1Dj}F!=V+KIxMyD*oFp%b z#YV?}4beGMYawpr3Et#t?_?;cRx93BS5C9hvL{aWTIB5_m%48!<&6tzyq5mBl5V(edT!0^KP{Y*e5RZI*ZA z{FUtRuD%_J{P6OOe?H0YM8Qwb19jc6tqjHN`fFeMZFaM7&Z%^IkTvBFMZqXKkXVhP z_lKN=3q~HLi)3eh$+_Yz?8F&uoMbXBIN!F`>Ww%MtPs?0TImSx%j1>(sHjiW#g>19 zTY{Bwdy--A2LCI!TIegbPGTuDkQ?Ev>-_R=;!+Z#tZ$@-LOy<$d?=YQ&i;u&jLGWZ zc6I5Eaoia8hrChxWM^yJggQ7M4jL1$6Xh4baZn0a!raC?MiR>aoeOCdA{qr@(|D%r z2A~Fd!*mW-!x1uqyH4)AwA;u1UCBjM>be!)F2RICVrnj8DP$JnRQ8k4&gyX8R5t*B zGX)Psd1TMQ1alVG0XxWuZ=Op|%2yb2f{Odp)j8n<=)Q`EPzWblb#=|tX{6=eCGN-u29^2X3wa9ZsH%o{& z828lPg!pM>L$?!R&R*b3r9e;LznLJZUe+_KU0Cps55@_6_(l<(lG%l$5&XS`&s{jO z$qaXvw6 zppm=;DzBGi2-jG(7Lm)49aY`A7sre3H%s9%2T>4K(wzVuWeZ zd_w+MR)2k?UkoT;V-CUzd|-fm*EF4@00BhxMr>-o1a}cae$Z}q)!>&-5oZIYd#9U! z`UmJxum*usN>`Y2%UsZ^Efx##+S(-iB#vlnYw2H(eBac-EJdxAs`*(Sl(eR^FM5@u z-hBb}lo(}{Dm*O8p1r&mz5DfF&1Wi;(b+`?>9{tjgxeAgt62-NesHM_yLp-GP49e} zk1;4*2t?Yro|Im#V`wJ_Ujmi30#iWoo5=L`t(G-j8OoPt6jCst$85*;flxe+=movl z9Sw!RCn~#qR2Mij>pFeTjvy86y8)A?n%RMGLdLbZsH3=~!1BuSB#B&?YeDiUUN)1c zHBUE-Z=?jlWE?%n?N26rBbT@5Y>577JNp}p6~q+YhYPW>B{U?8hOaC_N#1q;$m4-M z*lRNgQBKb7NqX_k(QwA?J{i$GD2NQBVC$>7PpE8hLy6Y@jc3SL>;>!oUoQw=ne#T@ zyR9)19-5Hi%NW>z)fyB;{3P)jK2%zeFm&M6(^%RTj=U=_N_eqE&Y+{ z4}7kK)bEWu*Q94#2S9Pf&Fil_as~Q%S6;R}&n%ZZ3YXfdaS@_D>&sXY8NM-69`EZ3Np+u%81l)nh?q_5s-9FiOzu%CrpL-ws zX^jVmLh5txEQTXRAjkcnLq?7U=!RVrv)YZjkdem4X6%|;k=nN^%?YJ#)M>uc_Aq4- z%z8pl`?KafDQ#AX86M0wb!7p!@9v2ck;Dy8;3-e1t>9m4+h#!@;A4&{Q270jNX-Zx z?VJ9TqE=5|5;dAJsibhKJ#0zMJ*vZXklXm%v-aB?4JoDv`s%&1PqX-LIo=u zD2hw0RRmD~tI93sRbwY{EX-*6-bj^|uQ+ioOLcy~gC`MDP~{VXmhNJ4T;3Ks6Gy&p ziDspRZx>nL-k_CmwSCEe6P5fJNlzkJn=<|6R`&ixH?h$CsQ;6Aphf#}pmexjvcsBG zBnK`l5Uj;`W!o}@snm(N8N}jLpvxQW+iG9Qrme;DbUsov3FP}63ob7+F9E_;4qUwR zJsEK9jg*;Yu8HX|S9s^x4!r<=Nu?~;mCV8K+Z?`gx(zWFNyBg^-qzh1cCj*-fn`M`1N|vXcAd4Ap(H_j9q-IE zs)6otPzoL8N1I;{K%xxDftu?x&@J*thUc1A3UUfBz(q=3g4=?A6^Vy(q2(zhqRI!C zOKQqTmCGT9>h3G~B<$~o$2`jDWyy9Z3$#nVGu*+FyjDbc z|6xdRZ_mgw37Z`?CKO1$)P~n8NM4*R`ZcgPxq|h;qfXjQ_<^BMitX;yrW124ysYZ3 z^3BtayS4BmNw7rqIq&zjDj^0p_J`~rfZi^rBA~z&VZMAm=(T%hq?qv4r>de!In)?8 z12Z0xReQN}kI22pytqMjOk9XZLNICIqAsS3-L-WkKvOmSy2@m^s&ui!1Q8JjuGzs1 z8xHuv{eV=U2(HTgyVV;N;?L8B*t<-IujsxPj?DzP58vwEa)*c~6xAtJ{u zaAS~D0}5Tl=7#+(;QA!nH-Z$^mPAYlJr8K$&9!flcx80@f?6~;o=wk$V!djWb<$2_ zc`;8yIJO#laG=ES`dc6Fq{J5AryGrROd3Lg-*}J-P=u+UF}@jS(+V zVu%7oa7~af`@g#WQ(#c@A3dRmY%%(o(+taxxyE}+dl&|7!roTKGNj&W7<$k)ZU6eC zS5$@4lHk_YPqxZXl~Men)NPN8Ko%}6E1-|VT?2C3IqpWyAn7;ZwiSYRCWUCAiXM*f zfCp#uPoEc*h3Z2U+zO8WKewlc5TkRxuigxQ z8*xSA5TIM^?BD>`j$x0M-v6fbUuM#?M0wT1zj^@xjk18>TDso34+J5;NNP4UTDour z>UFSpM){|dhW{TIP(znQy?7td>yZrY$y{CWhiXRSX&`?#s`yVnk7oPzZa{eH4yPdP z;>UVGy}d(X zt6>(e#yB1kj>#i~Kh|GPO?o4rxlB4|_&^)q%J8(ldK{+l)xTirz97?#b=y_hdP75a zn_jHR&>Ct7dGp}fp}yg(SFTl<0arT}o5Y3%#9x~JTuKbe z8gjs@svkhrNfA^s;h)w=YKmm7a&&%9X8k>xb>Ff8+0ycK+ri7LcHgG!Cv5d>ke!;0 zvyFIO2tO|I66@8g8U{9nO_~RO?1|wCwC+xevu1oa=mQc&iy!Mf_i^1bnmZh;OwSqI z0C{?pc&tdFj}5x7;-eWoMFV=ltA1D95D!2OW7>WnTL_(l-SY`gI=AA~yn;eBIK?-9 zfm3orUCS;e@o?&eLaF28239uj)t|eR|JsG1Ja~|F$rk|U65JTcK3u-PHM=XkS;Y@3 zF#`kj;!n}L2Cx!Ok!4g5`dt^O|WKb?jjgwYk+k<;MDHl)1}GgF7w5Xwm!r4_4WZ|UV-qE zzu64_dZ>P!T+*x#&^M;W69B2#bKvDIM`FunsrHu73s%7h={EybR8PIpR~d;xqTl2y z84XYYYEqL??JRZAILqVql;h?#iTs<*4Co8JHHEi5LClC?-9vm;BhHBwpVRLju!xFn zXRvE18S6!O8!u<++OjIWpmn#_6grP)%2YhHNiw?uCPl*?gJskXt+XztK3sXLrlhc~ z*TaeG4j@vPnGnmIu`OP#k|sn`ObM>TYyq7&v?mWs~zA#@*PWPRDedJR|Z60uFTn&urRgA z#&bUXeBFNz=6vwG0+rKM)BVp0z&(-OA~saF`)qj=O?H2-mL>B76$b0sm6M)CSf`?n2SH=tbiGV*>Q+WdL1T6l%2k3W&#?Ie^dH3n-^Zs|3|6%5T zWcKH?0N%&{-pv0aZf5w!-=_D=4GbbN9c5SBqHG!>pv!=AzZ!heu0)~CRofH=F@%+apH6-*ARwga8xHhpR-3w_ zHszL|A#Y@sDi4KLvW#g8|IPn!*#HKp^EBsPs_*GasMBRY#D+$85?6HOq>em}ZEoh> zMPIU!hEGq@Mln7Ldiq*xf+iddKobt03~#5BJ>TowP1?bz3Z4I9sJOQPzI}C=Bx6xM zP}Banqp$Oh91h)5Z{pSAwNFpeR53`(kyC|7p;sj4@BE{!8!BBr z2QR$N?M&oePm?{6Uo zXfZGV{R5MT*uI;CzTWco_NV`lLTGfPWdr&k2qyR<_}@c%;(h}hv%G(t z`PYSuD+3bG!_qP5-`0e94Ipy$zFYl4CclslNdZiq{r&|N%6|{J09RAy8O zu~h+@eV6F%edj!sGqMY}aMbEKwswRcbbHxK|Ir8iQ`H)anwb9r#qb^G{c8bbu%X-n zlPKt;G#v%rjBJfF+^^)HO?CeBI4bODG?$xo!AuMFHBp%m5;@4*Pbac#pLT89pRGMo)#bCTR=S1qw0PX7n%{S^r*?T=*k=Z4oXZPbum8q1>Qbvy- zq(q5mD5su#I2NA_HWLvI^v;Cz_m(qIiQ&+wSe?6dF4CQjyNwDDYWc2W+SAi{8Me0F zksFK5$!y$l2ocgOa6WKPy__!p;WTjgVBy@$>3-4bw2si@%Ot0l>^37Ju2cy3CaLPx z$!76$M|SHR4fzo2=<88xsgECTIxKuTY^GDn65dEq09elL)%bR(pv#zw&_Rxl(8W^v zUOy92X6=!Cn;gs@VZ*Z-wu{pxU?q_3CE+Mp5ZLEhl>D?}Qn1cB`y|QXOvOW|un$?s zm2$9VwPv=M#GTYBP^FDLUI0%Gae8b#EdjUl!w>&7YKwuQj6tI&g}O9xPdC{3Hng=3 zyKRewR@c*tT`Qd|o}Rsz@lrMit)f{npRd-| zKpRu<29qg{RzX|nB9$Xd|6^lacnZ(ge0o-iblX+BRj-~PhUFMe>Su`uiQ1ph%u&-7 zw?QX65>>n9|9-DJT&CANq)w@ZwkZ41`!!qB*ReJ@k-`_&LaSO&j~hx;`wKnP?>dr= z3SLmvOz;&rPq=(nxl#OSGe58|qCz1Hw;@)wxmnfYWTcU~pBlGONSW{IDb_|p5TVVEE z`mjjP0p{p1Vpfl~vdH(&qtBABeTijZKeh0DXs5e@a_I8jciJ1Q3V44&Cd1{}4e78* z9sbzcsn@}^k}kWltbFNWP`!0Fvp1+2_2rzOH`~TQW3N!?tb1W`PSDLw;P4~ml>H}d+?`Ht}76i1J- zRpok7Cqe|w#^j;gPMeH}ID&;?DOiQbqL zP1u;|(VFaVADue+=)G(<_4q@lFpuwWdcG9ZScPK9K*x@CvP-TK&cfDmOJm}xOjY~f z`MTe@nr&a!&c_AgGO<>CP6{Q>mtxPU!LQDMhm`4BYrA1X(r4hQwvo$ErZ4Sw+*g$9 zI~uS5=X)jQfhaOKg5!Y7;uBdXQr2U9|5WNYYVnb+KJylv0VgO>}8#Kw{M7 zRZ*hk$pq2O^ zm!mG@<;{2}6^<%AC1p@p81H%ksJ3Bp-Hb(5ObxJ`yQ5&m zJXayyOC%(g#7;VtctO*Vo^R;rRS)UX1ei+kwsMf*T*7pt;A>qu?OZ+%lZo|Rr`3MZ zDI)_qdo6bOg-L`B321FHwXvw`U|(KTEQQvitOS(u0ZA2puKXX=W{LLRNji(iY3omF z^7-<5$ifUvhY_6)K-VA&xMs#$odJ7tqNi$;iO+MFg-P+D$+EYYP%XJRO@gJ1gCRao zbMK`-`|YQrwZ06*rw#m$E$=v=zwB^WU~cRS$V6tw$g^k9KN)Xx z=sM?#BW*iBalJE%v(Pr-vlH_fKXQ9N?DkuQ!=vhx=1`~If>Oy62XA~(;R~wUI?DX1 zt_~}v7q(gMKT2h@a1!c-$9y-ac&^pcP4OlmytL=w!;3!6nE?~UpZ?H8AdJ@^cag_c zUMVGch^5o`WiUk*wFup=9EBHew9hd&>1||)%q68i{vezwwVAbqN1Ewvo!?v|e|kPo zx~sRFg>4^IY$}+p7HGri`-H(fT%I(Jblj=$(@bsNj;-&UnyclSDdNE5V9h32Lv~U4 zae1tj!xgf)J2Ii)X89lQBAOE&&m!0Rt2vt=#p-C}Mab*16A6?(V2IIB=aS zqY{}!whWC=zq`0yBAc@-kcs--I^z)zf~!27f9g#_fW{=Wd}RF`gS+KKCwfra2sSiy zM_b)&>C z%$L^KF9wywW6H6J5aIIFR!ZIBG;sBJYgo+2U3sdCmYun);N+r@YO79d-n4TuO(vz! z@8No1GDd6{BHWgboQkRD-hnO?IA~rL!jAtC4NGj@)OK0BPNIy(4SCST|r}QHIP{>qU?C_?UVJXR2&6P;FMi!g6asmC~oodPQ8N7d10aj zZDX!e@Pa&Voudsyy%0mlJtW=4UZkMTwfxFe!wNQslVSc*TW^ZXQ=wM*5H~!uUMfkBv@v4zmsj|0wZROl+HvWym<4%!j z&yiExs?|qgf8u9>R4bu$<2ZKX9CFi~FlXUl7;RYXZ~6{@>o7ExML?$=TJIj)+S%jj?Y$i> zX`g)AL(0BkH>@-Js62#Dk(s0PEd=p7$*x_+qX%pEQX6PKS?gFy8c-y5Ny*R$_qii) zbB6F`2@DrLcXm8B6a-txHSSOtlSqK#kr?6MTIL+;55C^Cd0X>nvcE|{*Ylu*U~zQy zD|PmP3Hzb*nxwsE!{ve4VV3z6vhLl-`rMu1xTq|OI>rCw9Qf_NSyiJ};@^TJbpl`x)lH%sJ{f|@h4aL&xE;0z*v?l3ih~t< z?P6Om*=oDr>vnZTovl-+UCwS&hjK`WrS$M)&H7t5!^duN%Jb7~7^O{^!OCkE`NnnQ z#^!}4$ZXc0W(tP&{Jw>!$WMxFDNQsCacV+>>}FNF9iDrGIv)G-GOFXw#cbINR}T%M zH!*YnFlNLJ*#V=)DqVzr_05&+H4?I@0*KPw@!ABtf2u=De3o-<#)krD!EJp(ZxQe% z+oW#j&1*SR@tkpC4&&wHSd5E=J)YUHq?A>vs~XX`H5wTlL?Zobs4`xI^2(&IAKJp? z4{o%_&B9`Fw=Bo!hY9YNpkggl*w{{Wxyq}k+uj$5p=>06LR=ph(A&j}v3YxrF zCN*9z^knK&u9z6$kVsvUCW~`vt~3szeq1Wb6F2Gw<99vra56I&HM|1Zoy}FE`Y@7Cc`W;KNZpmV(!^acoE&tzLpRI{VXK+BZshj9JO5 zy~Lwc>$ox2c96u_JjSG|&0}O&wiiDIb~eeU{Nsrx0l}oczSoQ$iQM@? z>Y6->k)BV7l*cYqCY(FA)uMu&Ho}k7Ki~FR{;A7j$!}GIdK|u)qgotb!KgNSolXzo z4&i85B!zN*E$J)FxIay1%t1Lrce55Nb@|n0VMtJOjZQ9h$AGa_Ra@jV3r2oZbcqPD zyWo{$qXnyEm&HQr@s|@?;62+g$XuKn!p)+T5J?NGHDS-Do#^zmqb*&fTzq_SU%Dul z3T!bb7Hfaj)4}I8lh`3YLauWWr$Ze%Hu$nxVt0RK5npiM5^ia;PEZEp_e?@0Q$hI@UPE3H;??ljyAQSlU`9{N+7YB7yQ|68Y z7>Fc&Il8tvd=U3Q&noKJp`g-nQdNG>uzEjIrt5QMzdf%%B6F~y&r4Z|4s zIZi#~XflvfOXlW#Iyv8It~1n!?R0D^eW|GRaDf|%tTcho*@UTvzBA_<(ZcZE|(vc!^Ivqr`no!cE7 z_VddTTsMfVDd$HPvP^l(A^K)YJ83wYTdp@Qos)w~GnfXqk9X=T8>f@UR+r0#nffXU zKgA>P=P0uYI;uauu4<3*vYz5gj)h-ly(r7k|cZ!Z#++yUj>shN?0=Z<{6tc;3FHI?2S5{w? z(`d_Ydd-<@x?-=DCBZJ%TD-EqRgO)23vPwih&LzG={A1K>Ef(4d7K>W+2lQMfDE(Q zpICjMde*LPbP7LdVrVi+v$99l+LkM7cbS_7+RHOf2(X(}E&M4c#EY&2-X7<+J$twg zvH>s;{j-l7vTzH?uNV5?i!B+27a4&H;)p~~&o&x@uJH}^syTM0Rbtu=&Fwp_E0Qi- zD6flvFOC?`TY%S*@u{|`trM2Poe5_RT$ZsgLggbrHM6Saa&@m^BpK@p^U{m zI3cbmc^sx)TL)`IG#9qDCbpy@{GypKxKQKf-_aW_G06Ji=0lAO&eRiCZkf)tLc&!| zeyhTF;*?x5@T^&GA97MrX>+n&+#;4T<9t=MjN9Xs*|-H7p+iQ0C|dNz8-Tacc?xr& zK-}{U8M_VP*IDOP#y}GlvOXzhgWH@tMNNSQC=uE}=Kv>zT-*kHXqS9z%jE53M$$oT z?^ZrmF-=P_N)+N2Ra3HBrJEcl{YjC4+iXb;a7bO+lp>(^fFH9@ltY7`uxB-oim~rc zfeNf`i4R1v;NG5j&L;23;;@^$h-|G0+V%BWhDRIaqM#|GPAva8OMgD^L3JSJTX+^w zwKHg6gsmnH{cmuaa0%E=jr#t%zke>$3UI~fgzM7%$=rU*x%v!1w1>|=;{NxD4glex z&iKh6^6>Y9k;FqjyK2QEDMw}V!x@8s=2_Yb)MJjDBl2fD=viTu1b z^N}tCx041j(MVx}>}={ByB4Od6#mRK_)WHKN&vJ1MTo?cElH}YgP=pa;uG7sT&}Aa zG+4h}z>%b<@BY{Uf!+bg2&mja+E`&~Rg8-Qc@PS3V8!vwz%DqDBTsVQp^RwX#gp{O zm7jSCzdWh+y}&KYJFX^H-4M))O>wd|<}q-G6GOp#NntnXxr)RCHUEEn-W0hmyJZ}h zoub|k*L^ZlHhhay5EsdzPOW&7%S#0?r|DRoJ2zjAJ_EMD2B9bhHP?m#UbYlXSy~O& z3&j4Def9T$z;6PumRP)D&Qb%wqhGzT@MCYT=0W(!I$hh%xKB_f08^&;Qe9BoMY2C| zfh`I6a4E*B%uSNknzXq84qjAgYD^SsT`R*eXMtRh<)EOLbG8B8hMs3Mjw6Pbw(`9H zaEIbs0PhZYk0|`{-|lLN58Cj%l$1#49`xW2@uC?YRcZmsj>|!?T&i_gPB1i)gN&a`7|6%8U#P+|p^FLPbkn543&ERV^ zH`dA!ha-}%o>e9a?i6K2^b2HOU-H_!oKVITQT?SzP;$m_4Sz5JNh1vzK zb9+n7dO2S4Ii(O`fA4&&{72dOg~#Z=8%Xhb(VDpTeXi)nT=qmOi?R1GrA#R*tVO8JzgvM_C7cJi^-SPgi=zB<;qgaf0E zN|_7D@HjfdZBbl$?0Jy2*J8Zy1iqlV z=Q`PkYrcs9-%IvBQ4yMn_o`?9iKR6Bt8v6Gkk>Z>9KCo>uf&q^U6Mox#k4IzIir4Y zJU)c%%dUH*uP}ET!h$aaZY1(@U|SzoSLXD*cpBEXoIdEd!i!rn+dH0U+kvCcLGm&= z#WUM_w8(AwHUvTbW}5#1rFhEAF%{u`x-j0x<)3c_{-kiagDSCqR`e@Pf&Au{P071= z=#*R)XGrdJ9&hc*0$Bqz5A3p@$W9EgSvxg(ql;Y=Axznx;*&WUnxbjq<~(S@%@?<2 znQ6@q7-RR^!@NZf_8*pPBDvFkp%|UhjF0!zdLg^@hIIEK3eUsJ4R(Is;h}?;e#^&|!2>oC!dxGyJzgJZP(m4B{ zA7Kd)TH}I7R#9w_{>r1 zh2)j>Qh@r%#>qi-OP}2jHP?#wU6k(Bhx(d5+rwYj_y}Y>d<4=6fV;4Vcy+1-s@wO9 zCiqUAii7mdK};lMQ){0p{KU~SzH>jPraC@)V{DQszbC-F$wx7V!HkLhJZ=1NPOBT= ziumw~4G`24Kg76z^}}PB(f3OU5ugOGtM}p-t5zS#%&432eGO)lr;iL;f{-C)R!hEq zcrti!dPhE-_93>l!QjFpM>_oP2s)&A^iKTJ+(M2JTB*#>09Xu}Jv%!01O9-RX;wP) zpz2c*ZZ&tu21wA3VMv0nn~3Otw{ei-!;Xk#Q4oE4R6BCzgOSyKG@MpuP} z8ZH6AR|uk!->;~d|6JIKnAp&{ewzf?B#%WUmX*t01ZNW>Gpo7D=9Y2xl{0emtm@Y6ft!{E8E zw@t2Mp6H~BV7D4)6j6QNLF1^{5>S@aKf?@x4fbNiPoKFS3(>Y+fqE5HSQheO4(g`~ zaAbKrh($^tYYk?dJ0JU0M4`@jX^N{EdGw^QT8-009+YETz$QI^N27lp}p zqzoxBdU#4wq9O*3rTt+9Dy4tTi3M8Z=wPR414Pgrg~8+TZk}hCl4gw*fE0nX9`pZ9WB>WZPu}wL4ZH^~CnBxj zkKiKuYhc{a0-%q~s)YaW4gVU60Km7xwBh}IoZl1BTgj6W8?twFy!`iw)qSq7z{9-r zbMfL849R2PHf!OG*m4^hxhOGxK)^)JY4Dl$% z7XCS9qx0K<+eKopj?C^<(xRxb0NT;HT+hBcba;4SZE&)!w|m)Pu4lTsdwa1O&O21X z#_#3(=zMt!b&WfYM~PCd?xc_qCHu?yCZ35kdXZqTTy9B?#{Ub2Y3;=u*{RiTWyVEO z0#$E6_L(hy93}EKW%FL&scQTv@Jc`DcGJb_K<~wvs*vrNozNw7W7ChN)LJmx`Y7mB z3XzEFfC(;$o!cnU!1eCcvaC`?^&pgea@X6d1?9s~v3(V_@cC>Y^2?~%@Yd=KA^#b>A6|C?rO8|Px8_a2{>i~Eh$iq zZ>0-28jZYBLSu0~%g&zXv(8mXJ0*Q*KPv~WVyKg79T{M$TjaW+UzT_#=DpgJWBw?m zkVne%r0u#sA_*Ino{uKHeL*&p(Y=F>R?Ge(6R}>oUCraqW@w726nhe9KT!#%2FenJ z$nH25Fk&QL+i{q^*~GA8BKbT6snLJeL()m;eUg=XL7T0&&nivwDtVub)&MQNbkWBp z6Fviwi^xmmsdjkt0!wzXz~hN5u3A@QdwJZhLc1F5I99iF=!9ZlV7FU6Q5qj?Aq-4@bIFBIYgI#qIdY2k=VtQ;FgE-2J`SFt3%`@B-C8xXq?1NZt9CRv2wqctTN9-9C2%wC~mIA5=HEOt=0~<_{n+2$pHSM zJ{&fRmMm9Ehrjx=!x;{1S(H^a5k)B%nm67rT$dF+soF*-RAnuSm6DvCHR$qvnY#E0 zKYmny91z{*Yg{WpnHtVJNC_{x=ophxds{`ulwpQSRf;2Z(V2X?y0oXCFNTr-a85Al z-PTSjVLvF~6{Ka!RDtT_c zcdCkzE*~2&q1v$XO+rqpXPmy@)_f%pvq8)gE@p_@bubnDCK&Gh$a|~LSlVv<*#mzl zZgIxj`H8JB)Wc^$lAu?W?M6(DWJ#rhEjJ^FMw{=vX-P}7=*M~bZdHT(JncvsMn43fEv<$VH>NfdQh=H!?cSx}t z?g#GNC{TEYX33y?TRXQ2vX=-+7?nO;Zij`|1)sABcMrnO zirPp07fWWyzJqa6$6QpYLE_V%MAp_HOaz&it8sjtl=@AJpWWT^Dx;5+c<%V(WH4#$ zO{)H+jVq8CGOZe4`yy*NdjwjPVZWH3zGk)Dbeqpp!|Q%(PBWfy_RaLZ>|OeH6ZcW= z4$}D@c$271RMyw;s25i1DPuQP^;`yCS6`eS?_}n3S9W%Q8V{z*T%5I`e7apHc;D^S zXbpOdmuhRH*k8={j%JHgQ7(*c1$4ervWlw1?~0<0IQ}{{-u9@K*S-F-vh7s`Du&bg z%;t@K?!^NGxytAi2)Dp_9H-R<+^w%oHcbAV@W5UFQcYCivuIT7s&GYBQs>WVCO}ZZ z#dj0SPu+jv33E`9o?|GSmgxEHqt6tjxrsO9_|7j3)>t`C!=_PE6(rEvlb>QaMl_MH2Lk{P7;n*MkI?AJv_U@ zxzN!z$fF}7;+(fRPgpoavr&-2^JGMF89LPo zuW*RmTeo`PXb!itBX&AkmtI5O_3|x+=&Rb{)iPDM^}V>Q8jbV z=~`W_XQPujxvxPh?W>7)@GpTco>nUEI@OXKH>37T0WigoRU%BIWqm6jMXu%(-&nZ0 zxNSdcCOD&>6rFO}zo5B9w6imqA+lPVZuB-yYsl5XoNPut8jNZyYu!nT1(I((Y4O;k z=_QgXm!CvaB(gLl*|w$Q4D?zYDDO29>{U zPdQ^(I9sz5Z%eWu6H5D<t1k}I=jPO$f zRea8_TfdYab*xt5E6s8XcgnnZHEEf?*qb#D={^3S^kZ&Ts><>7b@kb(so&$F6N7lq zs3b?3-9WEGi&;3+!61;6+gs`GFf*-?ZjPYX_J>r@sx-)E%bjh0^VtqF%&Fq=h zQD0+sjutdg9QJxIQO=%)YF6;LuN=Gi$>QTnX#^ghZx2{vKANh`sH!si!cOgK8Gn?!?)8Jt3O_LAj;T6wK9CBJSrVX88#n4HPT`Vo`LKVS54FaW!N$C7nC{u5f8dEj5 z(x8~P(K<$coJdX0f%oF*{rVMC*`@auI-C9OS|M#$;&dj4LpBi=_5|RzbhWj)0#w6p za37-@#P;pS{ez*-vs&lTgGGCs1}obnHVz)dA;jY(t}BAfmBA*}wr~hKv7xo7?rvRV z+xp}f9C-yw#r!jlS5J@Ybjb;M(%mhCCcGtgNcz(cS#l%7U%cR|QTDwQPqbINUWK$1 z6wMX}`))^_X5pw9f5+3LsHkmMK#QAJqk?h4b4WQd>pW{i;{*j({E}Ex+wUq-r@^Eg=MU#nQ#Ux^9HMA|pMU30A%3i{2I+ofJ#a(G6 zlIGY+Ay;>5Mb&3;v8z>mJlF|OzSsjFlFiw6yqsOr?5H$cFyl5q=jpOYnwEV^_DbE; zeQU-ej2mH;lu=3gYMgyT82{zyFyd_{;sP}_$^)`<&NfkSFZ5D`~LA{q#PRV^8P5__Qo~HXabn0?H!UAye zDYb!8W>p&p?H#KNlq>pR`l1x8B4Ri?_%VIJ1@v5F^e|<{jJ-#kzPxY_XjT*Zw?J95 z!B{|qGgv1%mU6D~d>?05p(@FFv^&2$XMg7#T<{Xkf}X#$OmXVC2IRSj?FRSkC0;TZ zY@E8mD~%Vcl$XbY2O(pCvSL4heYpfb&GG~TX&L-qK38xQ7KU5I{bqw{cup7CbdL$w z(GfIA%I!+-REjvb_v|A?<(w8js?rALBy+<#W2nCn+4rFV>HxAA?2FnkPa1AUDX#}8 z$MqF_H-7n*A`VL?p=I&NUb9?Dtj1xOm~mwvFkPsf$VgA64QYXjSgly!^Oc6XYJ%%S z2+(_%h5Cm+_v$U#?c?ey*|=KmCZ&PUWwKBCJTbf6OHt3|n2DkEc;?Hnlu$8m``ybC zpyp+c**2cbKybG|{qbXU)KZnRGQvWFAJw_OpRs6K=H9k&zjg(?yZpzJdV>XU-U?_mosLz z*PgMA!A-ySHf$$(^+E5HS56N!bGvhGiiG_8k;aJD}^L38Nto= znZ4BAOi00;7_X?sS!7!pI_8cSyg`ozh|)^-;src2|1+XEsZjN@#Q zV6}CyQ@X>=F)=F0Yuo+G{oN^#(zd$6sN2ih`uA0?FRF-S;&)GdtBP>nFX*bir8!(Y z-bgax{6WFMVJo2$>vpND$s1Kyg$?G+W~WR2;3-AJ#f#9Tt^*8`CAhlJ`{ov!>!q z5-J#Q7O>3bb{~KQCEl$xo6y;;{(QZS#t!$ygJZJ;T8(u+2xpo&_m3P#cbS0qCTNC%3fiu*8^0w* z7arX#mmuZLa-taKKl8|6cIJ-ncYSA-)6jBrN|v=2yqcg3fDm4j((j=|&PZeGz4`qv zEPxV=r#r%w?W!kaq)4dU5q0$P;=43qdCMNPiwgt6v}cVo2Q-)=*DLc;y+~{kc z+=3yC{cdhL!tuP<%Wh+9O1=pr<0AtV8dPcXoys56)EGKs3qnyI;K~4MYU-Q=2@tM{Af%Py&0% zCcQ#yJ}d3UreeYRyquZpY&XhjD8jn}Z*2G|{~##TOs+NL%y{xgNLgq}j41=GQ?Nz+rl$Ik$h) z!+s~*=<#!8rZ-Se|BFJrnOQb_0Q{Z*hXXm0c-%tn&H?J-6HPbAuSH4{wx^NC>8b;J zPXjo57?}gd%--kOK^jx6y-6Uq6z7plJedj{ zM@p~^s-K8DyjCq_TEP)1nW8yd_2HbyaF*O`9!St zp)ql+0tv50L3ADm(p}q*2!2iW)#^c)M+80HJ;`l)Atnnb$)2>$JqHq)mep1oCz~4< ziV>bGlt{rPfRyt-m@iemMMz%qq+QC=78~1(OQl-N8Q?;g;KLe69iG_4t zN&U#TP^kBvh2D2w5r>6(#=3D(=w_w8@s)vX_VfTdFvoC-xy}Bwp9VkX)}B(5iCgWF zT(N=T6WOeYFN#(E7h_DH^oFg#?kd!(O%BIxgQ=yUUVP# zIL_Jd9mhz)Qt-y2xRp;?waF)%*csRNQERP>YLozw3kH`25mf@&c@I;6~#q|Kg{Oihyc~!)X*1ea+Zo=n7HCG0-m~ zmtO!c$Y2^N4fwt}^wv&pX3yFnqZ(sTa)~@!-BP-a3-l%QB<*nblAZ;itSnzH^r2zG~tlzGWovdh{){SvV$XW;se+J<%LYrnIkK z+t8J;+DY^2CLLu0Z$V}77p!a2-s9CI)mpoX#BYBQNEM42AEg{fd;&;-lB1bu0l?nC zB9O9qG5|^H*mIF4d5FzPR!q8;q~;U0Wkg*N8T%iT}TyUkCeqsq|rCCx=D zx0)aAX-SRS#{Fj#R~K$hqfH)1JpbWlsui8W>B3;nGdB2V?2h8kqgoulxF|=^da~Ms z4bI3iHwsQMDAo?aF_K4x|nldcFjzu2RLsGckR!dFN_)fA3tIdgrQ9kHzZ%8d_)vRTUP zbdvM*2Pw{m=&OE~_RsF7F?BQ5oPrAwv-eCs-;CFIdAPb^Bke@K&e&V+IoZtpjj^uz zW={*K#RUX^OTHulTTe{?3%|6`8&`IvB9kG2oSbL;pBE1XnM${cTbYH;MR`AMq_`>O zs0YOxXU%??L)0T-5jL>ReF$gNy#QGg>2Wj#pb!B~*jlg=8C`7$BO$u&bJk@!D zji^x{+qSH@%Ma&@8g9*5`z4gM1<0YaGcORw0Xw2yk~14S93I3`{wxqL-X0QA7$wQWGVsQ@ zWFu_Fq7l3(wd5=5AMRr9LPAFTiy=woLxW=vU)Z1$>+eh|(C5KIw!1O?g}^kY?)Nxc z`DmDY_;u_LrN~$rxXaC6Uh-H)YW14pL%3?G?QXyFb=E5Kj~{zZ}I8^3Vnf_Ab(Y#+Ma2?W3hbK>UC<%Mmn z#b0JahP}*24`OTtQq@pOuL^=yBIlovm(@Qh4R7Kt&Cgd1^)T&Khr+jK3Pj@o@R3V2 z4gjAi08npq!kkSTkau7@2}N|iI144V+Z|YfUU8M6elhG1Y9AsC;^%h9B0;^h!$)a2 zeJ)Aap;2{eB$XUa|Cb&B0U3Q`u9KOIPZI(-429G}02x1Sp%Iyi;^wr;UjN$cB>C3YB4njN8A`c;ab-lI0iQZmd;W%{3j8B4g`5 z<;~$3e6_>OgLKVk1?CgXB;IlWOXVW1*-qG7eWvvoP^&Lccq{jaumb7@S$-?$R_enu zLYu%c9qqqSCddFQAYz^L7O3lWq`eh$D5~+JNF#OhCTQ@Wj6T8+fQ7N1N)ZN>uL}=f&Z~i z%nRX|9q#arj~c&I1yZ=Ze3pf?vLWhD#QA_ol(FgU!urKN=w(yl=M`u_NDvWD$HH*0|Awd-qSEItLa zTaaLqot-OHE-IbG->ztT34oc%A=Oi33D&HtpeEK3>VwvO2}w!Ty42r%<*@jHTmOBP z9RGTFsC0cpTCvwX%4bj=!M@Zbd%O04%e8A0eSYPOyXo$@|6>%Iq1-)qn@q$689$Xd zgRyB;he9`2*TA<_P|hjG{d@k*mLvF9#K;nBB6iEL*nVA7`cz_SOFzDgqR=;#Gj?LcbuXMv?I{crS Q9l+;;q1pKoeWzRh17;BKg#Z8m literal 0 HcmV?d00001 diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 751d6435dd8..90d0b7af57b 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -128,6 +128,7 @@ The following table lists the catalog specific properties and their default path | `lakehouse-iceberg` | [Lakehouse Iceberg catalog properties](lakehouse-iceberg-catalog.md#catalog-properties) | `catalogs/lakehouse-iceberg/conf/lakehouse-iceberg.conf` | | `jdbc-mysql` | [MySQL catalog properties](jdbc-mysql-catalog.md#catalog-properties) | `catalogs/jdbc-mysql/conf/jdbc-mysql.conf` | | `jdbc-postgresql` | [PostgreSQL catalog properties](jdbc-postgresql-catalog.md#catalog-properties) | `catalogs/jdbc-postgresql/conf/jdbc-postgresql.conf` | +| `jdbc-doris` | [Doris catalog properties](jdbc-doris-catalog.md#catalog-properties) | `catalogs/jdbc-doris/conf/jdbc-doris.conf` | :::info The Gravitino server automatically adds the catalog properties configuration directory to classpath. diff --git a/docs/index.md b/docs/index.md index 857002186ff..efcd2dc487b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -70,6 +70,7 @@ Gravitino currently supports the following catalogs: * [**Hive catalog**](./apache-hive-catalog.md) * [**MySQL catalog**](./jdbc-mysql-catalog.md) * [**PostgreSQL catalog**](./jdbc-postgresql-catalog.md) +* [**Doris catalog**](./jdbc-doris-catalog.md) **Fileset catalogs:** @@ -105,6 +106,7 @@ Gravitino supports different catalogs to manage the metadata in different source * [Hive catalog](./apache-hive-catalog.md): a complete guide to using Gravitino to manage Apache Hive data. * [MySQL catalog](./jdbc-mysql-catalog.md): a complete guide to using Gravitino to manage MySQL data. * [PostgreSQL catalog](./jdbc-postgresql-catalog.md): a complete guide to using Gravitino to manage PostgreSQL data. +* [Doris catalog](./jdbc-doris-catalog.md): a complete guide to using Gravitino to manage Doris data. * [Hadoop catalog](./hadoop-catalog.md): a complete guide to using Gravitino to manage fileset using Hadoop Compatible File System (HCFS). diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md new file mode 100644 index 00000000000..f23e92704e3 --- /dev/null +++ b/docs/jdbc-doris-catalog.md @@ -0,0 +1,177 @@ +--- +title: "Apache Doris catalog" +slug: /jdbc-doris-catalog +keywords: +- jdbc +- Apache Doris +- metadata +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Introduction + +Gravitino provides the ability to manage [Apache Doris](https://doris.apache.org/) metadata through JDBC connection.. + +:::caution +Gravitino saves some system information in schema and table comments, like +`(From Gravitino, DO NOT EDIT: gravitino.v1.uid1078334182909406185)`, please don't change or remove this message. +::: + +## Catalog + +### Catalog capabilities + +- Gravitino catalog corresponds to the Doris instance. +- Supports metadata management of Doris (1.2.x). +- Supports table index. +- Supports [column default value](./manage-relational-metadata-using-gravitino.md#table-column-default-value). + +### Catalog properties + +You can pass to a Doris data source any property that isn't defined by Gravitino by adding +`gravitino.bypass.` prefix as a catalog property. For example, catalog property +`gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. + +You can check the relevant data source configuration in +[data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) for +more details. + +Here are the catalog properties defined in Gravitino for Doris catalog: + +| Configuration item | Description | Default value | Required | Since Version | +|----------------------|-------------------------------------------------------------------------------------|---------------|----------|---------------| +| `jdbc-url` | JDBC URL for connecting to the database. For example, `jdbc:mysql://localhost:9030` | (none) | Yes | 0.5.0 | +| `jdbc-driver` | The driver of the JDBC connection. For example, `com.mysql.jdbc.Driver`. | (none) | Yes | 0.5.0 | +| `jdbc-user` | The JDBC user name. | (none) | Yes | 0.5.0 | +| `jdbc-password` | The JDBC password. | (none) | Yes | 0.5.0 | +| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by default. | `2` | No | 0.5.0 | +| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10` by default. | `10` | No | 0.5.0 | + +Before using the Doris Catalog, you must download the corresponding JDBC driver to the `catalogs/jdbc-doris/libs` directory. +Gravitino doesn't package the JDBC driver for Doris due to licensing issues. + +### Catalog operations + +Refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations) for more details. + +## Schema + +### Schema capabilities + +- Gravitino's schema concept corresponds to the Doris database. +- Supports creating schema. +- Supports dropping schema. + +### Schema properties + +- Support schema properties, including Doris database properties and user-defined properties. + +### Schema operations + +Please refer to +[Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#schema-operations) for more details. + +## Table + +### Table capabilities + +- Gravitino's table concept corresponds to the Doris table. +- Supports index. +- Supports [column default value](./manage-relational-metadata-using-gravitino.md#table-column-default-value). + +#### Table column types + +| Gravitino Type | Doris Type | +|----------------|------------| +| `Boolean` | `Boolean` | +| `Byte` | `TinyInt` | +| `Short` | `SmallInt` | +| `Integer` | `Int` | +| `Long` | `BigInt` | +| `Float` | `Float` | +| `Double` | `Double` | +| `Decimal` | `Decimal` | +| `Date` | `Date` | +| `Timestamp` | `Datetime` | +| `VarChar` | `VarChar` | +| `FixedChar` | `Char` | +| `String` | `String` | + +Doris doesn't support Gravitino `Fixed` `Struct` `List` `Map` `Timestamp_tz` `IntervalDay` `IntervalYear` `Union` `UUID` type. +The data types other than those listed above are mapped to Gravitino's +**[Unparsed Type](./manage-relational-metadata-using-gravitino.md#unparsed-type)** that +represents an unresolvable data type since 0.5.0. + +#### Table column auto-increment + +Unsupported for now. + +### Table properties + +- Doris supports table properties, and you can set them in the table properties. +- Only supports Doris table properties and doesn't support user-defined properties. + +### Table indexes + +- Supports PRIMARY_KEY + + Please be aware that the index can only apply to a single column. + + + + + ```json + { + "indexes": [ + { + "indexType": "primary_key", + "name": "PRIMARY", + "fieldNames": [["id"]] + } + ] + } + ``` + + + + + ```java + Index[] indexes = new Index[] { + Indexes.of(IndexType.PRIMARY_KEY, "PRIMARY", new String[][]{{"id"}}) + } + ``` + + + + +### Table operations + +Please refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#table-operations) for more details. + +#### Alter table operations + +Gravitino supports these table alteration operations: + +- `RenameTable` +- `UpdateComment` +- `AddColumn` +- `DeleteColumn` +- `UpdateColumnType` +- `UpdateColumnPosition` +- `UpdateColumnComment` +- `SetProperty` + +Please be aware that: + + - Not all table alteration operations can be processed in batches. + - Schema changes, such as adding/modifying/dropping columns can be processed in batches. + - Supports modifying multiple column comments at the same time. + - Doesn't support modifying the column type and column comment at the same time. + - The schema alteration in Doris is asynchronous. You might get an outdated schema if you + execute a schema query immediately after the alteration. It is recommended to pause briefly + after the schema alteration. Gravitino will add the schema alteration status into + the schema information in the upcoming version to solve this problem. diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index 6ec1bf615ed..aba936b28c6 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -33,7 +33,7 @@ Gravitino saves some system information in schema and table comment, like `(From ### Catalog properties -You can pass to a MySQL data source any property that isn't defined by Gravitino by adding `gravitino.bypass` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. +You can pass to a MySQL data source any property that isn't defined by Gravitino by adding `gravitino.bypass.` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. Check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 6b85373db51..5f673ecac68 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -32,7 +32,7 @@ Gravitino saves some system information in schema and table comment, like `(From ### Catalog properties -Any property that isn't defined by Gravitino can pass to MySQL data source by adding `gravitino.bypass` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. +Any property that isn't defined by Gravitino can pass to MySQL data source by adding `gravitino.bypass.` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. You can check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) If you use JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-database`, `jdbc-user` and `jdbc-password` to catalog properties. diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 1c8923b82f2..a5d1b67b4b2 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -39,7 +39,7 @@ Builds with Hadoop 2.10.x, there may be compatibility issues when accessing Hado | `uri` | The URI configuration of the Iceberg catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db`. | (none) | Yes | 0.2.0 | | `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-hive/` for local fs or `hdfs://namespace/hdfs/path` for HDFS. | (none) | Yes | 0.2.0 | -Any properties not defined by Gravitino with `gravitino.bypass` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. +Any properties not defined by Gravitino with `gravitino.bypass.` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. #### JDBC catalog diff --git a/docs/manage-relational-metadata-using-gravitino.md b/docs/manage-relational-metadata-using-gravitino.md index 2e44f98ffc2..028ee5cf334 100644 --- a/docs/manage-relational-metadata-using-gravitino.md +++ b/docs/manage-relational-metadata-using-gravitino.md @@ -21,6 +21,7 @@ For more details, please refer to the related doc. - [**Apache Hive**](./apache-hive-catalog.md) - [**MySQL**](./jdbc-mysql-catalog.md) - [**PostgreSQL**](./jdbc-postgresql-catalog.md) +- [**Doris**](./jdbc-doris-catalog.md) - [**Apache Iceberg**](./lakehouse-iceberg-catalog.md) Assuming: @@ -93,6 +94,7 @@ Currently, Gravitino supports the following catalog providers: | `lakehouse-iceberg` | [Iceberg catalog property](./lakehouse-iceberg-catalog.md#catalog-properties) | | `jdbc-mysql` | [MySQL catalog property](./jdbc-mysql-catalog.md#catalog-properties) | | `jdbc-postgresql` | [PostgreSQL catalog property](./jdbc-postgresql-catalog.md#catalog-properties) | +| `jdbc-doris` | [Doris catalog property](./jdbc-doris-catalog.md#catalog-properties) | ### Load a catalog @@ -305,6 +307,7 @@ Currently, Gravitino supports the following schema property: | `lakehouse-iceberg` | [Iceberg scheme property](./lakehouse-iceberg-catalog.md#schema-properties) | | `jdbc-mysql` | [MySQL schema property](./jdbc-mysql-catalog.md#schema-properties) | | `jdbc-postgresql` | [PostgreSQL schema property](./jdbc-postgresql-catalog.md#schema-properties) | +| `jdbc-doris` | [Doris schema property](./jdbc-doris-catalog.md#schema-properties) | ### Load a schema @@ -727,6 +730,7 @@ The following is the table property that Gravitino supports: | `lakehouse-iceberg` | [Iceberg table property](./lakehouse-iceberg-catalog.md#table-properties) | [Iceberg type mapping](./lakehouse-iceberg-catalog.md#table-column-types) | | `jdbc-mysql` | [MySQL table property](./jdbc-mysql-catalog.md#table-properties) | [MySQL type mapping](./jdbc-mysql-catalog.md#table-column-types) | | `jdbc-postgresql` | [PostgreSQL table property](./jdbc-postgresql-catalog.md#table-properties) | [PostgreSQL type mapping](./jdbc-postgresql-catalog.md#table-column-types) | +| `doris` | [Doris table property](./jdbc-doris-catalog.md#table-properties) | [Doris type mapping](./jdbc-doris-catalog.md#table-column-types) | #### Table partitioning, bucketing, sort ordering and indexes diff --git a/docs/webui.md b/docs/webui.md index 0800f7c7125..a758aa4ea0c 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -222,6 +222,19 @@ Creating a catalog requires these fields: |jdbc-password|The JDBC password | |jdbc-database|e.g. `pg_database` | + + + Follow the [JDBC Doris catalog](jdbc-doris-catalog) document + + + + |Key |Description | + |-------------|-----------------------------------------------------| + |jdbc-driver |e.g. `com.mysql.jdbc.Driver` | + |jdbc-url |e.g. `jdbc:mysql://localhost:9030` | + |jdbc-user |The JDBC user name | + |jdbc-password|The JDBC password | + From d3efbaf996c4f09371d8aab45b54c6d859d22e03 Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Mon, 22 Apr 2024 16:55:31 +0800 Subject: [PATCH 092/106] [#3083] improve(web): update regular expression of metalake and catalog name (#3087) ### What changes were proposed in this pull request? Support for the use of dashes in naming metalake and catalog by web UI. image ### Why are the changes needed? The dashes is a relatively common usage in naming. Fix: #3083 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Manually Test --- web/src/app/metalakes/CreateMetalakeDialog.js | 2 +- .../app/metalakes/metalake/rightContent/CreateCatalogDialog.js | 2 +- web/src/lib/utils/regex.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/app/metalakes/CreateMetalakeDialog.js b/web/src/app/metalakes/CreateMetalakeDialog.js index 6fcb417def6..788c92c8da4 100644 --- a/web/src/app/metalakes/CreateMetalakeDialog.js +++ b/web/src/app/metalakes/CreateMetalakeDialog.js @@ -45,7 +45,7 @@ const schema = yup.object().shape({ .required() .matches( nameRegex, - 'This field must start with a letter or underscore, and can only contain letters, numbers, and underscores' + 'This field must start with a letter or underscore, and can only contain letters, numbers, dashes, and underscores' ) }) diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index cfab9e650db..a6ff296de50 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -54,7 +54,7 @@ const schema = yup.object().shape({ .required() .matches( nameRegex, - 'This field must start with a letter or underscore, and can only contain letters, numbers, and underscores' + 'This field must start with a letter or underscore, and can only contain letters, numbers, dashes, and underscores' ), type: yup.mixed().oneOf(['relational', 'fileset', 'messaging']).required(), provider: yup.string().when('type', (type, schema) => { diff --git a/web/src/lib/utils/regex.js b/web/src/lib/utils/regex.js index 6a19ccea7af..7f8c039a076 100644 --- a/web/src/lib/utils/regex.js +++ b/web/src/lib/utils/regex.js @@ -3,6 +3,6 @@ * This software is licensed under the Apache License version 2. */ -export const nameRegex = /^[a-zA-Z_][a-zA-Z0-9_]*$/ +export const nameRegex = /^[a-zA-Z_-][a-zA-Z0-9_-]*$/ export const keyRegex = /^[a-zA-Z_][a-zA-Z0-9-_.]*$/ From 720ec29f3dafdb0d4ca79b4a40272d7d587321f4 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 22 Apr 2024 17:31:54 +0800 Subject: [PATCH 093/106] [#3089] fix:(jdbc-doris): Make class loaders of Doris Catalog can be GCed normally. (#3090) ### What changes were proposed in this pull request? Introduce a common class `MySQLProtocolCompatibleCatalogOperations` so that any JDBC catalog that is compatible with MySQL protocol can use it. ### Why are the changes needed? It's a bug that need to fix Fix: #3089 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Test locally. --- ...MySQLProtocolCompatibleCatalogOperations.java} | 12 ++++++------ .../gravitino/catalog/doris/DorisCatalog.java | 15 +++++++++++++++ .../gravitino/catalog/mysql/MysqlCatalog.java | 3 ++- 3 files changed, 23 insertions(+), 7 deletions(-) rename catalogs/{catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java => catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java} (82%) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java similarity index 82% rename from catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java rename to catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java index 403baff9469..c56a9808831 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MySQLCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java @@ -2,10 +2,9 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.catalog.mysql; -import com.datastrato.gravitino.catalog.jdbc.JdbcCatalogOperations; -import com.datastrato.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata; +package com.datastrato.gravitino.catalog.jdbc; + import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; @@ -16,10 +15,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MySQLCatalogOperations extends JdbcCatalogOperations { - private static final Logger LOG = LoggerFactory.getLogger(MySQLCatalogOperations.class); +public class MySQLProtocolCompatibleCatalogOperations extends JdbcCatalogOperations { + private static final Logger LOG = + LoggerFactory.getLogger(MySQLProtocolCompatibleCatalogOperations.class); - public MySQLCatalogOperations( + public MySQLProtocolCompatibleCatalogOperations( JdbcExceptionConverter exceptionConverter, JdbcTypeConverter jdbcTypeConverter, JdbcDatabaseOperations databaseOperation, diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java index 69fa0b39cb2..d24742857ad 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java @@ -10,11 +10,14 @@ import com.datastrato.gravitino.catalog.doris.operation.DorisDatabaseOperations; import com.datastrato.gravitino.catalog.doris.operation.DorisTableOperations; import com.datastrato.gravitino.catalog.jdbc.JdbcCatalog; +import com.datastrato.gravitino.catalog.jdbc.MySQLProtocolCompatibleCatalogOperations; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import com.datastrato.gravitino.connector.CatalogOperations; +import java.util.Map; /** Implementation of a Doris catalog in Gravitino. */ public class DorisCatalog extends JdbcCatalog { @@ -24,6 +27,18 @@ public String shortName() { return "jdbc-doris"; } + @Override + protected CatalogOperations newOps(Map config) { + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + return new MySQLProtocolCompatibleCatalogOperations( + createExceptionConverter(), + jdbcTypeConverter, + createJdbcDatabaseOperations(), + createJdbcTableOperations(), + createJdbcTablePropertiesMetadata(), + createJdbcColumnDefaultValueConverter()); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new DorisExceptionConverter(); diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java index 6824d7b46c0..01284ad4a3a 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcCatalog; import com.datastrato.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata; +import com.datastrato.gravitino.catalog.jdbc.MySQLProtocolCompatibleCatalogOperations; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; @@ -30,7 +31,7 @@ public String shortName() { @Override protected CatalogOperations newOps(Map config) { JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); - return new MySQLCatalogOperations( + return new MySQLProtocolCompatibleCatalogOperations( createExceptionConverter(), jdbcTypeConverter, createJdbcDatabaseOperations(), From dfdd5530f61630a628a06cefee346ba1266597e6 Mon Sep 17 00:00:00 2001 From: lwyang <1670906161@qq.com> Date: Mon, 22 Apr 2024 19:03:38 +0800 Subject: [PATCH 094/106] [#2851] feat(core): Add relational backend for Group Entity (#3031) ### What changes were proposed in this pull request? add relational backend for Group Entity [note]: wait for #2850 merged first ### Why are the changes needed? Fix: #2851 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? ut --------- Co-authored-by: yangliwei --- .../storage/relational/JDBCBackend.java | 10 + .../relational/mapper/GroupMetaMapper.java | 119 ++++ .../relational/mapper/GroupRoleRelMapper.java | 105 +++ .../relational/mapper/RoleMetaMapper.java | 20 +- .../storage/relational/po/GroupPO.java | 143 ++++ .../storage/relational/po/GroupRoleRelPO.java | 131 ++++ .../relational/service/GroupMetaService.java | 226 +++++++ .../service/MetalakeMetaService.java | 22 +- .../relational/service/RoleMetaService.java | 5 + .../relational/service/UserMetaService.java | 71 +- .../session/SqlSessionFactoryHelper.java | 4 + .../relational/utils/POConverters.java | 145 ++++- .../gravitino/storage/TestEntityStorage.java | 85 ++- .../storage/memory/TestMemoryEntityStore.java | 15 + .../storage/relational/TestJDBCBackend.java | 39 +- .../service/TestGroupMetaService.java | 614 ++++++++++++++++++ .../service/TestUserMetaService.java | 24 +- core/src/test/resources/h2/schema-h2.sql | 27 +- scripts/mysql/schema-0.5.0-mysql.sql | 27 +- 19 files changed, 1750 insertions(+), 82 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupRoleRelPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index bcd56709e66..8329ae7e6d3 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -20,6 +20,7 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.TableEntity; @@ -28,6 +29,7 @@ import com.datastrato.gravitino.storage.relational.converters.SQLExceptionConverterFactory; import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; +import com.datastrato.gravitino.storage.relational.service.GroupMetaService; import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.datastrato.gravitino.storage.relational.service.RoleMetaService; import com.datastrato.gravitino.storage.relational.service.SchemaMetaService; @@ -109,6 +111,8 @@ public void insert(E e, boolean overwritten) UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); } else if (e instanceof RoleEntity) { RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); + } else if (e instanceof GroupEntity) { + GroupMetaService.getInstance().insertGroup((GroupEntity) e, overwritten); } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); @@ -134,6 +138,8 @@ public E update( return (E) TopicMetaService.getInstance().updateTopic(ident, updater); case USER: return (E) UserMetaService.getInstance().updateUser(ident, updater); + case GROUP: + return (E) GroupMetaService.getInstance().updateGroup(ident, updater); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for update operation", entityType); @@ -158,6 +164,8 @@ public E get( return (E) TopicMetaService.getInstance().getTopicByIdentifier(ident); case USER: return (E) UserMetaService.getInstance().getUserByIdentifier(ident); + case GROUP: + return (E) GroupMetaService.getInstance().getGroupByIdentifier(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for get operation", entityType); @@ -181,6 +189,8 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea return TopicMetaService.getInstance().deleteTopic(ident); case USER: return UserMetaService.getInstance().deleteUser(ident); + case GROUP: + return GroupMetaService.getInstance().deleteGroup(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for delete operation", entityType); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java new file mode 100644 index 00000000000..b488d1dbc2b --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.GroupPO; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface GroupRoleRelMapper { + String RELATION_TABLE_NAME = "group_role_rel"; + String GROUP_TABLE_NAME = "group_meta"; + + @Insert({ + "" + }) + void batchInsertGroupRoleRel(@Param("groupRoleRels") List groupRoleRelPOS); + + @Insert({ + "" + }) + void batchInsertGroupRoleRelOnDuplicateKeyUpdate( + @Param("groupRoleRels") List groupRoleRelPOS); + + @Update( + "UPDATE " + + RELATION_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE group_id = #{groupId} AND deleted_at = 0") + void softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId); + + @Update({ + "" + }) + void softDeleteGroupRoleRelByGroupAndRoles( + @Param("groupId") Long groupId, @Param("roleIds") List roleIds); + + @Update( + "UPDATE " + + RELATION_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE group_id IN (SELECT group_id FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0") + void softDeleteGroupRoleRelByMetalakeId(Long metalakeId); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java index f9732d040e1..91a8b8d7bdd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java @@ -21,7 +21,8 @@ */ public interface RoleMetaMapper { String ROLE_TABLE_NAME = "role_meta"; - String RELATION_TABLE_NAME = "user_role_rel"; + String USER_RELATION_TABLE_NAME = "user_role_rel"; + String GROUP_RELATION_TABLE_NAME = "group_role_rel"; @Select( "SELECT role_id as roleId FROM " @@ -40,12 +41,27 @@ Long selectRoleIdByMetalakeIdAndName( + " FROM " + ROLE_TABLE_NAME + " ro JOIN " - + RELATION_TABLE_NAME + + USER_RELATION_TABLE_NAME + " re ON ro.role_id = re.role_id" + " WHERE re.user_id = #{userId}" + " AND ro.deleted_at = 0 AND re.deleted_at = 0") List listRolesByUserId(@Param("userId") Long userId); + @Select( + "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.securable_object as securableObject, ro.privileges as privileges," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + GROUP_RELATION_TABLE_NAME + + " ge ON ro.role_id = ge.role_id" + + " WHERE ge.group_id = #{groupId}" + + " AND ro.deleted_at = 0 AND ge.deleted_at = 0") + List listRolesByGroupId(Long groupId); + @Insert( "INSERT INTO " + ROLE_TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupPO.java new file mode 100644 index 00000000000..33e0ab27c50 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupPO.java @@ -0,0 +1,143 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class GroupPO { + private Long groupId; + private String groupName; + private Long metalakeId; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getGroupId() { + return groupId; + } + + public String getGroupName() { + return groupName; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public String getAuditInfo() { + return auditInfo; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof GroupPO)) { + return false; + } + GroupPO tablePO = (GroupPO) o; + return Objects.equal(getGroupId(), tablePO.getGroupId()) + && Objects.equal(getGroupName(), tablePO.getGroupName()) + && Objects.equal(getMetalakeId(), tablePO.getMetalakeId()) + && Objects.equal(getAuditInfo(), tablePO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), tablePO.getCurrentVersion()) + && Objects.equal(getLastVersion(), tablePO.getLastVersion()) + && Objects.equal(getDeletedAt(), tablePO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getGroupId(), + getGroupName(), + getMetalakeId(), + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final GroupPO groupPO; + + private Builder() { + groupPO = new GroupPO(); + } + + public Builder withGroupId(Long groupId) { + groupPO.groupId = groupId; + return this; + } + + public Builder withGroupName(String groupName) { + groupPO.groupName = groupName; + return this; + } + + public Builder withMetalakeId(Long metalakeId) { + groupPO.metalakeId = metalakeId; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + groupPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + groupPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + groupPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + groupPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(groupPO.groupId != null, "Group id is required"); + Preconditions.checkArgument(groupPO.groupName != null, "Group name is required"); + Preconditions.checkArgument(groupPO.metalakeId != null, "Metalake id is required"); + Preconditions.checkArgument(groupPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument(groupPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(groupPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(groupPO.deletedAt != null, "Deleted at is required"); + } + + public GroupPO build() { + validate(); + return groupPO; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupRoleRelPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupRoleRelPO.java new file mode 100644 index 00000000000..82ebac8a835 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/GroupRoleRelPO.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class GroupRoleRelPO { + private Long groupId; + private Long roleId; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getGroupId() { + return groupId; + } + + public Long getRoleId() { + return roleId; + } + + public String getAuditInfo() { + return auditInfo; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof GroupRoleRelPO)) { + return false; + } + GroupRoleRelPO groupRoleRelPO = (GroupRoleRelPO) o; + return Objects.equal(getGroupId(), groupRoleRelPO.getGroupId()) + && Objects.equal(getRoleId(), groupRoleRelPO.getRoleId()) + && Objects.equal(getAuditInfo(), groupRoleRelPO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), groupRoleRelPO.getCurrentVersion()) + && Objects.equal(getLastVersion(), groupRoleRelPO.getLastVersion()) + && Objects.equal(getDeletedAt(), groupRoleRelPO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getGroupId(), + getRoleId(), + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final GroupRoleRelPO groupRoleRelPO; + + private Builder() { + groupRoleRelPO = new GroupRoleRelPO(); + } + + public Builder withGroupId(Long groupId) { + groupRoleRelPO.groupId = groupId; + return this; + } + + public Builder withRoleId(Long roleId) { + groupRoleRelPO.roleId = roleId; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + groupRoleRelPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + groupRoleRelPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + groupRoleRelPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + groupRoleRelPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(groupRoleRelPO.groupId != null, "Group id is required"); + Preconditions.checkArgument(groupRoleRelPO.roleId != null, "Role id is required"); + Preconditions.checkArgument(groupRoleRelPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument( + groupRoleRelPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(groupRoleRelPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(groupRoleRelPO.deletedAt != null, "Deleted at is required"); + } + + public GroupRoleRelPO build() { + validate(); + return groupRoleRelPO; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java new file mode 100644 index 00000000000..d02001e9e79 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java @@ -0,0 +1,226 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.storage.relational.mapper.GroupMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.GroupRoleRelMapper; +import com.datastrato.gravitino.storage.relational.po.GroupPO; +import com.datastrato.gravitino.storage.relational.po.GroupRoleRelPO; +import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +/** The service class for group metadata. It provides the basic database operations for group. */ +public class GroupMetaService { + private static final GroupMetaService INSTANCE = new GroupMetaService(); + + public static GroupMetaService getInstance() { + return INSTANCE; + } + + private GroupMetaService() {} + + private GroupPO getGroupPOByMetalakeIdAndName(Long metalakeId, String groupName) { + GroupPO GroupPO = + SessionUtils.getWithoutCommit( + GroupMetaMapper.class, + mapper -> mapper.selectGroupMetaByMetalakeIdAndName(metalakeId, groupName)); + + if (GroupPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.GROUP.name().toLowerCase(), + groupName); + } + return GroupPO; + } + + private Long getGroupIdByMetalakeIdAndName(Long metalakeId, String groupName) { + Long groupId = + SessionUtils.getWithoutCommit( + GroupMetaMapper.class, + mapper -> mapper.selectGroupIdBySchemaIdAndName(metalakeId, groupName)); + + if (groupId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.GROUP.name().toLowerCase(), + groupName); + } + return groupId; + } + + public GroupEntity getGroupByIdentifier(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + GroupPO groupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); + List rolePOs = RoleMetaService.getInstance().listRolesByGroupId(groupPO.getGroupId()); + + return POConverters.fromGroupPO(groupPO, rolePOs, identifier.namespace()); + } + + public void insertGroup(GroupEntity GroupEntity, boolean overwritten) { + try { + Preconditions.checkArgument( + GroupEntity.namespace() != null + && !GroupEntity.namespace().isEmpty() + && GroupEntity.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(GroupEntity.namespace().level(0)); + GroupPO.Builder builder = GroupPO.builder().withMetalakeId(metalakeId); + GroupPO GroupPO = POConverters.initializeGroupPOWithVersion(GroupEntity, builder); + + List roleIds = Optional.ofNullable(GroupEntity.roleIds()).orElse(Lists.newArrayList()); + List groupRoleRelPOS = + POConverters.initializeGroupRoleRelsPOWithVersion(GroupEntity, roleIds); + + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + GroupMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertGroupMetaOnDuplicateKeyUpdate(GroupPO); + } else { + mapper.insertGroupMeta(GroupPO); + } + }), + () -> { + if (groupRoleRelPOS.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> { + if (overwritten) { + mapper.batchInsertGroupRoleRelOnDuplicateKeyUpdate(groupRoleRelPOS); + } else { + mapper.batchInsertGroupRoleRel(groupRoleRelPOS); + } + }); + }); + } catch (RuntimeException re) { + ExceptionUtils.checkSQLException( + re, Entity.EntityType.GROUP, GroupEntity.nameIdentifier().toString()); + throw re; + } + } + + public boolean deleteGroup(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + Long groupId = getGroupIdByMetalakeIdAndName(metalakeId, identifier.name()); + + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + GroupMetaMapper.class, mapper -> mapper.softDeleteGroupMetaByGroupId(groupId)), + () -> + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> mapper.softDeleteGroupRoleRelByGroupId(groupId))); + return true; + } + + public GroupEntity updateGroup( + NameIdentifier identifier, Function updater) { + Preconditions.checkArgument( + identifier != null + && !identifier.namespace().isEmpty() + && identifier.namespace().levels().length == 3, + "The identifier should not be null and should have three level."); + + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); + GroupPO oldGroupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); + List rolePOs = + RoleMetaService.getInstance().listRolesByGroupId(oldGroupPO.getGroupId()); + GroupEntity oldGroupEntity = + POConverters.fromGroupPO(oldGroupPO, rolePOs, identifier.namespace()); + + GroupEntity newEntity = (GroupEntity) updater.apply((E) oldGroupEntity); + Preconditions.checkArgument( + Objects.equals(oldGroupEntity.id(), newEntity.id()), + "The updated group entity id: %s should be same with the group entity id before: %s", + newEntity.id(), + oldGroupEntity.id()); + + Set oldRoleIds = + oldGroupEntity.roleIds() == null + ? Sets.newHashSet() + : Sets.newHashSet(oldGroupEntity.roleIds()); + Set newRoleIds = + newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + + Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); + Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); + + if (insertRoleIds.isEmpty() && deleteRoleIds.isEmpty()) { + return newEntity; + } + try { + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + GroupMetaMapper.class, + mapper -> + mapper.updateGroupMeta( + POConverters.updateGroupPOWithVersion(oldGroupPO, newEntity), + oldGroupPO)), + () -> { + if (insertRoleIds.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> + mapper.batchInsertGroupRoleRel( + POConverters.initializeGroupRoleRelsPOWithVersion( + newEntity, Lists.newArrayList(insertRoleIds)))); + }, + () -> { + if (deleteRoleIds.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> + mapper.softDeleteGroupRoleRelByGroupAndRoles( + newEntity.id(), Lists.newArrayList(deleteRoleIds))); + }); + } catch (RuntimeException re) { + ExceptionUtils.checkSQLException( + re, Entity.EntityType.GROUP, newEntity.nameIdentifier().toString()); + throw re; + } + return newEntity; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 066f4178332..e32ad5a1130 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -16,6 +16,8 @@ import com.datastrato.gravitino.storage.relational.mapper.CatalogMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.FilesetMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.FilesetVersionMapper; +import com.datastrato.gravitino.storage.relational.mapper.GroupMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.GroupRoleRelMapper; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TableMetaMapper; @@ -180,7 +182,15 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { () -> SessionUtils.doWithoutCommit( UserMetaMapper.class, - mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId))); + mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> mapper.softDeleteGroupRoleRelByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + GroupMetaMapper.class, + mapper -> mapper.softDeleteGroupMetasByMetalakeId(metalakeId))); } else { List catalogEntities = CatalogMetaService.getInstance() @@ -201,7 +211,15 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { () -> SessionUtils.doWithoutCommit( UserMetaMapper.class, - mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId))); + mapper -> mapper.softDeleteUserMetasByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + GroupRoleRelMapper.class, + mapper -> mapper.softDeleteGroupRoleRelByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + GroupMetaMapper.class, + mapper -> mapper.softDeleteGroupMetasByMetalakeId(metalakeId))); } } return true; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java index c0656628c9f..5ec05d3f45a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java @@ -45,6 +45,11 @@ public List listRolesByUserId(Long userId) { RoleMetaMapper.class, mapper -> mapper.listRolesByUserId(userId)); } + public List listRolesByGroupId(Long groupId) { + return SessionUtils.getWithoutCommit( + RoleMetaMapper.class, mapper -> mapper.listRolesByGroupId(groupId)); + } + public void insertRole(RoleEntity roleEntity, boolean overwritten) { try { Preconditions.checkArgument( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java index cc1de84a19a..5ebe1a78171 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java @@ -25,7 +25,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; /** The service class for user metadata. It provides the basic database operations for user. */ public class UserMetaService { @@ -94,16 +93,7 @@ public void insertUser(UserEntity userEntity, boolean overwritten) { UserPO.Builder builder = UserPO.builder().withMetalakeId(metalakeId); UserPO userPO = POConverters.initializeUserPOWithVersion(userEntity, builder); - List roleIds = userEntity.roleIds(); - if (roleIds == null) { - roleIds = - Optional.ofNullable(userEntity.roleNames()).orElse(Lists.newArrayList()).stream() - .map( - roleName -> - RoleMetaService.getInstance() - .getRoleIdByMetalakeIdAndName(metalakeId, roleName)) - .collect(Collectors.toList()); - } + List roleIds = Optional.ofNullable(userEntity.roleIds()).orElse(Lists.newArrayList()); List userRoleRelPOs = POConverters.initializeUserRoleRelsPOWithVersion(userEntity, roleIds); @@ -193,34 +183,41 @@ public UserEntity updateUser( if (insertRoleIds.isEmpty() && deleteRoleIds.isEmpty()) { return newEntity; } - SessionUtils.doMultipleWithCommit( - () -> + + try { + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + UserMetaMapper.class, + mapper -> + mapper.updateUserMeta( + POConverters.updateUserPOWithVersion(oldUserPO, newEntity), oldUserPO)), + () -> { + if (insertRoleIds.isEmpty()) { + return; + } SessionUtils.doWithoutCommit( - UserMetaMapper.class, + UserRoleRelMapper.class, mapper -> - mapper.updateUserMeta( - POConverters.updateUserPOWithVersion(oldUserPO, newEntity), oldUserPO)), - () -> { - if (insertRoleIds.isEmpty()) { - return; - } - SessionUtils.doWithoutCommit( - UserRoleRelMapper.class, - mapper -> - mapper.batchInsertUserRoleRel( - POConverters.initializeUserRoleRelsPOWithVersion( - newEntity, Lists.newArrayList(insertRoleIds)))); - }, - () -> { - if (deleteRoleIds.isEmpty()) { - return; - } - SessionUtils.doWithoutCommit( - UserRoleRelMapper.class, - mapper -> - mapper.softDeleteUserRoleRelByUserAndRoles( - newEntity.id(), Lists.newArrayList(deleteRoleIds))); - }); + mapper.batchInsertUserRoleRel( + POConverters.initializeUserRoleRelsPOWithVersion( + newEntity, Lists.newArrayList(insertRoleIds)))); + }, + () -> { + if (deleteRoleIds.isEmpty()) { + return; + } + SessionUtils.doWithoutCommit( + UserRoleRelMapper.class, + mapper -> + mapper.softDeleteUserRoleRelByUserAndRoles( + newEntity.id(), Lists.newArrayList(deleteRoleIds))); + }); + } catch (RuntimeException re) { + ExceptionUtils.checkSQLException( + re, Entity.EntityType.USER, newEntity.nameIdentifier().toString()); + throw re; + } return newEntity; } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 485435e1e8e..64526ebf353 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -10,6 +10,8 @@ import com.datastrato.gravitino.storage.relational.mapper.CatalogMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.FilesetMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.FilesetVersionMapper; +import com.datastrato.gravitino.storage.relational.mapper.GroupMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.GroupRoleRelMapper; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; @@ -92,6 +94,8 @@ public void init(Config config) { configuration.addMapper(UserMetaMapper.class); configuration.addMapper(RoleMetaMapper.class); configuration.addMapper(UserRoleRelMapper.class); + configuration.addMapper(GroupMetaMapper.class); + configuration.addMapper(GroupRoleRelMapper.class); // Create the SqlSessionFactory object, it is a singleton object if (sqlSessionFactory == null) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 591131071bb..5ab6245d58a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -13,6 +13,7 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; @@ -22,6 +23,8 @@ import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.FilesetPO; import com.datastrato.gravitino.storage.relational.po.FilesetVersionPO; +import com.datastrato.gravitino.storage.relational.po.GroupPO; +import com.datastrato.gravitino.storage.relational.po.GroupRoleRelPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.datastrato.gravitino.storage.relational.po.RolePO; import com.datastrato.gravitino.storage.relational.po.SchemaPO; @@ -642,13 +645,14 @@ public static UserPO initializeUserPOWithVersion(UserEntity userEntity, UserPO.B /** * Update UserPO version * - * @param oldUserPO the old UserEntity object + * @param oldUserPO the old UserPO object * @param newUser the new TableEntity object * @return UserPO object with updated version */ public static UserPO updateUserPOWithVersion(UserPO oldUserPO, UserEntity newUser) { Long lastVersion = oldUserPO.getLastVersion(); - // Will set the version to the last version + 1 when having some fields need be multiple version + // TODO: set the version to the last version + 1 when having some fields need be multiple + // version Long nextVersion = lastVersion; try { return UserPO.builder() @@ -678,15 +682,55 @@ public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespa List roleNames = rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); - return UserEntity.builder() - .withId(userPO.getUserId()) - .withName(userPO.getUserName()) - .withRoleNames(roleNames.isEmpty() ? null : roleNames) - .withRoleIds(roleIds.isEmpty() ? null : roleIds) - .withNamespace(namespace) - .withAuditInfo( - JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)) - .build(); + + UserEntity.Builder builder = + UserEntity.builder() + .withId(userPO.getUserId()) + .withName(userPO.getUserName()) + .withNamespace(namespace) + .withAuditInfo( + JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)); + if (!roleNames.isEmpty()) { + builder.withRoleNames(roleNames); + } + if (!roleIds.isEmpty()) { + builder.withRoleIds(roleIds); + } + return builder.build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to deserialize json object:", e); + } + } + + /** + * Convert {@link GroupPO} to {@link GroupEntity} + * + * @param groupPO GroupPO object to be converted + * @param rolePOs list of rolePO + * @param namespace Namespace object to be associated with the group + * @return GroupEntity object from GroupPO object + */ + public static GroupEntity fromGroupPO( + GroupPO groupPO, List rolePOs, Namespace namespace) { + try { + List roleNames = + rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); + List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); + + GroupEntity.Builder builder = + GroupEntity.builder() + .withId(groupPO.getGroupId()) + .withName(groupPO.getGroupName()) + .withNamespace(namespace) + .withAuditInfo( + JsonUtils.anyFieldMapper().readValue(groupPO.getAuditInfo(), AuditInfo.class)); + if (!roleNames.isEmpty()) { + builder.withRoleNames(roleNames); + } + if (!roleIds.isEmpty()) { + builder.withRoleIds(roleIds); + } + return builder.build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); } @@ -748,4 +792,83 @@ public static RolePO initializeRolePOWithVersion(RoleEntity roleEntity, RolePO.B throw new RuntimeException("Failed to serialize json object:", e); } } + + /** + * Initialize GroupPO + * + * @param groupEntity GroupEntity object + * @return GroupPO object with version initialized + */ + public static GroupPO initializeGroupPOWithVersion( + GroupEntity groupEntity, GroupPO.Builder builder) { + try { + return builder + .withGroupId(groupEntity.id()) + .withGroupName(groupEntity.name()) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(groupEntity.auditInfo())) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Update GroupPO version + * + * @param oldGroupPO the old GroupPO object + * @param newGroup the new GroupEntity object + * @return GroupPO object with updated version + */ + public static GroupPO updateGroupPOWithVersion(GroupPO oldGroupPO, GroupEntity newGroup) { + Long lastVersion = oldGroupPO.getLastVersion(); + // TODO: set the version to the last version + 1 when having some fields need be multiple + // version + Long nextVersion = lastVersion; + try { + return GroupPO.builder() + .withGroupId(oldGroupPO.getGroupId()) + .withGroupName(newGroup.name()) + .withMetalakeId(oldGroupPO.getMetalakeId()) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(newGroup.auditInfo())) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Initialize GroupRoleRelPO + * + * @param groupEntity GroupEntity object + * @param roleIds list of role ids + * @return GroupRoleRelPO object with version initialized + */ + public static List initializeGroupRoleRelsPOWithVersion( + GroupEntity groupEntity, List roleIds) { + try { + List groupRoleRelPOS = Lists.newArrayList(); + for (Long roleId : roleIds) { + GroupRoleRelPO roleRelPO = + GroupRoleRelPO.builder() + .withGroupId(groupEntity.id()) + .withRoleId(roleId) + .withAuditInfo( + JsonUtils.anyFieldMapper().writeValueAsString(groupEntity.auditInfo())) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + groupRoleRelPOS.add(roleRelPO); + } + return groupRoleRelPOS; + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 8ce48ab031b..42fad98d86a 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -236,6 +236,8 @@ void testRestart(String type) throws IOException { auditInfo); UserEntity user1 = createUser(RandomIdGenerator.INSTANCE.nextId(), "metalake", "user1", auditInfo); + GroupEntity group1 = + createGroup(RandomIdGenerator.INSTANCE.nextId(), "metalake", "group1", auditInfo); // Store all entities store.put(metalake); @@ -246,6 +248,7 @@ void testRestart(String type) throws IOException { store.put(fileset1); store.put(topic1); store.put(user1); + store.put(group1); Assertions.assertDoesNotThrow( () -> @@ -288,6 +291,13 @@ void testRestart(String type) throws IOException { AuthorizationUtils.ofUser("metalake", "user1"), Entity.EntityType.USER, UserEntity.class)); + + Assertions.assertDoesNotThrow( + () -> + store.get( + AuthorizationUtils.ofGroup("metalake", "group1"), + EntityType.GROUP, + GroupEntity.class)); } // It will automatically close the store we create before, then we reopen the entity store @@ -334,6 +344,12 @@ void testRestart(String type) throws IOException { AuthorizationUtils.ofUser("metalake", "user1"), Entity.EntityType.USER, UserEntity.class)); + Assertions.assertDoesNotThrow( + () -> + store.get( + AuthorizationUtils.ofGroup("metalake", "group1"), + Entity.EntityType.GROUP, + GroupEntity.class)); destroy(type); } } @@ -464,9 +480,9 @@ public void testAuthorizationEntityDelete(String type) throws IOException { store.put(oneUser); UserEntity anotherUser = createUser(2L, "metalake", "anotherUser", auditInfo); store.put(anotherUser); - GroupEntity oneGroup = createGroup("metalake", "oneGroup", auditInfo); + GroupEntity oneGroup = createGroup(1L, "metalake", "oneGroup", auditInfo); store.put(oneGroup); - GroupEntity anotherGroup = createGroup("metalake", "anotherGroup", auditInfo); + GroupEntity anotherGroup = createGroup(2L, "metalake", "anotherGroup", auditInfo); store.put(anotherGroup); RoleEntity oneRole = createRole("metalake", "oneRole", auditInfo); store.put(oneRole); @@ -533,6 +549,8 @@ void testEntityDelete(String type) throws IOException { 2L, Namespace.of("metalake", "catalog", "schema2"), "topic1", auditInfo); UserEntity user1 = createUser(1L, "metalake", "user1", auditInfo); UserEntity user2 = createUser(2L, "metalake", "user2", auditInfo); + GroupEntity group1 = createGroup(1L, "metalake", "group1", auditInfo); + GroupEntity group2 = createGroup(2L, "metalake", "group2", auditInfo); // Store all entities store.put(metalake); @@ -548,6 +566,8 @@ void testEntityDelete(String type) throws IOException { store.put(topic1InSchema2); store.put(user1); store.put(user2); + store.put(group1); + store.put(group2); validateAllEntityExist( metalake, @@ -563,10 +583,14 @@ void testEntityDelete(String type) throws IOException { topic1, topic1InSchema2, user1, - user2); + user2, + group1, + group2); validateDeleteUser(store, user1); + validateDeleteGroup(store, group1); + validateDeleteTable(store, schema2, table1, table1InSchema2); validateDeleteFileset(store, schema2, fileset1, fileset1InSchema2); @@ -587,7 +611,7 @@ void testEntityDelete(String type) throws IOException { topic1, topic1InSchema2); - validateDeleteMetalake(store, metalake, catalogCopy, user2); + validateDeleteMetalake(store, metalake, catalogCopy, user2, group2); // Store all entities again // metalake @@ -675,6 +699,9 @@ void testEntityDelete(String type) throws IOException { UserEntity userNew = createUser(RandomIdGenerator.INSTANCE.nextId(), "metalake", "userNew", auditInfo); store.put(userNew); + GroupEntity groupNew = + createGroup(RandomIdGenerator.INSTANCE.nextId(), "metalake", "groupNew", auditInfo); + store.put(groupNew); validateDeleteTableCascade(store, table1New); @@ -686,7 +713,7 @@ void testEntityDelete(String type) throws IOException { validateDeleteCatalogCascade(store, catalogNew, schema2New); - validateDeleteMetalakeCascade(store, metalakeNew, catalogNew, schema2New, userNew); + validateDeleteMetalakeCascade(store, metalakeNew, catalogNew, schema2New, userNew, groupNew); destroy(type); } @@ -1188,31 +1215,28 @@ public static TopicEntity createTopicEntity( private static UserEntity createUser(Long id, String metalake, String name, AuditInfo auditInfo) { return UserEntity.builder() .withId(id) - .withNamespace( - Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME)) + .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) .withRoleNames(null) .build(); } - private static GroupEntity createGroup(String metalake, String name, AuditInfo auditInfo) { + private static GroupEntity createGroup( + Long id, String metalake, String name, AuditInfo auditInfo) { return GroupEntity.builder() - .withId(1L) - .withNamespace( - Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME)) + .withId(id) + .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList()) + .withRoleNames(null) .build(); } private static RoleEntity createRole(String metalake, String name, AuditInfo auditInfo) { return RoleEntity.builder() .withId(1L) - .withNamespace( - Namespace.of( - metalake, CatalogEntity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) + .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) .withSecurableObject(SecurableObjects.of("catalog")) @@ -1283,9 +1307,11 @@ private void validateDeleteMetalakeCascade( BaseMetalake metalake, CatalogEntity catalog, SchemaEntity schema2, - UserEntity userNew) + UserEntity userNew, + GroupEntity groupNew) throws IOException { Assertions.assertTrue(store.exists(userNew.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertTrue(store.exists(groupNew.nameIdentifier(), Entity.EntityType.GROUP)); Assertions.assertTrue( store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, true)); @@ -1295,6 +1321,7 @@ private void validateDeleteMetalakeCascade( Assertions.assertFalse(store.exists(schema2.nameIdentifier(), Entity.EntityType.SCHEMA)); Assertions.assertFalse(store.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); Assertions.assertFalse(store.exists(userNew.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertFalse(store.exists(groupNew.nameIdentifier(), EntityType.GROUP)); } private void validateDeleteCatalogCascade( @@ -1373,10 +1400,15 @@ private void validateDeleteSchemaCascade( } private static void validateDeleteMetalake( - EntityStore store, BaseMetalake metalake, CatalogEntity catalogCopy, UserEntity user2) + EntityStore store, + BaseMetalake metalake, + CatalogEntity catalogCopy, + UserEntity user2, + GroupEntity group2) throws IOException { // Now delete catalog 'catalogCopy' and metalake Assertions.assertTrue(store.exists(user2.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertTrue(store.exists(group2.nameIdentifier(), Entity.EntityType.GROUP)); Assertions.assertThrowsExactly( NonEmptyEntityException.class, @@ -1387,6 +1419,7 @@ private static void validateDeleteMetalake( store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE); Assertions.assertFalse(store.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); Assertions.assertFalse(store.exists(user2.nameIdentifier(), Entity.EntityType.USER)); + Assertions.assertFalse(store.exists(group2.nameIdentifier(), Entity.EntityType.GROUP)); } private static void validateDeleteCatalog( @@ -1508,6 +1541,16 @@ private void validateDeleteUser(EntityStore store, UserEntity user1) throws IOEx Assertions.assertTrue(store.exists(user.nameIdentifier(), Entity.EntityType.USER)); } + private void validateDeleteGroup(EntityStore store, GroupEntity group1) throws IOException { + Assertions.assertTrue(store.delete(group1.nameIdentifier(), EntityType.GROUP)); + Assertions.assertFalse(store.exists(group1.nameIdentifier(), Entity.EntityType.GROUP)); + + GroupEntity group = + createGroup(RandomIdGenerator.INSTANCE.nextId(), "metalake", "group1", group1.auditInfo()); + store.put(group); + Assertions.assertTrue(store.exists(group.nameIdentifier(), EntityType.GROUP)); + } + private void validateDeleteTable( EntityStore store, SchemaEntity schema2, TableEntity table1, TableEntity table1InSchema2) throws IOException { @@ -1546,7 +1589,9 @@ private static void validateAllEntityExist( TopicEntity topic1, TopicEntity topic1InSchema2, UserEntity user1, - UserEntity user2) + UserEntity user2, + GroupEntity group1, + GroupEntity group2) throws IOException { // Now try to get Assertions.assertEquals( @@ -1583,6 +1628,10 @@ private static void validateAllEntityExist( user1, store.get(user1.nameIdentifier(), Entity.EntityType.USER, UserEntity.class)); Assertions.assertEquals( user2, store.get(user2.nameIdentifier(), Entity.EntityType.USER, UserEntity.class)); + Assertions.assertEquals( + group1, store.get(group1.nameIdentifier(), Entity.EntityType.GROUP, GroupEntity.class)); + Assertions.assertEquals( + group2, store.get(group2.nameIdentifier(), Entity.EntityType.GROUP, GroupEntity.class)); } private void validateDeletedFileset(EntityStore store) throws IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java index 9ec63ed5c07..7b55ebe8a96 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java @@ -21,6 +21,7 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; @@ -207,6 +208,15 @@ public void testEntityStoreAndRetrieve() throws Exception { .withRoleNames(null) .build(); + GroupEntity groupEntity = + GroupEntity.builder() + .withId(1L) + .withName("group") + .withNamespace(Namespace.of("metalake", "catalog", "db")) + .withAuditInfo(auditInfo) + .withRoleNames(null) + .build(); + InMemoryEntityStore store = new InMemoryEntityStore(); store.initialize(Mockito.mock(Config.class)); store.setSerDe(Mockito.mock(EntitySerDe.class)); @@ -217,6 +227,7 @@ public void testEntityStoreAndRetrieve() throws Exception { store.put(tableEntity); store.put(filesetEntity); store.put(userEntity); + store.put(groupEntity); Metalake retrievedMetalake = store.get(metalake.nameIdentifier(), EntityType.METALAKE, BaseMetalake.class); @@ -242,6 +253,10 @@ public void testEntityStoreAndRetrieve() throws Exception { store.get(userEntity.nameIdentifier(), EntityType.USER, UserEntity.class); Assertions.assertEquals(userEntity, retrievedUser); + GroupEntity retrievedGroup = + store.get(groupEntity.nameIdentifier(), EntityType.GROUP, GroupEntity.class); + Assertions.assertEquals(groupEntity, retrievedGroup); + store.delete(metalake.nameIdentifier(), EntityType.METALAKE); NameIdentifier id = metalake.nameIdentifier(); Assertions.assertThrows( diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 569429829cd..5c6c4aa2e51 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -31,6 +31,7 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; @@ -745,13 +746,18 @@ public static UserEntity createUserEntity( } public static UserEntity createUserEntity( - Long id, Namespace namespace, String name, AuditInfo auditInfo, List roleNames) { + Long id, + Namespace namespace, + String name, + AuditInfo auditInfo, + List roleNames, + List roleIds) { return UserEntity.builder() .withId(id) .withName(name) .withNamespace(namespace) .withRoleNames(roleNames) - .withRoleIds(null) + .withRoleIds(roleIds) .withAuditInfo(auditInfo) .build(); } @@ -768,4 +774,33 @@ public static RoleEntity createRoleEntity( .withPrivileges(Lists.newArrayList(Privileges.fromName(Privilege.Name.LOAD_CATALOG))) .build(); } + + public static GroupEntity createGroupEntity( + Long id, Namespace namespace, String name, AuditInfo auditInfo) { + return GroupEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withRoleNames(null) + .withRoleIds(null) + .withAuditInfo(auditInfo) + .build(); + } + + public static GroupEntity createGroupEntity( + Long id, + Namespace namespace, + String name, + AuditInfo auditInfo, + List roleNames, + List roleIds) { + return GroupEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(auditInfo) + .build(); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java new file mode 100644 index 00000000000..4adef8db3e3 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java @@ -0,0 +1,614 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.authorization.AuthorizationUtils; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.GroupEntity; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.TestJDBCBackend; +import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; +import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.time.Instant; +import java.util.List; +import java.util.function.Function; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TestGroupMetaService extends TestJDBCBackend { + + String metalakeName = "metalake"; + + @Test + void getGroupByIdentifier() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // get not exist group + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + groupMetaService.getGroupByIdentifier( + AuthorizationUtils.ofGroup(metalakeName, "group1"))); + + // get group + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo); + groupMetaService.insertGroup(group1, false); + Assertions.assertEquals(group1, groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + + // get group with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + GroupEntity group2 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + groupMetaService.insertGroup(group2, false); + GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); + Assertions.assertEquals(group2.name(), actualGroup.name()); + Assertions.assertEquals( + Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + } + + @Test + void insertGroup() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // insert group + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group1, false)); + Assertions.assertEquals(group1, groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + + // insert duplicate group + GroupEntity group1Exist = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo); + Assertions.assertThrows( + AlreadyExistsException.class, () -> groupMetaService.insertGroup(group1Exist, false)); + + // insert overwrite + GroupEntity group1Overwrite = + createGroupEntity( + group1.id(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1Overwrite", + auditInfo); + Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group1Overwrite, true)); + Assertions.assertEquals( + "group1Overwrite", + groupMetaService.getGroupByIdentifier(group1Overwrite.nameIdentifier()).name()); + Assertions.assertEquals( + group1Overwrite, groupMetaService.getGroupByIdentifier(group1Overwrite.nameIdentifier())); + + // insert group with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + GroupEntity group2 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2, false)); + GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); + Assertions.assertEquals(group2.name(), actualGroup.name()); + Assertions.assertEquals( + Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + + // insert duplicate group with roles + GroupEntity group2Exist = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo); + Assertions.assertThrows( + AlreadyExistsException.class, () -> groupMetaService.insertGroup(group2Exist, false)); + + // insert overwrite group with roles + GroupEntity group2Overwrite = + createGroupEntity( + group1.id(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2Overwrite", + auditInfo); + Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2Overwrite, true)); + Assertions.assertEquals( + "group2Overwrite", + groupMetaService.getGroupByIdentifier(group2Overwrite.nameIdentifier()).name()); + Assertions.assertEquals( + group2Overwrite, groupMetaService.getGroupByIdentifier(group2Overwrite.nameIdentifier())); + } + + @Test + void deleteGroup() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + // delete group + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group1, false)); + Assertions.assertEquals(group1, groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + Assertions.assertTrue(groupMetaService.deleteGroup(group1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + + // delete group with roles + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + GroupEntity group2 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + groupMetaService.insertGroup(group2, false); + List rolePOs = + SessionUtils.doWithCommitAndFetchResult( + RoleMetaMapper.class, mapper -> mapper.listRolesByGroupId(group2.id())); + Assertions.assertEquals(2, rolePOs.size()); + GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); + Assertions.assertEquals(group2.name(), actualGroup.name()); + Assertions.assertEquals( + Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + + Assertions.assertTrue(groupMetaService.deleteGroup(group2.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group2.nameIdentifier())); + rolePOs = + SessionUtils.doWithCommitAndFetchResult( + RoleMetaMapper.class, mapper -> mapper.listRolesByGroupId(group2.id())); + Assertions.assertEquals(0, rolePOs.size()); + } + + @Test + void updateGroup() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + groupMetaService.insertGroup(group1, false); + GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(group1.name(), actualGroup.name()); + Assertions.assertEquals( + Sets.newHashSet(group1.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role3, false); + + // update group (grant) + Function grantUpdater = + group -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(group.auditInfo().creator()) + .withCreateTime(group.auditInfo().createTime()) + .withLastModifier("grantGroup") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(group.roleNames()); + List roleIds = Lists.newArrayList(group.roleIds()); + roleNames.add(role3.name()); + roleIds.add(role3.id()); + + return GroupEntity.builder() + .withNamespace(group.namespace()) + .withId(group.id()) + .withName(group.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + + Assertions.assertNotNull(groupMetaService.updateGroup(group1.nameIdentifier(), grantUpdater)); + GroupEntity grantGroup = + GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(group1.id(), grantGroup.id()); + Assertions.assertEquals(group1.name(), grantGroup.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantGroup.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantGroup.roleIds())); + Assertions.assertEquals("creator", grantGroup.auditInfo().creator()); + Assertions.assertEquals("grantGroup", grantGroup.auditInfo().lastModifier()); + + // update group (revoke) + Function revokeUpdater = + group -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(group.auditInfo().creator()) + .withCreateTime(group.auditInfo().createTime()) + .withLastModifier("revokeGroup") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(group.roleNames()); + List roleIds = Lists.newArrayList(group.roleIds()); + roleIds.remove(roleNames.indexOf("role2")); + roleNames.remove("role2"); + + return GroupEntity.builder() + .withNamespace(group.namespace()) + .withId(group.id()) + .withName(group.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + + Assertions.assertNotNull(groupMetaService.updateGroup(group1.nameIdentifier(), revokeUpdater)); + GroupEntity revokeGroup = + GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(group1.id(), revokeGroup.id()); + Assertions.assertEquals(group1.name(), revokeGroup.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeGroup.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeGroup.roleIds())); + Assertions.assertEquals("creator", revokeGroup.auditInfo().creator()); + Assertions.assertEquals("revokeGroup", revokeGroup.auditInfo().lastModifier()); + + RoleEntity role4 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role4", + auditInfo); + roleMetaService.insertRole(role4, false); + + // update group (grant & revoke) + Function grantRevokeUpdater = + group -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(group.auditInfo().creator()) + .withCreateTime(group.auditInfo().createTime()) + .withLastModifier("grantRevokeUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(group.roleNames()); + List roleIds = Lists.newArrayList(group.roleIds()); + roleIds.remove(roleNames.indexOf("role3")); + roleNames.remove("role3"); + roleIds.add(role4.id()); + roleNames.add(role4.name()); + + return GroupEntity.builder() + .withNamespace(group.namespace()) + .withId(group.id()) + .withName(group.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + Assertions.assertNotNull( + groupMetaService.updateGroup(group1.nameIdentifier(), grantRevokeUpdater)); + GroupEntity grantRevokeGroup = + GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(group1.id(), grantRevokeGroup.id()); + Assertions.assertEquals(group1.name(), grantRevokeGroup.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeGroup.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeGroup.roleIds())); + Assertions.assertEquals("creator", grantRevokeGroup.auditInfo().creator()); + Assertions.assertEquals("grantRevokeUser", grantRevokeGroup.auditInfo().lastModifier()); + + // no update + Function noUpdater = + group -> { + AuditInfo updateAuditInfo = + AuditInfo.builder() + .withCreator(group.auditInfo().creator()) + .withCreateTime(group.auditInfo().createTime()) + .withLastModifier("noUpdateUser") + .withLastModifiedTime(Instant.now()) + .build(); + + List roleNames = Lists.newArrayList(group.roleNames()); + List roleIds = Lists.newArrayList(group.roleIds()); + + return GroupEntity.builder() + .withNamespace(group.namespace()) + .withId(group.id()) + .withName(group.name()) + .withRoleNames(roleNames) + .withRoleIds(roleIds) + .withAuditInfo(updateAuditInfo) + .build(); + }; + Assertions.assertNotNull(groupMetaService.updateGroup(group1.nameIdentifier(), noUpdater)); + GroupEntity noUpdaterGroup = + GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(group1.id(), noUpdaterGroup.id()); + Assertions.assertEquals(group1.name(), noUpdaterGroup.name()); + Assertions.assertEquals( + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterGroup.roleNames())); + Assertions.assertEquals( + Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterGroup.roleIds())); + Assertions.assertEquals("creator", noUpdaterGroup.auditInfo().creator()); + Assertions.assertEquals("grantRevokeUser", noUpdaterGroup.auditInfo().lastModifier()); + } + + @Test + void deleteMetalake() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + roleMetaService.insertRole(role3, false); + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + GroupEntity group2 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo, + Lists.newArrayList(role3.name()), + Lists.newArrayList(role3.id())); + groupMetaService.insertGroup(group1, false); + groupMetaService.insertGroup(group2, false); + + Assertions.assertEquals( + group1.name(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).name()); + Assertions.assertEquals(2, roleMetaService.listRolesByGroupId(group1.id()).size()); + Assertions.assertEquals( + group2.name(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).name()); + Assertions.assertEquals(1, roleMetaService.listRolesByGroupId(group2.id()).size()); + + // delete metalake without cascade + Assertions.assertTrue( + MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), false)); + + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group2.nameIdentifier())); + Assertions.assertEquals(0, roleMetaService.listRolesByGroupId(group1.id()).size()); + Assertions.assertEquals(0, roleMetaService.listRolesByGroupId(group2.id()).size()); + } + + @Test + void deleteMetalakeCascade() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + GroupMetaService groupMetaService = GroupMetaService.getInstance(); + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo); + RoleEntity role2 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role2", + auditInfo); + RoleEntity role3 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role3", + auditInfo); + roleMetaService.insertRole(role1, false); + roleMetaService.insertRole(role2, false); + roleMetaService.insertRole(role3, false); + GroupEntity group1 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group1", + auditInfo, + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); + GroupEntity group2 = + createGroupEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofGroupNamespace(metalakeName), + "group2", + auditInfo, + Lists.newArrayList(role3.name()), + Lists.newArrayList(role3.id())); + groupMetaService.insertGroup(group1, false); + groupMetaService.insertGroup(group2, false); + + Assertions.assertEquals( + group1.name(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).name()); + Assertions.assertEquals(2, roleMetaService.listRolesByGroupId(group1.id()).size()); + Assertions.assertEquals( + group2.name(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).name()); + Assertions.assertEquals(1, roleMetaService.listRolesByGroupId(group2.id()).size()); + + // delete metalake with cascade + Assertions.assertTrue( + MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), true)); + + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group1.nameIdentifier())); + Assertions.assertThrows( + NoSuchEntityException.class, + () -> groupMetaService.getGroupByIdentifier(group2.nameIdentifier())); + Assertions.assertEquals(0, roleMetaService.listRolesByGroupId(group1.id()).size()); + Assertions.assertEquals(0, roleMetaService.listRolesByGroupId(group2.id()).size()); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java index 9a509e0dcc8..dabd372c12f 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java @@ -78,7 +78,8 @@ void getUserByIdentifier() { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); userMetaService.insertUser(user2, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); @@ -155,7 +156,8 @@ void insertUser() { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2, false)); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); @@ -236,7 +238,8 @@ void deleteUser() { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); userMetaService.insertUser(user2, false); List rolePOs = SessionUtils.doWithCommitAndFetchResult( @@ -288,7 +291,8 @@ void updateUser() { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); userMetaService.insertUser(user1, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user1.nameIdentifier()); Assertions.assertEquals(user1.name(), actualUser.name()); @@ -500,14 +504,16 @@ void deleteMetalake() { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList("role3")); + Lists.newArrayList(role3.name()), + Lists.newArrayList(role3.id())); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); @@ -570,14 +576,16 @@ void deleteMetalakeCascade() { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList("role1", "role2")); + Lists.newArrayList(role1.name(), role2.name()), + Lists.newArrayList(role1.id(), role2.id())); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList("role3")); + Lists.newArrayList(role3.name()), + Lists.newArrayList(role3.id())); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); diff --git a/core/src/test/resources/h2/schema-h2.sql b/core/src/test/resources/h2/schema-h2.sql index 0b2c269b9bc..0fa534548dc 100644 --- a/core/src/test/resources/h2/schema-h2.sql +++ b/core/src/test/resources/h2/schema-h2.sql @@ -164,4 +164,29 @@ CREATE TABLE IF NOT EXISTS `user_role_rel` ( PRIMARY KEY (`id`), CONSTRAINT `uk_ui_ri_del` UNIQUE (`user_id`, `role_id`, `deleted_at`), KEY `idx_rid` (`role_id`) -) ENGINE=InnoDB; \ No newline at end of file +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `group_meta` ( + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', + PRIMARY KEY (`group_id`), + CONSTRAINT `uk_mid_gr_del` UNIQUE (`metalake_id`, `group_name`, `deleted_at`) + ) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `group_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + CONSTRAINT `uk_gi_ri_del` UNIQUE (`group_id`, `role_id`, `deleted_at`), + KEY `idx_gid` (`group_id`) + ) ENGINE=InnoDB; \ No newline at end of file diff --git a/scripts/mysql/schema-0.5.0-mysql.sql b/scripts/mysql/schema-0.5.0-mysql.sql index f78942a65c9..99bf52e3404 100644 --- a/scripts/mysql/schema-0.5.0-mysql.sql +++ b/scripts/mysql/schema-0.5.0-mysql.sql @@ -156,4 +156,29 @@ CREATE TABLE IF NOT EXISTS `user_role_rel` ( PRIMARY KEY (`id`), UNIQUE KEY `uk_ui_ri_del` (`user_id`, `role_id`, `deleted_at`), KEY `idx_rid` (`role_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; + +CREATE TABLE IF NOT EXISTS `group_meta` ( + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', + PRIMARY KEY (`group_id`), + UNIQUE KEY `uk_mid_gr_del` (`metalake_id`, `group_name`, `deleted_at`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group metadata'; + +CREATE TABLE IF NOT EXISTS `group_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_gi_ri_del` (`group_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`group_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group role relation'; \ No newline at end of file From c6047f4550d494e5595b2a2e85be1be1090ae529 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Mon, 22 Apr 2024 19:46:53 +0800 Subject: [PATCH 095/106] [#3085] improve(web): add custom svg icons and generating usage (#3097) ### What changes were proposed in this pull request? Add custom svg icons and generating usage. ### Why are the changes needed? Fix: #3085 ### Does this PR introduce _any_ user-facing change? image ### How was this patch tested? N/A --- build.gradle.kts | 1 + web/README.md | 10 + web/licenses/antfu-install-pkg.txt | 21 + web/licenses/antfu-utils.txt | 21 + web/licenses/boolbase.txt | 13 + web/licenses/buffer-crc32.txt | 19 + web/licenses/cheerio-select.txt | 11 + web/licenses/cheerio.txt | 21 + web/licenses/confbox.txt | 118 +++ web/licenses/css-select.txt | 11 + web/licenses/css-what.txt | 11 + web/licenses/csso.txt | 20 + web/licenses/dom-serializer.txt | 7 + web/licenses/domelementtype.txt | 11 + web/licenses/domhandler.txt | 11 + web/licenses/domutils.txt | 11 + web/licenses/end-of-stream.txt | 21 + web/licenses/entities.txt | 11 + web/licenses/esbuild.txt | 9 + web/licenses/execa.txt | 9 + web/licenses/extract-zip.txt | 23 + web/licenses/fd-slicer.txt | 21 + web/licenses/fs-minipass.txt | 15 + web/licenses/get-stream.txt | 9 + web/licenses/htmlparser2.txt | 18 + web/licenses/human-signals.txt | 201 +++++ web/licenses/iconify-tools.txt | 21 + web/licenses/iconify-utils.txt | 21 + web/licenses/is-stream.txt | 9 + web/licenses/kolorist.txt | 21 + web/licenses/local-pkg.txt | 21 + web/licenses/merge-stream.txt | 21 + web/licenses/mimic-function.txt | 9 + web/licenses/minizlib.txt | 26 + web/licenses/mkdirp.txt | 21 + web/licenses/mlly.txt | 21 + web/licenses/onetime.txt | 9 + .../parse5-htmlparser2-tree-adapter.txt | 19 + web/licenses/parse5.txt | 19 + web/licenses/pathe.txt | 44 ++ web/licenses/pend.txt | 23 + web/licenses/pkg-types.txt | 44 ++ web/licenses/pump.txt | 21 + web/licenses/strip-final-newline.txt | 9 + web/licenses/svgo.txt | 21 + web/licenses/trysound-sax.txt | 15 + web/licenses/tsx.txt | 21 + web/licenses/types-tar.txt | 8 + web/licenses/types-yauzl.txt | 8 + web/licenses/ufo.txt | 21 + web/licenses/yauzl.txt | 21 + web/package.json | 6 +- web/pnpm-lock.yaml | 742 ++++++++++++++++++ web/src/app/layout.js | 2 + .../app/metalakes/metalake/MetalakeTree.js | 4 +- web/src/components/Icon.js | 7 +- web/src/lib/icons/iconify-icons.css | 22 + web/src/lib/icons/iconify-icons.js | 53 ++ web/src/lib/icons/svg/doris.svg | 7 + web/src/lib/icons/svg/hive.svg | 51 ++ 60 files changed, 2039 insertions(+), 3 deletions(-) create mode 100644 web/licenses/antfu-install-pkg.txt create mode 100644 web/licenses/antfu-utils.txt create mode 100644 web/licenses/boolbase.txt create mode 100644 web/licenses/buffer-crc32.txt create mode 100644 web/licenses/cheerio-select.txt create mode 100644 web/licenses/cheerio.txt create mode 100644 web/licenses/confbox.txt create mode 100644 web/licenses/css-select.txt create mode 100644 web/licenses/css-what.txt create mode 100644 web/licenses/csso.txt create mode 100644 web/licenses/dom-serializer.txt create mode 100644 web/licenses/domelementtype.txt create mode 100644 web/licenses/domhandler.txt create mode 100644 web/licenses/domutils.txt create mode 100644 web/licenses/end-of-stream.txt create mode 100644 web/licenses/entities.txt create mode 100644 web/licenses/esbuild.txt create mode 100644 web/licenses/execa.txt create mode 100644 web/licenses/extract-zip.txt create mode 100644 web/licenses/fd-slicer.txt create mode 100644 web/licenses/fs-minipass.txt create mode 100644 web/licenses/get-stream.txt create mode 100644 web/licenses/htmlparser2.txt create mode 100644 web/licenses/human-signals.txt create mode 100644 web/licenses/iconify-tools.txt create mode 100644 web/licenses/iconify-utils.txt create mode 100644 web/licenses/is-stream.txt create mode 100644 web/licenses/kolorist.txt create mode 100644 web/licenses/local-pkg.txt create mode 100644 web/licenses/merge-stream.txt create mode 100644 web/licenses/mimic-function.txt create mode 100644 web/licenses/minizlib.txt create mode 100644 web/licenses/mkdirp.txt create mode 100644 web/licenses/mlly.txt create mode 100644 web/licenses/onetime.txt create mode 100644 web/licenses/parse5-htmlparser2-tree-adapter.txt create mode 100644 web/licenses/parse5.txt create mode 100644 web/licenses/pathe.txt create mode 100644 web/licenses/pend.txt create mode 100644 web/licenses/pkg-types.txt create mode 100644 web/licenses/pump.txt create mode 100644 web/licenses/strip-final-newline.txt create mode 100644 web/licenses/svgo.txt create mode 100644 web/licenses/trysound-sax.txt create mode 100644 web/licenses/tsx.txt create mode 100644 web/licenses/types-tar.txt create mode 100644 web/licenses/types-yauzl.txt create mode 100644 web/licenses/ufo.txt create mode 100644 web/licenses/yauzl.txt create mode 100644 web/src/lib/icons/iconify-icons.css create mode 100644 web/src/lib/icons/iconify-icons.js create mode 100644 web/src/lib/icons/svg/doris.svg create mode 100644 web/src/lib/icons/svg/hive.svg diff --git a/build.gradle.kts b/build.gradle.kts index 08b2dccf8bf..c98483c1ff5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -478,6 +478,7 @@ tasks.rat { "web/yarn.lock", "web/package-lock.json", "web/pnpm-lock.yaml", + "web/src/lib/icons/svg/**/*.svg", "**/LICENSE.*", "**/NOTICE.*", "ROADMAP.md", diff --git a/web/README.md b/web/README.md index 386aceebc74..781de26f252 100644 --- a/web/README.md +++ b/web/README.md @@ -63,6 +63,8 @@ The Gravitino Web UI only works in the latest version of the Chrome browser. You ### Development scripts +#### Lint and format styles + This command runs ESLint to help you inspect the code. If errors occur, please make modifications based on the provided prompts. ```bash @@ -81,6 +83,14 @@ This command automatically formats the code. pnpm format ``` +#### Custom SVG icons + +If you need to add a custom icon, please add your SVG icon in the `./src/lib/icons/svg` directory and run the following command: + +```bash +pnpm gen:icons +``` + ## Self-hosting deployment ### Static HTML export diff --git a/web/licenses/antfu-install-pkg.txt b/web/licenses/antfu-install-pkg.txt new file mode 100644 index 00000000000..d47cea55b87 --- /dev/null +++ b/web/licenses/antfu-install-pkg.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Anthony Fu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/antfu-utils.txt b/web/licenses/antfu-utils.txt new file mode 100644 index 00000000000..d47cea55b87 --- /dev/null +++ b/web/licenses/antfu-utils.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Anthony Fu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/boolbase.txt b/web/licenses/boolbase.txt new file mode 100644 index 00000000000..a689aae96e8 --- /dev/null +++ b/web/licenses/boolbase.txt @@ -0,0 +1,13 @@ +Copyright (c) 2014-2015, Felix Boehm + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/web/licenses/buffer-crc32.txt b/web/licenses/buffer-crc32.txt new file mode 100644 index 00000000000..9d2e5162a2d --- /dev/null +++ b/web/licenses/buffer-crc32.txt @@ -0,0 +1,19 @@ +The MIT License + +Copyright (c) 2013-2024 Brian J. Brennan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the +Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/cheerio-select.txt b/web/licenses/cheerio-select.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/cheerio-select.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/cheerio.txt b/web/licenses/cheerio.txt new file mode 100644 index 00000000000..b0c8b193581 --- /dev/null +++ b/web/licenses/cheerio.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 The Cheerio contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/confbox.txt b/web/licenses/confbox.txt new file mode 100644 index 00000000000..71e6facaf67 --- /dev/null +++ b/web/licenses/confbox.txt @@ -0,0 +1,118 @@ +MIT License + +Copyright (c) Pooya Parsa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- + +js-yaml: https://github.com/nodeca/js-yaml/tree/master + +(The MIT License) + +Copyright (C) 2011-2015 by Vitaly Puzrin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- + +smol-toml: https://github.com/squirrelchat/smol-toml/blob/mistress/LICENSE + +Copyright (c) Squirrel Chat et al., All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +jsonc-parser: https://github.com/microsoft/node-jsonc-parser/blob/main/LICENSE.md + +The MIT License (MIT) + +Copyright (c) Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +json5: https://github.com/json5/json5/blob/main/LICENSE.md + +MIT License + +Copyright (c) 2012-2018 Aseem Kishore, and others (https://github.com/json5/json5/graphs/contributors) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +detect-indent: https://github.com/sindresorhus/detect-indent/blob/main/license + +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/css-select.txt b/web/licenses/css-select.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/css-select.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/css-what.txt b/web/licenses/css-what.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/css-what.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/csso.txt b/web/licenses/csso.txt new file mode 100644 index 00000000000..a200643d9a7 --- /dev/null +++ b/web/licenses/csso.txt @@ -0,0 +1,20 @@ +Copyright (C) 2015-2021 by Roman Dvornov +Copyright (C) 2011-2015 by Sergey Kryzhanovsky + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/dom-serializer.txt b/web/licenses/dom-serializer.txt new file mode 100644 index 00000000000..8a3a2faf4de --- /dev/null +++ b/web/licenses/dom-serializer.txt @@ -0,0 +1,7 @@ +Copyright © 2022 The Cheerio contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/domelementtype.txt b/web/licenses/domelementtype.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/domelementtype.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/domhandler.txt b/web/licenses/domhandler.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/domhandler.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/domutils.txt b/web/licenses/domutils.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/domutils.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/end-of-stream.txt b/web/licenses/end-of-stream.txt new file mode 100644 index 00000000000..8ff6bf68f98 --- /dev/null +++ b/web/licenses/end-of-stream.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathias Buus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/entities.txt b/web/licenses/entities.txt new file mode 100644 index 00000000000..c464f863ea2 --- /dev/null +++ b/web/licenses/entities.txt @@ -0,0 +1,11 @@ +Copyright (c) Felix Böhm +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/esbuild.txt b/web/licenses/esbuild.txt new file mode 100644 index 00000000000..52b49de6e06 --- /dev/null +++ b/web/licenses/esbuild.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2020 Evan Wallace + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/execa.txt b/web/licenses/execa.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/execa.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/extract-zip.txt b/web/licenses/extract-zip.txt new file mode 100644 index 00000000000..29210691350 --- /dev/null +++ b/web/licenses/extract-zip.txt @@ -0,0 +1,23 @@ +Copyright (c) 2014 Max Ogden and other contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/web/licenses/fd-slicer.txt b/web/licenses/fd-slicer.txt new file mode 100644 index 00000000000..e57596d2453 --- /dev/null +++ b/web/licenses/fd-slicer.txt @@ -0,0 +1,21 @@ +Copyright (c) 2014 Andrew Kelley + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/fs-minipass.txt b/web/licenses/fs-minipass.txt new file mode 100644 index 00000000000..19129e315fe --- /dev/null +++ b/web/licenses/fs-minipass.txt @@ -0,0 +1,15 @@ +The ISC License + +Copyright (c) Isaac Z. Schlueter and Contributors + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/web/licenses/get-stream.txt b/web/licenses/get-stream.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/get-stream.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/htmlparser2.txt b/web/licenses/htmlparser2.txt new file mode 100644 index 00000000000..fd75500960f --- /dev/null +++ b/web/licenses/htmlparser2.txt @@ -0,0 +1,18 @@ +Copyright 2010, 2011, Chris Winberry . All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/web/licenses/human-signals.txt b/web/licenses/human-signals.txt new file mode 100644 index 00000000000..6e735072004 --- /dev/null +++ b/web/licenses/human-signals.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 ehmicky + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/web/licenses/iconify-tools.txt b/web/licenses/iconify-tools.txt new file mode 100644 index 00000000000..119ebd5a29f --- /dev/null +++ b/web/licenses/iconify-tools.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021-PRESENT Vjacheslav Trushkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/iconify-utils.txt b/web/licenses/iconify-utils.txt new file mode 100644 index 00000000000..119ebd5a29f --- /dev/null +++ b/web/licenses/iconify-utils.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021-PRESENT Vjacheslav Trushkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/is-stream.txt b/web/licenses/is-stream.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/is-stream.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/kolorist.txt b/web/licenses/kolorist.txt new file mode 100644 index 00000000000..de68841bce4 --- /dev/null +++ b/web/licenses/kolorist.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2020-present Marvin Hagemeister + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/local-pkg.txt b/web/licenses/local-pkg.txt new file mode 100644 index 00000000000..d47cea55b87 --- /dev/null +++ b/web/licenses/local-pkg.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Anthony Fu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/merge-stream.txt b/web/licenses/merge-stream.txt new file mode 100644 index 00000000000..0e0cf1e3997 --- /dev/null +++ b/web/licenses/merge-stream.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Stephen Sugden (stephensugden.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/mimic-function.txt b/web/licenses/mimic-function.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/mimic-function.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/minizlib.txt b/web/licenses/minizlib.txt new file mode 100644 index 00000000000..49f7efe431c --- /dev/null +++ b/web/licenses/minizlib.txt @@ -0,0 +1,26 @@ +Minizlib was created by Isaac Z. Schlueter. +It is a derivative work of the Node.js project. + +""" +Copyright (c) 2017-2023 Isaac Z. Schlueter and Contributors +Copyright (c) 2017-2023 Node.js contributors. All rights reserved. +Copyright (c) 2017-2023 Joyent, Inc. and other Node contributors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" diff --git a/web/licenses/mkdirp.txt b/web/licenses/mkdirp.txt new file mode 100644 index 00000000000..0a034db7a73 --- /dev/null +++ b/web/licenses/mkdirp.txt @@ -0,0 +1,21 @@ +Copyright (c) 2011-2023 James Halliday (mail@substack.net) and Isaac Z. Schlueter (i@izs.me) + +This project is free software released under the MIT license: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/mlly.txt b/web/licenses/mlly.txt new file mode 100644 index 00000000000..e739abce461 --- /dev/null +++ b/web/licenses/mlly.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Pooya Parsa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/onetime.txt b/web/licenses/onetime.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/onetime.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/parse5-htmlparser2-tree-adapter.txt b/web/licenses/parse5-htmlparser2-tree-adapter.txt new file mode 100644 index 00000000000..f3265d4b88f --- /dev/null +++ b/web/licenses/parse5-htmlparser2-tree-adapter.txt @@ -0,0 +1,19 @@ +Copyright (c) 2013-2019 Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/parse5.txt b/web/licenses/parse5.txt new file mode 100644 index 00000000000..f3265d4b88f --- /dev/null +++ b/web/licenses/parse5.txt @@ -0,0 +1,19 @@ +Copyright (c) 2013-2019 Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/pathe.txt b/web/licenses/pathe.txt new file mode 100644 index 00000000000..743af992595 --- /dev/null +++ b/web/licenses/pathe.txt @@ -0,0 +1,44 @@ +MIT License + +Copyright (c) Pooya Parsa - Daniel Roe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +Copyright Joyent, Inc. and other Node contributors. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/pend.txt b/web/licenses/pend.txt new file mode 100644 index 00000000000..0bbb53ed41a --- /dev/null +++ b/web/licenses/pend.txt @@ -0,0 +1,23 @@ +The MIT License (Expat) + +Copyright (c) 2014 Andrew Kelley + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/pkg-types.txt b/web/licenses/pkg-types.txt new file mode 100644 index 00000000000..743af992595 --- /dev/null +++ b/web/licenses/pkg-types.txt @@ -0,0 +1,44 @@ +MIT License + +Copyright (c) Pooya Parsa - Daniel Roe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +Copyright Joyent, Inc. and other Node contributors. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/pump.txt b/web/licenses/pump.txt new file mode 100644 index 00000000000..8ff6bf68f98 --- /dev/null +++ b/web/licenses/pump.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathias Buus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/web/licenses/strip-final-newline.txt b/web/licenses/strip-final-newline.txt new file mode 100644 index 00000000000..fa7ceba3eb4 --- /dev/null +++ b/web/licenses/strip-final-newline.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/svgo.txt b/web/licenses/svgo.txt new file mode 100644 index 00000000000..38a47dacb90 --- /dev/null +++ b/web/licenses/svgo.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Kir Belevich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/trysound-sax.txt b/web/licenses/trysound-sax.txt new file mode 100644 index 00000000000..19129e315fe --- /dev/null +++ b/web/licenses/trysound-sax.txt @@ -0,0 +1,15 @@ +The ISC License + +Copyright (c) Isaac Z. Schlueter and Contributors + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/web/licenses/tsx.txt b/web/licenses/tsx.txt new file mode 100644 index 00000000000..51e4fd8646a --- /dev/null +++ b/web/licenses/tsx.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Hiroki Osame + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/types-tar.txt b/web/licenses/types-tar.txt new file mode 100644 index 00000000000..c3ee6e73d57 --- /dev/null +++ b/web/licenses/types-tar.txt @@ -0,0 +1,8 @@ +This project is licensed under the MIT license. +Copyrights are respective of each contributor listed at the beginning of each definition file. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/types-yauzl.txt b/web/licenses/types-yauzl.txt new file mode 100644 index 00000000000..c3ee6e73d57 --- /dev/null +++ b/web/licenses/types-yauzl.txt @@ -0,0 +1,8 @@ +This project is licensed under the MIT license. +Copyrights are respective of each contributor listed at the beginning of each definition file. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/web/licenses/ufo.txt b/web/licenses/ufo.txt new file mode 100644 index 00000000000..e739abce461 --- /dev/null +++ b/web/licenses/ufo.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Pooya Parsa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/licenses/yauzl.txt b/web/licenses/yauzl.txt new file mode 100644 index 00000000000..37538d4d091 --- /dev/null +++ b/web/licenses/yauzl.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Josh Wolfe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web/package.json b/web/package.json index 62a3b9a05fd..3f56f4403ba 100644 --- a/web/package.json +++ b/web/package.json @@ -13,7 +13,8 @@ "start": "next start", "lint": "next lint", "format": "prettier --write .", - "prettier:check": "prettier --check ." + "prettier:check": "prettier --check .", + "gen:icons": "tsx src/lib/icons/iconify-icons.js && prettier --write src/lib/icons/iconify-icons.css" }, "dependencies": { "@emotion/cache": "^11.11.0", @@ -45,6 +46,8 @@ }, "devDependencies": { "@iconify/react": "^4.1.1", + "@iconify/tools": "^4.0.4", + "@iconify/utils": "^2.1.23", "@next/bundle-analyzer": "^14.0.4", "@types/lodash-es": "^4.17.12", "@types/node": "^20.10.5", @@ -58,6 +61,7 @@ "postcss": "^8", "prettier": "^3.1.0", "tailwindcss": "^3.3.5", + "tsx": "^4.7.2", "typescript": "^5.3.3" } } diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index 4b8a3023228..32d3316b926 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -90,6 +90,12 @@ importers: '@iconify/react': specifier: ^4.1.1 version: 4.1.1(react@18.2.0) + '@iconify/tools': + specifier: ^4.0.4 + version: 4.0.4 + '@iconify/utils': + specifier: ^2.1.23 + version: 2.1.23 '@next/bundle-analyzer': specifier: ^14.0.4 version: 14.0.4 @@ -129,6 +135,9 @@ importers: tailwindcss: specifier: ^3.3.5 version: 3.4.1 + tsx: + specifier: ^4.7.2 + version: 4.7.2 typescript: specifier: ^5.3.3 version: 5.3.3 @@ -167,6 +176,12 @@ packages: peerDependencies: react: '>=16.9.0' + '@antfu/install-pkg@0.1.1': + resolution: {integrity: sha512-LyB/8+bSfa0DFGC06zpCEfs89/XoWZwws5ygEa5D+Xsm3OfI+aXQ86VgVG7Acyef+rSZ5HE7J8rrxzrQeM3PjQ==} + + '@antfu/utils@0.7.7': + resolution: {integrity: sha512-gFPqTG7otEJ8uP6wrhDv6mqwGWYZKNvAcCq6u9hOj0c+IKCEsY4L1oC9trPq2SaWIzAfHvqfBDxF591JkMf+kg==} + '@babel/code-frame@7.23.5': resolution: {integrity: sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==} engines: {node: '>=6.9.0'} @@ -263,6 +278,144 @@ packages: '@emotion/weak-memoize@0.3.1': resolution: {integrity: sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww==} + '@esbuild/aix-ppc64@0.19.12': + resolution: {integrity: sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.19.12': + resolution: {integrity: sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.19.12': + resolution: {integrity: sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.19.12': + resolution: {integrity: sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.19.12': + resolution: {integrity: sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.19.12': + resolution: {integrity: sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.19.12': + resolution: {integrity: sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.19.12': + resolution: {integrity: sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.19.12': + resolution: {integrity: sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.19.12': + resolution: {integrity: sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.19.12': + resolution: {integrity: sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.19.12': + resolution: {integrity: sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.19.12': + resolution: {integrity: sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.19.12': + resolution: {integrity: sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.19.12': + resolution: {integrity: sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.19.12': + resolution: {integrity: sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.19.12': + resolution: {integrity: sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-x64@0.19.12': + resolution: {integrity: sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-x64@0.19.12': + resolution: {integrity: sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/sunos-x64@0.19.12': + resolution: {integrity: sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.19.12': + resolution: {integrity: sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.19.12': + resolution: {integrity: sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.19.12': + resolution: {integrity: sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.4.0': resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -317,9 +470,15 @@ packages: peerDependencies: react: '>=16' + '@iconify/tools@4.0.4': + resolution: {integrity: sha512-hX1Z3i1Tm6JxyrDv45jNEijPpepZZfal/4leFGtUC04H9LsgRo597BOBFB9PUZsQdFGLOxVUUfv6lqU/dC+xXw==} + '@iconify/types@2.0.0': resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} + '@iconify/utils@2.1.23': + resolution: {integrity: sha512-YGNbHKM5tyDvdWZ92y2mIkrfvm5Fvhe6WJSkWu7vvOFhMtYDP0casZpoRz0XEHZCrYsR4stdGT3cZ52yp5qZdQ==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -625,6 +784,10 @@ packages: '@swc/helpers@0.5.2': resolution: {integrity: sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==} + '@trysound/sax@0.2.0': + resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} + engines: {node: '>=10.13.0'} + '@types/hoist-non-react-statics@3.3.5': resolution: {integrity: sha512-SbcrWzkKBw2cdwRTwQAswfpB9g9LJWfjtUeW/jvNwbhC8cpmmNYVePa+ncbUe0rGTQ7G3Ff6mYUN2VMfLVr+Sg==} @@ -661,9 +824,15 @@ packages: '@types/scheduler@0.16.8': resolution: {integrity: sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==} + '@types/tar@6.1.13': + resolution: {integrity: sha512-IznnlmU5f4WcGTh2ltRu/Ijpmk8wiWXfF0VA4s+HPjHZgvFggk1YaIkbo5krX/zUCzWF8N/l4+W/LNxnvAJ8nw==} + '@types/use-sync-external-store@0.0.3': resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==} + '@types/yauzl@2.10.3': + resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@typescript-eslint/parser@6.18.1': resolution: {integrity: sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA==} engines: {node: ^16.0.0 || >=18.0.0} @@ -837,6 +1006,9 @@ packages: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} + boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -852,6 +1024,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} @@ -878,10 +1053,21 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + + cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} + chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + chroma-js@2.4.2: resolution: {integrity: sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A==} @@ -926,6 +1112,9 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + confbox@0.1.7: + resolution: {integrity: sha512-uJcB/FKZtBMCJpK8MQji6bJHgu1tixKPxRLeGkNzBoOZzpnZUJm0jm2/sBDWcuBx1dYgxV4JU+g5hmNxCyAmdA==} + convert-source-map@1.9.0: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} @@ -943,15 +1132,34 @@ packages: css-in-js-utils@3.1.0: resolution: {integrity: sha512-fJAcud6B3rRu+KHYk+Bwf+WFL2MDCJJ1XG9x137tJQ0xYxor7XziQtuGFbWNdqrvF4Tk26O3H73nfVqXt/fW1A==} + css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + css-tree@1.1.3: resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==} engines: {node: '>=8.0.0'} + css-tree@2.2.1: + resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + + css-tree@2.3.1: + resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} + + css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} hasBin: true + csso@5.0.5: + resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} @@ -1018,6 +1226,19 @@ packages: dom-helpers@5.2.1: resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} @@ -1033,10 +1254,17 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + enhanced-resolve@5.15.0: resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==} engines: {node: '>=10.13.0'} + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + env-cmd@10.1.0: resolution: {integrity: sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==} engines: {node: '>=8.0.0'} @@ -1066,6 +1294,11 @@ packages: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} + esbuild@0.19.12: + resolution: {integrity: sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg==} + engines: {node: '>=12'} + hasBin: true + escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -1185,6 +1418,15 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + + extract-zip@2.0.1: + resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} + engines: {node: '>= 10.17.0'} + hasBin: true + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1210,6 +1452,9 @@ packages: fastq@1.16.0: resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} + fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -1255,6 +1500,10 @@ packages: fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -1276,6 +1525,14 @@ packages: get-intrinsic@1.2.2: resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + + get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + get-symbol-description@1.0.0: resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} engines: {node: '>= 0.4'} @@ -1368,6 +1625,13 @@ packages: hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + + human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + hyphenate-style-name@1.0.4: resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==} @@ -1479,6 +1743,10 @@ packages: is-shared-array-buffer@1.0.2: resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-string@1.0.7: resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} engines: {node: '>= 0.4'} @@ -1553,6 +1821,9 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + kolorist@1.8.0: + resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} @@ -1575,6 +1846,10 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + local-pkg@0.5.0: + resolution: {integrity: sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==} + engines: {node: '>=14'} + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -1603,6 +1878,15 @@ packages: mdn-data@2.0.14: resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} + mdn-data@2.0.28: + resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} + + mdn-data@2.0.30: + resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -1619,6 +1903,10 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -1629,10 +1917,34 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + + minipass@4.2.8: + resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} + engines: {node: '>=8'} + + minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + minipass@7.0.4: resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==} engines: {node: '>=16 || 14 >=14.17'} + minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + + mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + + mlly@1.6.1: + resolution: {integrity: sha512-vLgaHvaeunuOXHSmEbZ9izxPx3USsk8KCQ8iC+aTlp5sKRSoZvwhHh5L9VbKSaVC6sJDqbyohIS76E2VmHIPAA==} + mrmime@1.0.1: resolution: {integrity: sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==} engines: {node: '>=10'} @@ -1686,9 +1998,16 @@ packages: resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} engines: {node: '>=0.10.0'} + npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + nprogress@0.2.0: resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} + nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -1729,6 +2048,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -1753,6 +2076,12 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + + parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -1776,6 +2105,12 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -1791,6 +2126,9 @@ packages: resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} engines: {node: '>= 6'} + pkg-types@1.1.0: + resolution: {integrity: sha512-/RpmvKdxKf8uILTtoOhAgf30wYbP2Qw+L9p3Rvshx1JZVX+XQNZQFjlbmGHEGIm4CkVPlSn+NXmIM8+9oWQaSA==} + postcss-import@15.1.0: resolution: {integrity: sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==} engines: {node: '>=14.0.0'} @@ -1854,6 +2192,9 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -2275,6 +2616,9 @@ packages: side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -2355,6 +2699,10 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -2395,6 +2743,11 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} + svgo@3.2.0: + resolution: {integrity: sha512-4PP6CMW/V7l/GmKRKzsLR8xxjdHTV4IMvhTnpuHwwBazSIlw5W/5SmPjN8Dwyt7lKbSJrRDgp4t9ph0HgChFBQ==} + engines: {node: '>=14.0.0'} + hasBin: true + tailwindcss@3.4.1: resolution: {integrity: sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==} engines: {node: '>=14.0.0'} @@ -2404,6 +2757,10 @@ packages: resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} engines: {node: '>=6'} + tar@6.2.1: + resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} + engines: {node: '>=10'} + text-table@0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} @@ -2461,6 +2818,11 @@ packages: tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + tsx@4.7.2: + resolution: {integrity: sha512-BCNd4kz6fz12fyrgCTEdZHGJ9fWTGeUzXmQysh0RVocDY3h4frk05ZNCXSy4kIenF7y/QnrdiVpTsyNRn6vlAw==} + engines: {node: '>=18.0.0'} + hasBin: true + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -2493,6 +2855,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + ufo@1.5.3: + resolution: {integrity: sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==} + unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} @@ -2578,6 +2943,9 @@ packages: resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} engines: {node: '>= 14'} + yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2628,6 +2996,13 @@ snapshots: resize-observer-polyfill: 1.5.1 throttle-debounce: 5.0.0 + '@antfu/install-pkg@0.1.1': + dependencies: + execa: 5.1.1 + find-up: 5.0.0 + + '@antfu/utils@0.7.7': {} + '@babel/code-frame@7.23.5': dependencies: '@babel/highlight': 7.23.4 @@ -2744,6 +3119,75 @@ snapshots: '@emotion/weak-memoize@0.3.1': {} + '@esbuild/aix-ppc64@0.19.12': + optional: true + + '@esbuild/android-arm64@0.19.12': + optional: true + + '@esbuild/android-arm@0.19.12': + optional: true + + '@esbuild/android-x64@0.19.12': + optional: true + + '@esbuild/darwin-arm64@0.19.12': + optional: true + + '@esbuild/darwin-x64@0.19.12': + optional: true + + '@esbuild/freebsd-arm64@0.19.12': + optional: true + + '@esbuild/freebsd-x64@0.19.12': + optional: true + + '@esbuild/linux-arm64@0.19.12': + optional: true + + '@esbuild/linux-arm@0.19.12': + optional: true + + '@esbuild/linux-ia32@0.19.12': + optional: true + + '@esbuild/linux-loong64@0.19.12': + optional: true + + '@esbuild/linux-mips64el@0.19.12': + optional: true + + '@esbuild/linux-ppc64@0.19.12': + optional: true + + '@esbuild/linux-riscv64@0.19.12': + optional: true + + '@esbuild/linux-s390x@0.19.12': + optional: true + + '@esbuild/linux-x64@0.19.12': + optional: true + + '@esbuild/netbsd-x64@0.19.12': + optional: true + + '@esbuild/openbsd-x64@0.19.12': + optional: true + + '@esbuild/sunos-x64@0.19.12': + optional: true + + '@esbuild/win32-arm64@0.19.12': + optional: true + + '@esbuild/win32-ia32@0.19.12': + optional: true + + '@esbuild/win32-x64@0.19.12': + optional: true + '@eslint-community/eslint-utils@4.4.0(eslint@8.56.0)': dependencies: eslint: 8.56.0 @@ -2805,8 +3249,36 @@ snapshots: '@iconify/types': 2.0.0 react: 18.2.0 + '@iconify/tools@4.0.4': + dependencies: + '@iconify/types': 2.0.0 + '@iconify/utils': 2.1.23 + '@types/tar': 6.1.13 + axios: 1.6.8 + cheerio: 1.0.0-rc.12 + extract-zip: 2.0.1 + local-pkg: 0.5.0 + pathe: 1.1.2 + svgo: 3.2.0 + tar: 6.2.1 + transitivePeerDependencies: + - debug + - supports-color + '@iconify/types@2.0.0': {} + '@iconify/utils@2.1.23': + dependencies: + '@antfu/install-pkg': 0.1.1 + '@antfu/utils': 0.7.7 + '@iconify/types': 2.0.0 + debug: 4.3.4 + kolorist: 1.8.0 + local-pkg: 0.5.0 + mlly: 1.6.1 + transitivePeerDependencies: + - supports-color + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -3109,6 +3581,8 @@ snapshots: dependencies: tslib: 2.6.2 + '@trysound/sax@0.2.0': {} + '@types/hoist-non-react-statics@3.3.5': dependencies: '@types/react': 18.2.47 @@ -3146,8 +3620,18 @@ snapshots: '@types/scheduler@0.16.8': {} + '@types/tar@6.1.13': + dependencies: + '@types/node': 20.10.7 + minipass: 4.2.8 + '@types/use-sync-external-store@0.0.3': {} + '@types/yauzl@2.10.3': + dependencies: + '@types/node': 20.10.7 + optional: true + '@typescript-eslint/parser@6.18.1(eslint@8.56.0)(typescript@5.3.3)': dependencies: '@typescript-eslint/scope-manager': 6.18.1 @@ -3395,6 +3879,8 @@ snapshots: binary-extensions@2.2.0: {} + boolbase@1.0.0: {} + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -3415,6 +3901,8 @@ snapshots: node-releases: 2.0.14 update-browserslist-db: 1.0.13(browserslist@4.22.2) + buffer-crc32@0.2.13: {} + busboy@1.6.0: dependencies: streamsearch: 1.1.0 @@ -3442,6 +3930,25 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + cheerio-select@2.1.0: + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + + cheerio@1.0.0-rc.12: + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + chokidar@3.5.3: dependencies: anymatch: 3.1.3 @@ -3454,6 +3961,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + chownr@2.0.0: {} + chroma-js@2.4.2: {} classnames@2.5.1: {} @@ -3486,6 +3995,8 @@ snapshots: concat-map@0.0.1: {} + confbox@0.1.7: {} + convert-source-map@1.9.0: {} copy-to-clipboard@3.3.3: @@ -3510,13 +4021,37 @@ snapshots: dependencies: hyphenate-style-name: 1.0.4 + css-select@5.1.0: + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + css-tree@1.1.3: dependencies: mdn-data: 2.0.14 source-map: 0.6.1 + css-tree@2.2.1: + dependencies: + mdn-data: 2.0.28 + source-map-js: 1.0.2 + + css-tree@2.3.1: + dependencies: + mdn-data: 2.0.30 + source-map-js: 1.0.2 + + css-what@6.1.0: {} + cssesc@3.0.0: {} + csso@5.0.5: + dependencies: + css-tree: 2.2.1 + csstype@3.1.3: {} damerau-levenshtein@1.0.8: {} @@ -3570,6 +4105,24 @@ snapshots: '@babel/runtime': 7.23.8 csstype: 3.1.3 + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domutils@3.1.0: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + duplexer@0.1.2: {} eastasianwidth@0.2.0: {} @@ -3580,11 +4133,17 @@ snapshots: emoji-regex@9.2.2: {} + end-of-stream@1.4.4: + dependencies: + once: 1.4.0 + enhanced-resolve@5.15.0: dependencies: graceful-fs: 4.2.11 tapable: 2.2.1 + entities@4.5.0: {} + env-cmd@10.1.0: dependencies: commander: 4.1.1 @@ -3673,6 +4232,32 @@ snapshots: is-date-object: 1.0.5 is-symbol: 1.0.4 + esbuild@0.19.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.19.12 + '@esbuild/android-arm': 0.19.12 + '@esbuild/android-arm64': 0.19.12 + '@esbuild/android-x64': 0.19.12 + '@esbuild/darwin-arm64': 0.19.12 + '@esbuild/darwin-x64': 0.19.12 + '@esbuild/freebsd-arm64': 0.19.12 + '@esbuild/freebsd-x64': 0.19.12 + '@esbuild/linux-arm': 0.19.12 + '@esbuild/linux-arm64': 0.19.12 + '@esbuild/linux-ia32': 0.19.12 + '@esbuild/linux-loong64': 0.19.12 + '@esbuild/linux-mips64el': 0.19.12 + '@esbuild/linux-ppc64': 0.19.12 + '@esbuild/linux-riscv64': 0.19.12 + '@esbuild/linux-s390x': 0.19.12 + '@esbuild/linux-x64': 0.19.12 + '@esbuild/netbsd-x64': 0.19.12 + '@esbuild/openbsd-x64': 0.19.12 + '@esbuild/sunos-x64': 0.19.12 + '@esbuild/win32-arm64': 0.19.12 + '@esbuild/win32-ia32': 0.19.12 + '@esbuild/win32-x64': 0.19.12 + escalade@3.1.1: {} escape-string-regexp@1.0.5: {} @@ -3876,6 +4461,28 @@ snapshots: esutils@2.0.3: {} + execa@5.1.1: + dependencies: + cross-spawn: 7.0.3 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + + extract-zip@2.0.1: + dependencies: + debug: 4.3.4 + get-stream: 5.2.0 + yauzl: 2.10.0 + optionalDependencies: + '@types/yauzl': 2.10.3 + transitivePeerDependencies: + - supports-color + fast-deep-equal@3.1.3: {} fast-glob@3.3.2: @@ -3900,6 +4507,10 @@ snapshots: dependencies: reusify: 1.0.4 + fd-slicer@1.1.0: + dependencies: + pend: 1.2.0 + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0 @@ -3942,6 +4553,10 @@ snapshots: fraction.js@4.3.7: {} + fs-minipass@2.1.0: + dependencies: + minipass: 3.3.6 + fs.realpath@1.0.0: {} fsevents@2.3.3: @@ -3965,6 +4580,12 @@ snapshots: has-symbols: 1.0.3 hasown: 2.0.0 + get-stream@5.2.0: + dependencies: + pump: 3.0.0 + + get-stream@6.0.1: {} + get-symbol-description@1.0.0: dependencies: call-bind: 1.0.5 @@ -4069,6 +4690,15 @@ snapshots: dependencies: react-is: 16.13.1 + htmlparser2@8.0.2: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + + human-signals@2.1.0: {} + hyphenate-style-name@1.0.4: {} ignore@5.3.0: {} @@ -4174,6 +4804,8 @@ snapshots: dependencies: call-bind: 1.0.5 + is-stream@2.0.1: {} + is-string@1.0.7: dependencies: has-tostringtag: 1.0.0 @@ -4252,6 +4884,8 @@ snapshots: dependencies: json-buffer: 3.0.1 + kolorist@1.8.0: {} + language-subtag-registry@0.3.22: {} language-tags@1.0.9: @@ -4269,6 +4903,11 @@ snapshots: lines-and-columns@1.2.4: {} + local-pkg@0.5.0: + dependencies: + mlly: 1.6.1 + pkg-types: 1.1.0 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 @@ -4291,6 +4930,12 @@ snapshots: mdn-data@2.0.14: {} + mdn-data@2.0.28: {} + + mdn-data@2.0.30: {} + + merge-stream@2.0.0: {} + merge2@1.4.1: {} micromatch@4.0.5: @@ -4304,6 +4949,8 @@ snapshots: dependencies: mime-db: 1.52.0 + mimic-fn@2.1.0: {} + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 @@ -4314,8 +4961,30 @@ snapshots: minimist@1.2.8: {} + minipass@3.3.6: + dependencies: + yallist: 4.0.0 + + minipass@4.2.8: {} + + minipass@5.0.0: {} + minipass@7.0.4: {} + minizlib@2.1.2: + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + + mkdirp@1.0.4: {} + + mlly@1.6.1: + dependencies: + acorn: 8.11.3 + pathe: 1.1.2 + pkg-types: 1.1.0 + ufo: 1.5.3 + mrmime@1.0.1: {} ms@2.1.2: {} @@ -4376,8 +5045,16 @@ snapshots: normalize-range@0.1.2: {} + npm-run-path@4.0.1: + dependencies: + path-key: 3.1.1 + nprogress@0.2.0: {} + nth-check@2.1.1: + dependencies: + boolbase: 1.0.0 + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -4427,6 +5104,10 @@ snapshots: dependencies: wrappy: 1.0.2 + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + opener@1.5.2: {} optionator@0.9.3: @@ -4457,6 +5138,15 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + parse5-htmlparser2-tree-adapter@7.0.0: + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + + parse5@7.1.2: + dependencies: + entities: 4.5.0 + path-exists@4.0.0: {} path-is-absolute@1.0.1: {} @@ -4472,6 +5162,10 @@ snapshots: path-type@4.0.0: {} + pathe@1.1.2: {} + + pend@1.2.0: {} + picocolors@1.0.0: {} picomatch@2.3.1: {} @@ -4480,6 +5174,12 @@ snapshots: pirates@4.0.6: {} + pkg-types@1.1.0: + dependencies: + confbox: 0.1.7 + mlly: 1.6.1 + pathe: 1.1.2 + postcss-import@15.1.0(postcss@8.4.33): dependencies: postcss: 8.4.33 @@ -4537,6 +5237,11 @@ snapshots: proxy-from-env@1.1.0: {} + pump@3.0.0: + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + punycode@2.3.1: {} qrcode.react@3.1.0(react@18.2.0): @@ -5062,6 +5767,8 @@ snapshots: get-intrinsic: 1.2.2 object-inspect: 1.13.1 + signal-exit@3.0.7: {} + signal-exit@4.1.0: {} sirv@1.0.19: @@ -5153,6 +5860,8 @@ snapshots: strip-bom@3.0.0: {} + strip-final-newline@2.0.0: {} + strip-json-comments@3.1.1: {} styled-jsx@5.1.1(react@18.2.0): @@ -5184,6 +5893,16 @@ snapshots: supports-preserve-symlinks-flag@1.0.0: {} + svgo@3.2.0: + dependencies: + '@trysound/sax': 0.2.0 + commander: 7.2.0 + css-select: 5.1.0 + css-tree: 2.3.1 + css-what: 6.1.0 + csso: 5.0.5 + picocolors: 1.0.0 + tailwindcss@3.4.1: dependencies: '@alloc/quick-lru': 5.2.0 @@ -5213,6 +5932,15 @@ snapshots: tapable@2.2.1: {} + tar@6.2.1: + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + text-table@0.2.0: {} thenify-all@1.6.0: @@ -5258,6 +5986,13 @@ snapshots: tslib@2.6.2: {} + tsx@4.7.2: + dependencies: + esbuild: 0.19.12 + get-tsconfig: 4.7.2 + optionalDependencies: + fsevents: 2.3.3 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -5295,6 +6030,8 @@ snapshots: typescript@5.3.3: {} + ufo@1.5.3: {} + unbox-primitive@1.0.2: dependencies: call-bind: 1.0.5 @@ -5404,6 +6141,11 @@ snapshots: yaml@2.3.4: {} + yauzl@2.10.0: + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + yocto-queue@0.1.0: {} yup@1.3.3: diff --git a/web/src/app/layout.js b/web/src/app/layout.js index 1842e88a371..3aa9477bdf8 100644 --- a/web/src/app/layout.js +++ b/web/src/app/layout.js @@ -10,6 +10,8 @@ import Provider from '@/lib/provider' import Layout from './rootLayout/Layout' import StyledToast from '../components/StyledToast' +import '../lib/icons/iconify-icons.css' + export const metadata = { title: 'Gravitino', description: 'A high-performance, geo-distributed and federated metadata lake.', diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index 9435bf7e648..28bd1552aa9 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -45,13 +45,15 @@ const MetalakeTree = props => { case 'relational': switch (provider) { case 'hive': - return 'simple-icons:apachehive' + return 'custom-icons-hive' case 'lakehouse-iceberg': return 'openmoji:iceberg' case 'jdbc-mysql': return 'devicon:mysql-wordmark' case 'jdbc-postgresql': return 'devicon:postgresql-wordmark' + case 'jdbc-doris': + return 'custom-icons-doris' default: return 'bx:book' } diff --git a/web/src/components/Icon.js b/web/src/components/Icon.js index 1c9869019bb..137e6788fed 100644 --- a/web/src/components/Icon.js +++ b/web/src/components/Icon.js @@ -6,9 +6,14 @@ 'use client' import { Icon } from '@iconify/react' +import clsx from 'clsx' const IconifyIcon = ({ icon, ...props }) => { - return + return icon.startsWith('custom-icons') ? ( + + ) : ( + + ) } export default IconifyIcon diff --git a/web/src/lib/icons/iconify-icons.css b/web/src/lib/icons/iconify-icons.css new file mode 100644 index 00000000000..7d5de1f8e4d --- /dev/null +++ b/web/src/lib/icons/iconify-icons.css @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +.custom-icons-doris, +.custom-icons-hive { + display: inline-block; + width: 1em; + height: 1em; + background-repeat: no-repeat; + background-size: 100% 100%; +} + +.custom-icons-doris { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 512 512' width='512' height='512'%3E%3Cg fill-rule='evenodd'%3E%3Cpath fill='%2314a8c9' d='M177.5-.5h24q22.477 4.488 40 19.5A2984 2984 0 01306 84.5q23.81 30.358 11 67a122 122 0 01-14 26 1561 1561 0 00-46.5 47.5q-6.906 5.4-15 2L134 119.5q-29.48-41.033-6-86 17.649-27.408 49.5-34' opacity='.998'/%3E%3Cpath fill='%235168ac' d='M80.5 117.5a44.6 44.6 0 019 2.5L216 246.5q5.361 9.862-1 19L89.5 391q-15.194 6.261-21.5-8.5-1-127 0-254 2.835-9.52 12.5-11' opacity='.991'/%3E%3Cpath fill='%2351c9a2' d='M206.5 511.5h-27q-46.42-10.662-59.5-57-8.375-34.994 14-63a27563 27563 0 01197-198q21.361-25.778 27.5-59a1387 1387 0 0054.5 54 617 617 0 0126 33q20.97 44.308-8 84A24184 24184 0 01245.5 490a170 170 0 01-17 14q-10.424 6.057-22 7.5' opacity='.998'/%3E%3C/g%3E%3C/svg%3E"); +} + +.custom-icons-hive { + width: 1.12em; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1000 900' width='1000' height='900'%3E%3Cpath fill-rule='evenodd' stroke='%23fdee21' stroke-width='5.129' d='M596.797 704.59v192.85h47.346v-88.632l9.767.183v88.45h47.881V704.59h-47.88v71.552h-9.768V704.59zm163.92 0h-46.116v192.85h46.116zm7.55 0 36.429 192.85h54.984l36.429-192.85h-48.951l-12.067 82.517c-1.006 3.16-4.402 3.16-5.009 0l-12.06-82.52zm229.17 0h-91.538v192.85h91.538v-50.569l-50.898-.434v-21.679h24.248v-50.204H946.54v-19.397h50.898z'/%3E%3Cpath fill='%23fdee21' fill-rule='evenodd' d='M503.127 119.752c-7.71-21.115-272.92-124.22-276.31-112.9-48.7 12.695-68.52 68.826-102.02 104.38l-74.393-4.261C17.633 149.113.437 192.17 12.146 236.921c42.002 56.906 90.76 105.33 121.15 176.81 2.402 33.692 145.82 3.533 176.56-3.136-41.992 30.059-78.561 76.65-62.846 210.84 14.346 63.014 24.159 133.37 151.4 204.64 16.75 9.381 51.407 20.207 72.838 28.098 20.495 9.4 44.461 13.264 112.07-7.413 39.124-16.863 81.365-27.022 119.65-43.844l-46.017 2.16c-63.369 1.378-112.29 6.105-127.38-11.6l-58.32-100.68 34-66.04c47.056 4.826 62.675 42.986 104.15 57.518l48.885-36.215c141.99 83.816 198.48-53.12 214.67-159.77-1.728-43.392-93.952 13.61-88.362-6.68 2.166-46.643-35.854-107.67-60.42-155.22l28.43-110.01c12.9-11.49-59.72-133.86-119.02-149.12-52.03-13.39-130.46 52.493-130.46 52.493z'/%3E%3Cg fill-rule='evenodd'%3E%3Cg fill='%23c8c037'%3E%3Cpath d='M736.387 353.55c.51 11.816.452 26.187 1.594 35.15-.197 6.838-5.323 7.089-9.564 8.521l27.63 10.12c5.486 9.23 9.895 18.462 14.348 27.693 5.165 22.708 1.31 23.357-2.126 25.031-10.027.102-20.025.121-29.358-1.864 4.181 2.207 5.1 3.815 5.543 6.642.816 5.212-2.54 12.33-8.067 19.188 8.172 4.534 23.071 8.97 34.14 13.181l12.62-28.093c-5.715-41.27-23.045-79.309-46.76-115.57zm51.57 138.43c15.324 6.387 36.495 4.515 65.848-10.613 4.365-3.104 8.431-1.248.62 3.858-38.5 34.547-56.928 17.265-66.468 6.755'/%3E%3Cpath d='M856.747 476.84c4.552 8.717 1.794 15.805.266 23.167-4.754 37.449-16.423 67.056-39.321 97.994-74.478 119.74-149.67 44.817-209.62-3.945l-24.798 62.137c-1.317 5.446-6.79 9.688 31.937 26.738l39.452-27.867c150.46 114.48 257.08-160.24 202.08-178.22zm-230.53 107c-10.151.724-40.361 10.997-41.706 16.946l13.902-20.712zm-19.74-126.97c1.86 0 10.788 2.397 9.83 1.864-1.37-.763-.153 12.519 2.657 19.439l-10.893 23.7c20.865-24.063 53.858-22.462 84.486-25.564l-14.612-9.054c2.538-8.466-.45-15.165-1.86-22.368zm51.27-71.36c-22.387 6.863-43.816 18.075-58.203 41.976 10.976-39.395 13.213-37.899 16.757-39.846 15.351-6.865 27.922-2.586 41.446-2.13'/%3E%3C/g%3E%3Cg fill='%23fcf6a0'%3E%3Cpath d='M538.197 847.25c-27.07 29.87-87.47-2.26-137.63-18.64-127.46-81.05-152.36-157.35-154.09-232.2-6.553-107.22 26.37-169.52 68.014-184.27-27.561 53.129-40.41 148.5-27.631 219.42 10.315 39.455 10.529 106 76.545 141.62 32.327 18.199 23.571 32.368 45.431 49.413 23.955 18.679 90.747 36.437 129.36 24.658zm40.44-495.51c-45.831-64.991-110.29-89.387-182.98-92.053 14.568-4.749 29.136-7.222 43.704-14.246 3.567-3.748 2.401-10.344 1.063-17.042-69.97-18.27-114.13-40.85-170.03-61.781l150.91 36.215c101.91 3.872 93.158 29.032 157.34 148.91z'/%3E%3Cpath d='M627.707 320.4c-33.87-48.35-66.72-107.03-111.31-144.35-107.64-48.584-214.12-84.666-338.91-117.5l39.827-51.216c132.42 30.227 256.8 80.808 368.21 164.19 18.849 47.66 31.419 95.383 42.173 148.87zm68.28-85.44s-18.988-42.599-28.233-58.866c-21.446-23.766-32.29-68.054-76.213-88.922 13.892 3.752 23.399-.718 51.759 24.992l44.669 84.734z'/%3E%3Cpath d='M719.607 308.76c4.38-36.908 12.695-96.11 3.152-119.5-26.55-35.624-53.29-72.446-80.04-106.63-4.43-4.123-7.62-9.719-11.11-14.084 37.61 9.744 76.86 35.894 129.79 139.13z'/%3E%3C/g%3E%3Cpath d='M561.487 313.12c-14.98-11.79-28.33-53.49-51.03-62.16-21.011-1.758-28.715-8.368-56.344-2.739 10.021-5.02 19.482-11.446 30.134-14.885 7.215-1.703 13.983.184 20.867 1.134 1.7-.804 2.814-1.803 2.398-3.313-27.812-17.187-84.866-17.432-123.38-27.243 44.7 1.496 94.156-.982 127.75 9.79 27.013 23.974 35.788 69.12 49.605 99.416m81.91-201.093c6.201.563 39.574 53.353 41.142 62.137 2.775 19.202 9.62 40.728 11.46 60.819-5.497-19.269-11.486-38.537-18.974-57.806-2.19-5.306-7.724-17.302-23.107-34.269-7.163-11.959-8.71-21.359-10.52-30.88zm113.09 287.523h-14.278l15.781 4.142zm-97.42-14.26c-14.358-1.765-29.091-2-43.448 1.157-5.852 7.86-6.255 15.856-8.813 23.245 17.454-19.236 24.38-20.412 52.261-24.401zm195.82 99.28c-7.99 6.377-8.788 12.379-32.413 19.572-17.228 4.074-26.961-2.831-35.07-12.116 12.294 3.522 14.259 12.81 47.225 2.397zm-71.6 3.99c-5.75 15.812-11.949 31.791-18.199 46.6-14.883 17.454-8.54 7.508-30.819 34.617 7.129-10.998 16.4-21.566 21.254-33.02 3.41-7.542 7.653-15.543 9.532-22.244-5.907-2.95-17.999-3.183-19.639-2.125-19.808 11.683-23.382 24.047-35.059 36.086 8.464-13.53 15.693-28.298 25.505-40.476 1.118-1.58 8.383-1.886 12.85-2.819-6.941-1.19-19.787-3.684-20.82-3.572-12.923 3.106-20.164 14.361-29.757 21.836 6.994-10.098 13.436-20.612 22.317-29.292.879-1.02 24.883 1.083 37.324 6.047l.247 4.95 7.063 2.85 5.314-13.314zm-156.75 95.33c-3.188-1.09-31.35-4.711-31.35-3.861-10.813 7.696-12.453 15.059-12.221 21.436 17.989-17.697 17.319-17.747 43.572-17.575zm-21.39-127.28s20.322-2.963 19.926-1.065c-1.472 7.05 2.488 24.233 2.923 21.836l-11.424 9.32c23.926-12.03 49.113-9.612 74.656-11.45 0 0-21.507-5.938-20.192-6.125 4.797-.68-1.278-22.43-2.923-22.9 4.091-1.385 8.087-2.43 12.221-3.462-30.756-8.049-48.541-3.306-75.188 13.847z'/%3E%3Cpath d='m781.027 471.78-4.517 2.263c-.64-13.23-14.85-12.26-26.834-12.382l-9.83-2.396c2.99 2.766 12.23 2.55 8.502 8.521-3.137 1.677-4.58 8.87-6.376 14.912l-5.314.266c9.387 3.669 18.095 6.884 28.428 11.184l6.11.932 14.613-7.855z'/%3E%3Cpath fill='%23fff' d='M646.327 462.86c-2.287-4.525-13.794-3.882-16.275-.955-2.016 2.379-.125 17.642 2.46 15.068 3.736-3.285 8.617-4.659 14.48-4.394.739-2.82.477-5.864-.665-9.719m110.35 22.74c1.082-.791 3.369-8.443 2.63-10.733-1.512-5.403-12.212-4.266-12.212-4.266-2.61 1.536-4.899 9.96-3.945 12.74.456 2.352 11.81 3.102 13.526 2.259z'/%3E%3Cpath d='m299.107 749.64 6.76.661c40.07 19.675 106.05 48.847 201.57 33.171l14.988 23.584c-42.067 20.442-87.312 15.738-129.71 17.172zm-57.53-155.54 20.073 84.268c53.74 33.593 145.54 72.07 222.06 68.965l-16.91-38.8c-159.73-33.92-173.24-76.8-225.22-114.44zm49.95-172.61c9.81 65.302 22.907 114.81 79.69 156.28 34.804 25.03 69.351 49.954 111.37 70.524 0 0-3.684 19.171-6.763 18.829-125.05-13.89-216.6-117.58-227.69-164.94 10.74-36.329 26.149-59.021 43.389-80.693zm60.11-37c16.664 72.376 56.397 145.68 95.889 212.18 14.477 18.369 18.266 26.475 40.579 37.659 30.159 9.55 51.486 7.112 73.377 5.676-5.998-11.102-11.329-22.706-18.217-33.14-49.018-38.765-26.364-74.06-13.344-96.406-27.256-6.887-63.37-21.525-68.345-40.778-8-62.271-3.937-82.025 4.068-114.01-36.909 7.782-74.309 15.494-114.01 28.82z'/%3E%3Cpath stroke='%23000' stroke-width='.114' d='M216.237.058c-12.131 7.925-23.818 19.256-36.391 32.135-20.843 21.575-34.699 42.621-55.915 58.224-4.243 3.659-16.151 9.551-31.121 10.425-7.052.352-11.646 1.519-24.604-.108-11.428-6.188-22.285-2.164-32.881 10.51-11.619 16.709-26.334 48.437-32.453 68.41-12.611 50.937 19.541 92.905 50.374 125.25 27.429 26.796 43.243 43.542 54.505 68.107 8.527 15.933 14.844 37.799 21.683 53.13 2.466 4.86 1.953 4.835 8.591 6.3 14.333 3.059 34.215 3.083 51.915 4.604 7.659.107 18.175-.178 28.14-1.217 13.731-2.592 29.863-5.133 43.384-9.81 13.213-3.253 24.985-7.759 35.597-11.906-1.368 4.644-11.478 9.115-15.268 15.002-35.97 51.49-45.823 97.308-39.56 169.6 3.542 32.042 10.895 58.605 21.991 88.997 5.078 13.908 15.882 35.934 26.236 50.565 30.79 43.506 99.672 99.374 195.56 120.88 16.73 2.286 35.715 1.067 53.571-3.689 47.326-14.346 143.78-48.275 143.78-48.275s-85.619 7.083-124.83 3.206c-9.078-1.42-19.08-1.901-25.405-8.087-1.06-1.37-4.914-9.132-2.425-9.202 3.395-.095 13.142-4.14 28.19-5.482-32.128-3.459-31.67-3.418-34.015-9.27-3.648-9.085-9.23-21.502-14.977-32.367 14.118 1.19 45.376 2.945 55.984-8.003 0 0-18.497 2.136-34.843.219-5.508-.646-14.891-3.995-17.71-5.03-7.339-2.858-13.38-3.746-14.788-5.61-2.549-6.485-4.276-8.666-7.351-17.355-4.166-11.504-4.496-24.337-5.29-36.083 10.693 13.18 24.318 24.275 42.356 29.94.232-.487 23.4 10.073 40.226 4.643l3.489-1.126c0 .312-11.148.977-15.237-1.047-34.219-14.545-39.37-27.276-44.895-33.599l-14.777-22.133c4.494-8.96 7.035-9.301 12.989-9.384 18.011 2 25.848 3.54 36.807.896 7.405 15.24 9.406 30.57 26.265 41.662 56.166 16.672 68.369-4.951 81.973-24.057 40.412 29.658 106.2 38.803 151.89.514 58.086-67.212 76.476-173.17 71.325-179.22-7.254-12.307-16.772-24.999-24.945-23.245-29.13 7.952-39.871 22.73-68.735 19.36 3.438-.195 9.219-.288 9.263-.616 2.224-24.44-.179-36.267-1.252-38.297-8.759-19.317-20.261-39.646-28.278-54.88-2.067-3.161-8.137-27.166-18.308-36.649-4.362-3.724-15.039-13.546-15.039-13.546l-.905 10.44s4.174.63 5.763 7.097c6.022 24.517 36.687 82.485 38.938 85.263 10.868 17.545 1.104 39.4 9.367 51.663.794 1.557 16.857-.137 29.307.628 20.473-4.403 19.609-13.426 37.294-14.782 11.913-.913 13.108 21.46 12.95 23.196-2.22 24.704-9.734 53.427-21.235 79.658-23.912 46.07-50.691 87.446-88.67 93.216-46.326 8.095-70.4-12.158-95.512-25.055l-9.607 8.194c-32.637 32.5-71.389 29.614-87.239-12.851-7.89-16.473-18.172-26.57-26.976-40.7l-46.688 33.627c-3.886 7.97-8.665 20.54-14.47 34.58-4.036 9.76-7.42 26.525-7.2 40.456-5.984 10.195 20.73 51.542 37.61 76.692 4.98 7.42 14.279 20.385 14.59 21.193 3.347 8.683 10.679 16.349 11.124 17.284 31.337 40.872-39.531 31.46-54.167 28.99-28.876-4.6-57.121-16.662-83.67-32.791a339 339 0 01-4.605-2.848c-31.551-19.866-60.581-45.283-85.264-70.423-14.718-16.666-28.732-50.827-39.083-75.045-15.513-58.226-37.776-159.15 22.532-235.63 3.823-4.372 7.948-11.65 11.46-13.09 17.918-12.12 37.375-20.38 58.298-24.966l-2.112-13.563c-10.45 2.374-45.633 15.243-55.581 20.629-22.623 6.558-41.388 13.406-70.22 20.625-9.373 1.146-18.592 1.167-27.58-.054-20.74-2.818-56.093-.3-58.348-2.396-14.044-19.585-17.89-54.676-30.703-73.456l-.152-.187-.162-.178c-7.47-10.04-16.177-18.23-24.752-26.75C54.652 291.5 28.37 262 21.012 230.74c-1.85-9.23-6.614-18.79-4.082-46.1l.047-.134.044-.133c7.388-25.513 19.407-46.31 39.806-67.153 21.228.342 42.24.809 58.607 3.935 7.51 1.256 23.124 3.296 39.252 9.304 40.822 15.207 94.76 40.209 94.76 40.209-40.488-22.236-85.76-51.115-114.35-56.943-4.253-.602-6.868-2.543-8.13-5.93 42.998-25.264 50.864-55.009 79.146-81.686 12.925-5.626 17.965-8.498 28.947-9.692 101.57 16.06 165.51 56.515 216.09 83.566 20.53 11.164 39.191 19.606 57.015 29.773 15.624 5.14 62.771 40.734 76.57 59.925 14.016 29.595 24.25 61.563 33.584 92.411 6.693 31.375 12.564 44.102 12.564 44.102s-5.688-26.243-4.74-30.958c5.898 2.204 19.84 6.558 25.62 5.881 0 0-25.817-13.346-29.2-25.14-10.856-37.848-21.744-96.766-24.68-99.838-8.274-10.41-42.777-36.816-63.941-49.03-8.006-4.62-12.333-7.486-12.671-9.518 6.76-6.83 15.11-15.865 22.598-21.606 7.177-5.503 13.71-11.709 23.713-15.423 43.963-19.8 69.1 7.618 75.066 2.066 0 0-9.458-10.841-5.257-9.056 4.302 2.33 18.323 5.078 19.862 6.509 16.023 12.534 57.913 58.344 83.16 106.68 6.065 11.84 8.53 19.636 5.717 33.893-2.82 14.27-5.001 22.117-8.077 31.647-2.779 6.371-18.498 49.988-18.437 55.848-3.167 23.932 10.264 53.893 10.264 53.893.127-8.133-.5-12.466.236-18.282l.904-10.393s-.573-2.764-.469-3.86c.682-7.187 2.445-13.229 2.963-17.345 5.047-31.255 13.822-53.755 23.775-81.182 2.958-6.922 6.828-10.771 6.63-16.041.163-9.352-8.205-21.904-14.252-34.163-6.1-12.39-13.39-26.202-22.96-40.76-21.82-31.376-40.42-55.925-74.48-71.289-9.53-4.182-47.03-8.27-59.98-5.841-15.71 3.273-29.22 6.5-39.99 13.398-16.95 10.854-30.27 27.659-45.89 37.56-34.56-17.281-51.24-30.215-54.35-31.998-20.54-11.006-45.2-23.778-71.73-35.89-12.72-11.862-91.71-40.233-164.04-45.892zm442.08 644.59c-21.3-16.65-39.23-33.79-51.05-51.44-3.858 20.758-17.864 35.542-28.688 50.083-2.155 3.41-3.708 8.06 6.902 22.879 2.86 3.948 13.207 4.623 20.192 4.26-7.106-5.371-17.918-11.052-19.66-15.976 12.501 8.51 24.076 10.957 34.538 9.586 2.39-.269 5.36-2.804 7.656-6.706 4.663-10.02 8.315-12.433 12.005-15.13l8.501 10.651z' color='%23000'/%3E%3C/g%3E%3C/svg%3E"); +} diff --git a/web/src/lib/icons/iconify-icons.js b/web/src/lib/icons/iconify-icons.js new file mode 100644 index 00000000000..c2498787117 --- /dev/null +++ b/web/src/lib/icons/iconify-icons.js @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +import fs from 'node:fs' +import path from 'node:path' + +import { cleanupSVG, importDirectory, runSVGO } from '@iconify/tools' +import { getIconsCSS } from '@iconify/utils' + +const SVGs = { + dir: 'src/lib/icons/svg', + prefix: 'custom-icons' +} + +const target = path.join(__dirname, 'iconify-icons.css') + +;(async () => { + const icons = [] + + const iconSet = await importDirectory(SVGs.dir, { + prefix: SVGs.prefix + }) + + await iconSet.forEach(async name => { + const svg = iconSet.toSVG(name) + cleanupSVG(svg) + runSVGO(svg) + iconSet.fromSVG(name, svg) + }) + + icons.push(iconSet.export()) + + await fs.promises.writeFile( + target, + ` +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +${icons + .map(iconSet => + getIconsCSS(iconSet, Object.keys(iconSet.icons), { + iconSelector: '.{prefix}-{name}' + }) + ) + .join('\n')} + `, + 'utf8' + ) +})() diff --git a/web/src/lib/icons/svg/doris.svg b/web/src/lib/icons/svg/doris.svg new file mode 100644 index 00000000000..7bd05ce60a8 --- /dev/null +++ b/web/src/lib/icons/svg/doris.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/web/src/lib/icons/svg/hive.svg b/web/src/lib/icons/svg/hive.svg new file mode 100644 index 00000000000..031ad38977a --- /dev/null +++ b/web/src/lib/icons/svg/hive.svg @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7acf29f7d1f25ca6acb472979fdc24d35fce8f95 Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 22 Apr 2024 21:05:34 +0800 Subject: [PATCH 096/106] [#3072] fix(spark-connector): fix local run SparkIcebergCatalogIT failed for non-utc timezone (#3074) ### What changes were proposed in this pull request? configurate spark time zone to UTC ### Why are the changes needed? if not specifying timezone, Iceberg partition path varies for different time zone in local machine. Fix: #3072 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? test in local machine --- .../integration/test/spark/SparkEnvIT.java | 1 + .../spark/iceberg/SparkIcebergCatalogIT.java | 4 +-- .../test/util/spark/SparkUtilIT.java | 31 +++++++++++++------ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkEnvIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkEnvIT.java index 096402e6121..d6d9acea313 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkEnvIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/SparkEnvIT.java @@ -147,6 +147,7 @@ private void initSparkEnv() { .config(GravitinoSparkConfig.GRAVITINO_METALAKE, metalakeName) .config("hive.exec.dynamic.partition.mode", "nonstrict") .config("spark.sql.warehouse.dir", warehouse) + .config("spark.sql.session.timeZone", TIME_ZONE_UTC) .enableHiveSupport() .getOrCreate(); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java index 27cc184ce6f..9ce15a12778 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/spark/iceberg/SparkIcebergCatalogIT.java @@ -188,12 +188,12 @@ void testIcebergPartitions() { String insertData = String.format( - "INSERT into %s values(2,'a',cast('2024-01-01 12:00:00.0' as timestamp));", + "INSERT into %s values(2,'a',cast('2024-01-01 12:00:00' as timestamp));", tableName); sql(insertData); List queryResult = getTableData(tableName); Assertions.assertEquals(1, queryResult.size()); - Assertions.assertEquals("2,a,2024-01-01 12:00:00.0", queryResult.get(0)); + Assertions.assertEquals("2,a,2024-01-01 12:00:00", queryResult.get(0)); String partitionExpression = partitionPaths.get(func); Path partitionPath = new Path(getTableLocation(tableInfo), partitionExpression); checkDirExists(partitionPath); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java index 94eb17d465e..a58ef78776f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/spark/SparkUtilIT.java @@ -21,10 +21,13 @@ import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.spark.connector.table.SparkBaseTable; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.spark.sql.AnalysisException; @@ -46,6 +49,8 @@ public abstract class SparkUtilIT extends AbstractIT { protected abstract SparkSession getSparkSession(); + protected final String TIME_ZONE_UTC = "UTC"; + protected Set getCatalogs() { return convertToStringSet(sql("SHOW CATALOGS"), 0); } @@ -89,21 +94,27 @@ protected List getTableData(String tableName) { return getQueryData(getSelectAllSql(tableName)); } + private String sparkObjectToString(Object item) { + if (item instanceof Object[]) { + return Arrays.stream((Object[]) item) + .map(i -> sparkObjectToString(i)) + .collect(Collectors.joining(",")); + } else if (item instanceof Timestamp) { + Timestamp timestamp = (Timestamp) item; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sdf.setTimeZone(TimeZone.getTimeZone(TIME_ZONE_UTC)); + return sdf.format(timestamp); + } else { + return item.toString(); + } + } + protected List getQueryData(String querySql) { return sql(querySql).stream() .map( line -> Arrays.stream(line) - .map( - item -> { - if (item instanceof Object[]) { - return Arrays.stream((Object[]) item) - .map(Object::toString) - .collect(Collectors.joining(",")); - } else { - return item.toString(); - } - }) + .map(item -> sparkObjectToString(item)) .collect(Collectors.joining(","))) .collect(Collectors.toList()); } From d169683326ac0ed416e5ac54bd400ed1458cd2cc Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 22 Apr 2024 21:26:14 +0800 Subject: [PATCH 097/106] [#3023] doc(core): add event listener document (#3021) ### What changes were proposed in this pull request? add event listener document ### Why are the changes needed? Fix: #3023 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? document --- docs/gravitino-server-config.md | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 90d0b7af57b..91a000d62de 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -88,6 +88,45 @@ Gravitino server uses tree lock to ensure the consistency of the data. The tree Refer to [Iceberg REST catalog service](iceberg-rest-service.md) for configuration details. +### Event listener configuration + +Gravitino provides event listener mechanism to allow users to capture the events which are provided by Gravitino server to integrate some custom operations. + +To leverage the event listener, you must implement the `EventListenerPlugin` interface and place the JAR file in the classpath of the Gravitino server. Then, add configurations to gravitino.conf to enable the event listener. + +| Property name | Description | Default value | Required | Since Version | +|--------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.eventListener.names` | The name of the event listener, For multiple listeners, separate names with a comma, like "audit,sync" | (none) | Yes | 0.5.0 | +| `gravitino.eventListener.{name}.className` | The class name of the event listener, replace `{name}` with the actual listener name. | (none) | Yes | 0.5.0 | +| `gravitino.eventListener.{name}.{key}` | Custom properties that will be passed to the event listener plugin. | (none) | Yes | 0.5.0 | + +#### Event + +Gravitino triggers a specific event upon the completion of the operation, with varying events being generated for different operations. + +| operation type | event | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| table operation | `CreateTableEvent`, `AlterTableEvent`, `DropTableEvent`, `LoadTableEvent`, `ListTableEvent`, `PurgeTableFailureEvent`, `CreateTableFailureEvent`, `AlterTableFailureEvent`, `DropTableFailureEvent`, `LoadTableFailureEvent`, `ListTableFailureEvent`, `PurgeTableFailureEvent` | +| fileset operation | `CreateFileSetEvent`, `AlterFileSetEvent`, `DropFileSetEvent`, `LoadFileSetEvent`, `ListFileSetEvent`, `CreateFileSetFailureEvent`, `AlterFileSetFailureEvent`, `DropFileSetFailureEvent`, `LoadFileSetFailureEvent`, `ListFileSetFailureEvent` | +| topic operation | `CreateTopicEvent`, `AlterTopicEvent`, `DropTopicEvent`, `LoadTopicEvent`, `ListTopicEvent`, `CreateTopicFailureEvent`, `AlterTopicFailureEvent`, `DropTopicFailureEvent`, `LoadTopicFailureEvent`, `ListTopicFailureEvent` | +| schema operation | `CreateSchemaEvent`, `AlterSchemaEvent`, `DropSchemaEvent`, `LoadSchemaEvent`, `ListSchemaEvent`, `CreateSchemaFailureEvent`, `AlterSchemaFailureEvent`, `DropSchemaFailureEvent`, `LoadSchemaFailureEvent`, `ListSchemaFailureEvent` | +| catalog operation | `CreateCatalogEvent`, `AlterCatalogEvent`, `DropCatalogEvent`, `LoadCatalogEvent`, `ListCatalogEvent`, `CreateCatalogFailureEvent`, `AlterCatalogFailureEvent`, `DropCatalogFailureEvent`, `LoadCatalogFailureEvent`, `ListCatalogFailureEvent` | +| metalake operation | `CreateMetalakeEvent`, `AlterMetalakeEvent`, `DropMetalakeEvent`, `LoadMetalakeEvent`, `ListMetalakeEvent`, `CreateMetalakeFailureEvent`, `AlterMetalakeFailureEvent`, `DropMetalakeFailureEvent`, `LoadMetalakeFailureEvent`, `ListMetalakeFailureEvent` | + +#### Event listener plugin + +The `EventListenerPlugin` defines an interface for event listeners that manage the lifecycle and state of a plugin. This includes handling its initialization, startup, and shutdown processes, as well as handing events triggered by various operations. + +The plugin provides several operational modes for how to process event, supporting both synchronous and asynchronous processing approaches. + +- **SYNC**: Events are processed synchronously, immediately following the associated operation. This mode ensures events are processed before the operation's result is returned to the client, but it may delay the main process if event processing takes too long. + +- **ASYNC_SHARED**: This mode employs a shared queue and dispatcher for asynchronous event processing. It prevents the main process from being blocked, though there's a risk events might be dropped if not promptly consumed. Sharing a dispatcher can lead to poor isolation in case of slow listeners. + +- **ASYNC_ISOLATED**: Events are processed asynchronously, with each listener having its own dedicated queue and dispatcher thread. This approach offers better isolation but at the expense of multiple queues and dispatchers. + +For more details, please refer to the definition of the plugin. + ### Security configuration Refer to [security](security.md) for HTTPS and authentication configurations. From 2a1ba00d5c46888d4b62ec1f7f9d3c276507553d Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Mon, 22 Apr 2024 21:46:00 +0800 Subject: [PATCH 098/106] [#2967] feat(api,server): Add the more privileges (#2965) ### What changes were proposed in this pull request? Add the more privileges and check the privilege constraint about securable object. ### Why are the changes needed? Fix: #2967 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add new UT. --------- Co-authored-by: Heng Qin --- .../gravitino/authorization/Privilege.java | 100 +- .../gravitino/authorization/Privileges.java | 954 +++++++++++++++++- .../authorization/SecurableObject.java | 82 +- .../authorization/SecurableObjects.java | 220 +++- .../authorization/TestSecurableObjects.java | 94 ++ .../gravitino/client/DTOConverters.java | 9 + .../client/GravitinoAdminClient.java | 2 +- .../datastrato/gravitino/client/TestRole.java | 47 +- .../gravitino/dto/authorization/RoleDTO.java | 13 +- .../dto/authorization/SecurableObjectDTO.java | 111 ++ .../dto/requests/RoleCreateRequest.java | 5 +- .../gravitino/dto/util/DTOConverters.java | 21 +- .../dto/responses/TestResponses.java | 5 +- .../gravitino/proto/RoleEntitySerDe.java | 9 +- .../TestAccessControlManager.java | 12 +- ...estAccessControlManagerForPermissions.java | 6 +- .../datastrato/gravitino/meta/TestEntity.java | 12 +- .../gravitino/proto/TestEntityProtoSerDe.java | 8 +- .../gravitino/storage/TestEntityStorage.java | 4 +- .../storage/relational/TestJDBCBackend.java | 4 +- meta/src/main/proto/gravitino_meta.proto | 7 +- .../server/web/rest/RoleOperations.java | 5 +- .../server/web/rest/TestRoleOperations.java | 19 +- 23 files changed, 1619 insertions(+), 130 deletions(-) create mode 100644 api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java index d27bde422e5..e33bd19eb70 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java @@ -4,13 +4,13 @@ */ package com.datastrato.gravitino.authorization; -import com.datastrato.gravitino.annotation.Evolving; +import com.datastrato.gravitino.annotation.Unstable; /** * The interface of a privilege. The privilege represents the ability to execute kinds of operations * for kinds of entities */ -@Evolving +@Unstable public interface Privilege { /** @return The generic name of the privilege. */ @@ -21,7 +21,99 @@ public interface Privilege { /** The name of this privilege. */ enum Name { - /** The privilege of load a catalog. */ - LOAD_CATALOG + /** The privilege to create a catalog. */ + CREATE_CATALOG(0L, 1L), + /** The privilege to drop a catalog. */ + DROP_CATALOG(0L, 1L << 1), + /** The privilege to alter a catalog. */ + ALTER_CATALOG(0L, 1L << 2), + /** The privilege to use a catalog. */ + USE_CATALOG(0L, 1L << 3), + /** The privilege to create a schema. */ + CREATE_SCHEMA(0L, 1L << 4), + /** The privilege to drop a schema. */ + DROP_SCHEMA(0L, 1L << 5), + /** The privilege to alter a schema. */ + ALTER_SCHEMA(0L, 1L << 6), + /** the privilege to use a schema. */ + USE_SCHEMA(0L, 1L << 7), + /** The privilege to create a table. */ + CREATE_TABLE(0L, 1L << 8), + /** The privilege to drop a table. */ + DROP_TABLE(0L, 1L << 9), + /** The privilege to write a table. */ + WRITE_TABLE(0L, 1L << 10), + /** The privilege to read a table. */ + READ_TABLE(0L, 1L << 11), + /** The privilege to create a fileset. */ + CREATE_FILESET(0L, 1L << 12), + /** The privilege to drop a fileset. */ + DROP_FILESET(0L, 1L << 13), + /** The privilege to write a fileset. */ + WRITE_FILESET(0L, 1L << 14), + /** The privilege to read a fileset. */ + READ_FILESET(0L, 1L << 15), + /** The privilege to create a topic. */ + CREATE_TOPIC(0L, 1L << 16), + /** The privilege to drop a topic. */ + DROP_TOPIC(0L, 1L << 17), + /** The privilege to write a topic. */ + WRITE_TOPIC(0L, 1L << 18), + /** The privilege to read a topic. */ + READ_TOPIC(0L, 1L << 19), + /** The privilege to create a metalake. */ + CREATE_METALAKE(0L, 1L << 20), + /** The privilege to manage a metalake, including drop and alter a metalake. */ + MANAGE_METALAKE(0L, 1L << 21), + /** The privilege to use a metalake, the user can load the information of the metalake. */ + USE_METALAKE(0L, 1L << 22), + /** The privilege to add a user */ + ADD_USER(0L, 1L << 23), + /** The privilege to remove a user */ + REMOVE_USER(0L, 1L << 24), + /** The privilege to get a user */ + GET_USER(0L, 1L << 25), + /** The privilege to add a group */ + ADD_GROUP(0L, 1L << 26), + /** The privilege to remove a group */ + REMOVE_GROUP(0L, 1L << 27), + /** The privilege to get a group */ + GET_GROUP(0L, 1L << 28), + /** The privilege to create a role */ + CREATE_ROLE(0L, 1L << 29), + /** The privilege to delete a role */ + DELETE_ROLE(0L, 1L << 30), + /** The privilege to grant a role to the user or the group. */ + GRANT_ROLE(0L, 1L << 31), + /** The privilege to revoke a role from the user or the group. */ + REVOKE_ROLE(0L, 1L << 32), + /** The privilege to get a role */ + GET_ROLE(0L, 1L << 33); + + private final long highBits; + private final long lowBits; + + Name(long highBits, long lowBits) { + this.highBits = highBits; + this.lowBits = lowBits; + } + + /** + * Return the low bits of Name + * + * @return The low bits of Name + */ + public long getLowBits() { + return lowBits; + } + + /** + * Return the high bits of Name + * + * @return The high bits of Name + */ + public long getHighBits() { + return highBits; + } } } diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java index 9eb4f3f9ef8..f91e83e33a8 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java @@ -4,6 +4,41 @@ */ package com.datastrato.gravitino.authorization; +import static com.datastrato.gravitino.authorization.Privilege.Name.ADD_GROUP; +import static com.datastrato.gravitino.authorization.Privilege.Name.ADD_USER; +import static com.datastrato.gravitino.authorization.Privilege.Name.ALTER_CATALOG; +import static com.datastrato.gravitino.authorization.Privilege.Name.ALTER_SCHEMA; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_CATALOG; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_FILESET; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_METALAKE; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_ROLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_SCHEMA; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_TABLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.CREATE_TOPIC; +import static com.datastrato.gravitino.authorization.Privilege.Name.DELETE_ROLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.DROP_CATALOG; +import static com.datastrato.gravitino.authorization.Privilege.Name.DROP_FILESET; +import static com.datastrato.gravitino.authorization.Privilege.Name.DROP_SCHEMA; +import static com.datastrato.gravitino.authorization.Privilege.Name.DROP_TABLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.DROP_TOPIC; +import static com.datastrato.gravitino.authorization.Privilege.Name.GET_GROUP; +import static com.datastrato.gravitino.authorization.Privilege.Name.GET_ROLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.GET_USER; +import static com.datastrato.gravitino.authorization.Privilege.Name.GRANT_ROLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.MANAGE_METALAKE; +import static com.datastrato.gravitino.authorization.Privilege.Name.READ_FILESET; +import static com.datastrato.gravitino.authorization.Privilege.Name.READ_TABLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.READ_TOPIC; +import static com.datastrato.gravitino.authorization.Privilege.Name.REMOVE_GROUP; +import static com.datastrato.gravitino.authorization.Privilege.Name.REMOVE_USER; +import static com.datastrato.gravitino.authorization.Privilege.Name.REVOKE_ROLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.USE_CATALOG; +import static com.datastrato.gravitino.authorization.Privilege.Name.USE_METALAKE; +import static com.datastrato.gravitino.authorization.Privilege.Name.USE_SCHEMA; +import static com.datastrato.gravitino.authorization.Privilege.Name.WRITE_FILESET; +import static com.datastrato.gravitino.authorization.Privilege.Name.WRITE_TABLE; +import static com.datastrato.gravitino.authorization.Privilege.Name.WRITE_TOPIC; + /** The helper class for {@link Privilege}. */ public class Privileges { @@ -26,34 +61,935 @@ public static Privilege fromString(String privilege) { */ public static Privilege fromName(Privilege.Name name) { switch (name) { - case LOAD_CATALOG: - return LoadCatalog.get(); + // Catalog + case CREATE_CATALOG: + return CreateCatalog.get(); + case DROP_CATALOG: + return DropCatalog.get(); + case ALTER_CATALOG: + return AlterCatalog.get(); + case USE_CATALOG: + return UseCatalog.get(); + + // Schema + case CREATE_SCHEMA: + return CreateSchema.get(); + case DROP_SCHEMA: + return DropSchema.get(); + case ALTER_SCHEMA: + return AlterSchema.get(); + case USE_SCHEMA: + return UseSchema.get(); + + // Table + case CREATE_TABLE: + return CreateTable.get(); + case DROP_TABLE: + return DropTable.get(); + case WRITE_TABLE: + return WriteTable.get(); + case READ_TABLE: + return ReadTable.get(); + + // Fileset + case CREATE_FILESET: + return CreateFileset.get(); + case DROP_FILESET: + return DropFileset.get(); + case WRITE_FILESET: + return WriteFileset.get(); + case READ_FILESET: + return ReadFileset.get(); + + // Topic + case CREATE_TOPIC: + return CreateTopic.get(); + case DROP_TOPIC: + return DropTopic.get(); + case WRITE_TOPIC: + return WriteTopic.get(); + case READ_TOPIC: + return ReadTopic.get(); + + // Metalake + case CREATE_METALAKE: + return CreateMetalake.get(); + case MANAGE_METALAKE: + return ManageMetalake.get(); + case USE_METALAKE: + return UseMetalake.get(); + + // User + case ADD_USER: + return AddUser.get(); + case REMOVE_USER: + return RemoveUser.get(); + case GET_USER: + return GetUser.get(); + + // Group + case ADD_GROUP: + return AddGroup.get(); + case REMOVE_GROUP: + return RemoveGroup.get(); + case GET_GROUP: + return GetGroup.get(); + + // Role + case CREATE_ROLE: + return CreateRole.get(); + case DELETE_ROLE: + return DeleteRole.get(); + case GRANT_ROLE: + return GrantRole.get(); + case REVOKE_ROLE: + return RevokeRole.get(); + case GET_ROLE: + return GetRole.get(); + default: throw new IllegalArgumentException("Don't support the privilege: " + name); } } - /** The privilege of load a catalog. */ - public static class LoadCatalog implements Privilege { - private static final LoadCatalog INSTANCE = new LoadCatalog(); + /** The privilege to create a catalog. */ + public static class CreateCatalog implements Privilege { + + private static final CreateCatalog INSTANCE = new CreateCatalog(); + + private CreateCatalog() {} + + /** @return The instance of the privilege. */ + public static CreateCatalog get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_CATALOG; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create catalog"; + } + } + + /** The privilege to alter a catalog. */ + public static class AlterCatalog implements Privilege { + + private static final AlterCatalog INSTANCE = new AlterCatalog(); + + private AlterCatalog() {} + + /** @return The instance of the privilege. */ + public static AlterCatalog get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return ALTER_CATALOG; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "alter catalog"; + } + } + + /** The privilege to drop a catalog. */ + public static class DropCatalog implements Privilege { + + private static final DropCatalog INSTANCE = new DropCatalog(); + + private DropCatalog() {} + + /** @return The instance of the privilege. */ + public static DropCatalog get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DROP_CATALOG; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "drop catalog"; + } + } + + /** The privilege to use a catalog. */ + public static class UseCatalog implements Privilege { + private static final UseCatalog INSTANCE = new UseCatalog(); + + /** @return The instance of the privilege. */ + public static UseCatalog get() { + return INSTANCE; + } + + private UseCatalog() {} + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return USE_CATALOG; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "use catalog"; + } + } + + /** The privilege to use a schema. */ + public static class UseSchema implements Privilege { + + private static final UseSchema INSTANCE = new UseSchema(); + + /** @return The instance of the privilege. */ + public static UseSchema get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return USE_SCHEMA; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "use schema"; + } + } + + /** The privilege to create a schema. */ + public static class CreateSchema implements Privilege { + + private static final CreateSchema INSTANCE = new CreateSchema(); + + /** @return The instance of the privilege. */ + public static CreateSchema get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_SCHEMA; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create schema"; + } + } + + /** The privilege to alter a schema. */ + public static class AlterSchema implements Privilege { + + private static final AlterSchema INSTANCE = new AlterSchema(); + + /** @return The instance of the privilege. */ + public static AlterSchema get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return ALTER_SCHEMA; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "alter schema"; + } + } + + /** The privilege to drop a schema. */ + public static class DropSchema implements Privilege { + + private static final DropSchema INSTANCE = new DropSchema(); + + /** @return The instance of the privilege. */ + public static DropSchema get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DROP_SCHEMA; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "drop schema"; + } + } + + /** The privilege to create a table. */ + public static class CreateTable implements Privilege { + + private static final CreateTable INSTANCE = new CreateTable(); + + private CreateTable() {} + + /** @return The instance of the privilege. */ + public static CreateTable get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_TABLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create table"; + } + } + + /** The privilege to drop a table. */ + public static class DropTable implements Privilege { + + private static final DropTable INSTANCE = new DropTable(); + + private DropTable() {} + + /** @return The instance of the privilege. */ + public static DropTable get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DROP_TABLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "drop table"; + } + } + + /** The privilege to read a table. */ + public static class ReadTable implements Privilege { + + private static final ReadTable INSTANCE = new ReadTable(); + + private ReadTable() {} + + /** @return The instance of the privilege. */ + public static ReadTable get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return READ_TABLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "read table"; + } + } + + /** The privilege to write a table. */ + public static class WriteTable implements Privilege { + + private static final WriteTable INSTANCE = new WriteTable(); + + private WriteTable() {} + + /** @return The instance of the privilege. */ + public static WriteTable get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return WRITE_TABLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "write table"; + } + } + + /** The privilege to create a fileset. */ + public static class CreateFileset implements Privilege { + + private static final CreateFileset INSTANCE = new CreateFileset(); + + private CreateFileset() {} + + /** @return The instance of the privilege. */ + public static CreateFileset get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_FILESET; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create fileset"; + } + } + + /** The privilege to drop a fileset. */ + public static class DropFileset implements Privilege { + + private static final DropFileset INSTANCE = new DropFileset(); + + private DropFileset() {} /** @return The instance of the privilege. */ - public static LoadCatalog get() { + public static DropFileset get() { return INSTANCE; } - private LoadCatalog() {} + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DROP_FILESET; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "drop fileset"; + } + } + + /** The privilege to read a fileset. */ + public static class ReadFileset implements Privilege { + + private static final ReadFileset INSTANCE = new ReadFileset(); + + private ReadFileset() {} + + /** @return The instance of the privilege. */ + public static ReadFileset get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return READ_FILESET; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "read fileset"; + } + } + + /** The privilege to write a fileset. */ + public static class WriteFileset implements Privilege { + + private static final WriteFileset INSTANCE = new WriteFileset(); + + private WriteFileset() {} + + /** @return The instance of the privilege. */ + public static WriteFileset get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return WRITE_FILESET; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "write fileset"; + } + } + + /** The privilege to create a topic. */ + public static class CreateTopic implements Privilege { + + private static final CreateTopic INSTANCE = new CreateTopic(); + + private CreateTopic() {} + + /** @return The instance of the privilege. */ + public static CreateTopic get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_TOPIC; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create topic"; + } + } + + /** The privilege to drop a topic. */ + public static class DropTopic implements Privilege { + + private static final DropTopic INSTANCE = new DropTopic(); + + private DropTopic() {} + + /** @return The instance of the privilege. */ + public static DropTopic get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DROP_TOPIC; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "drop topic"; + } + } + + /** The privilege to read a topic. */ + public static class ReadTopic implements Privilege { + + private static final ReadTopic INSTANCE = new ReadTopic(); + + private ReadTopic() {} + + /** @return The instance of the privilege. */ + public static ReadTopic get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return READ_TOPIC; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "read topic"; + } + } + + /** The privilege to write a topic. */ + public static class WriteTopic implements Privilege { + + private static final WriteTopic INSTANCE = new WriteTopic(); + + private WriteTopic() {} + + /** @return The instance of the privilege. */ + public static WriteTopic get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return WRITE_TOPIC; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "write topic"; + } + } + + /** The privilege to manage a metalake. */ + public static class ManageMetalake implements Privilege { + + private static final ManageMetalake INSTANCE = new ManageMetalake(); + + private ManageMetalake() {} + + /** @return The instance of the privilege. */ + public static ManageMetalake get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return MANAGE_METALAKE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "manage metalake"; + } + } + + /** The privilege to manage a metalake. */ + public static class CreateMetalake implements Privilege { + + private static final CreateMetalake INSTANCE = new CreateMetalake(); + + private CreateMetalake() {} + + /** @return The instance of the privilege. */ + public static CreateMetalake get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_METALAKE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create metalake"; + } + } + + /** The privilege to use a metalake. */ + public static class UseMetalake implements Privilege { + + private static final UseMetalake INSTANCE = new UseMetalake(); + + private UseMetalake() {} + + /** @return The instance of the privilege. */ + public static UseMetalake get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return USE_METALAKE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "use metalake"; + } + } + + /** The privilege to get a user. */ + public static class GetUser implements Privilege { + + private static final GetUser INSTANCE = new GetUser(); + + private GetUser() {} + + /** @return The instance of the privilege. */ + public static GetUser get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return GET_USER; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "get user"; + } + } + + /** The privilege to add a user. */ + public static class AddUser implements Privilege { + + private static final AddUser INSTANCE = new AddUser(); + + private AddUser() {} + + /** @return The instance of the privilege. */ + public static AddUser get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return ADD_USER; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "add user"; + } + } + + /** The privilege to remove a user. */ + public static class RemoveUser implements Privilege { + + private static final RemoveUser INSTANCE = new RemoveUser(); + + private RemoveUser() {} + + /** @return The instance of the privilege. */ + public static RemoveUser get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return REMOVE_USER; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "remove user"; + } + } + + /** The privilege to add a group. */ + public static class AddGroup implements Privilege { + + private static final AddGroup INSTANCE = new AddGroup(); + + private AddGroup() {} + + /** @return The instance of the privilege. */ + public static AddGroup get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return ADD_GROUP; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "add group"; + } + } + + /** The privilege to remove a group. */ + public static class RemoveGroup implements Privilege { + + private static final RemoveGroup INSTANCE = new RemoveGroup(); + + private RemoveGroup() {} + + /** @return The instance of the privilege. */ + public static RemoveGroup get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return REMOVE_GROUP; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "remove group"; + } + } + + /** The privilege to get a group. */ + public static class GetGroup implements Privilege { + + private static final GetGroup INSTANCE = new GetGroup(); + + private GetGroup() {} + + /** @return The instance of the privilege. */ + public static GetGroup get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return GET_GROUP; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "get group"; + } + } + + /** The privilege to create a role. */ + public static class CreateRole implements Privilege { + + private static final CreateRole INSTANCE = new CreateRole(); + + private CreateRole() {} + + /** @return The instance of the privilege. */ + public static CreateRole get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return CREATE_ROLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "create role"; + } + } + + /** The privilege to get a role. */ + public static class GetRole implements Privilege { + + private static final GetRole INSTANCE = new GetRole(); + + private GetRole() {} + + /** @return The instance of the privilege. */ + public static GetRole get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return GET_ROLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "get role"; + } + } + + /** The privilege to delete a role. */ + public static class DeleteRole implements Privilege { + + private static final DeleteRole INSTANCE = new DeleteRole(); + + private DeleteRole() {} + + /** @return The instance of the privilege. */ + public static DeleteRole get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return DELETE_ROLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "delete role"; + } + } + + /** The privilege to grant a role to the user or the group. */ + public static class GrantRole implements Privilege { + + private static final GrantRole INSTANCE = new GrantRole(); + + private GrantRole() {} + + /** @return The instance of the privilege. */ + public static GrantRole get() { + return INSTANCE; + } + + /** @return The generic name of the privilege. */ + @Override + public Name name() { + return GRANT_ROLE; + } + + /** @return A readable string representation for the privilege. */ + @Override + public String simpleString() { + return "grant role"; + } + } + + /** The privilege to revoke a role from the user or the group. */ + public static class RevokeRole implements Privilege { + + private static final RevokeRole INSTANCE = new RevokeRole(); + + private RevokeRole() {} + + /** @return The instance of the privilege. */ + public static RevokeRole get() { + return INSTANCE; + } /** @return The generic name of the privilege. */ @Override public Name name() { - return Name.LOAD_CATALOG; + return REVOKE_ROLE; } /** @return A readable string representation for the privilege. */ @Override public String simpleString() { - return "load catalog"; + return "revoke role"; } } } diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java index b3d8aa5ce2c..e045efa3817 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java @@ -4,21 +4,42 @@ */ package com.datastrato.gravitino.authorization; -import com.datastrato.gravitino.annotation.Evolving; +import com.datastrato.gravitino.annotation.Unstable; import javax.annotation.Nullable; /** * The securable object is the entity which access can be granted. Unless allowed by a grant, access - * is denied. Gravitino organizes the securable objects using tree structure. The securable object - * may be a catalog, a table or a schema, etc. For example, `catalog1.schema1.table1` represents a - * table named `table1`. It's in the schema named `schema1`. The schema is in the catalog named - * `catalog1`. Similarly, `catalog1.schema1.topic1` can represent a topic. - * `catalog1.schema1.fileset1` can represent a fileset. `*` represents all the catalogs. If you want - * to use other securable objects which represents all entities," you can use their parent entity, + * is denied. Gravitino organizes the securable objects using tree structure.
+ * There are three fields in the securable object: parent, name, and type.
+ * The types include 6 kinds: CATALOG,SCHEMA,TABLE,FILESET,TOPIC and METALAKE.
+ * You can use the helper class `SecurableObjects` to create the securable object which you need. + *
+ * You can use full name and type of the securable object in the RESTFUL API.
+ * For example,
+ * If you want to use a catalog named `catalog1`, you can use the code + * `SecurableObjects.ofCatalog("catalog1")` to create the securable object, or you can use full name + * `catalog1` and type `CATALOG` in the RESTFUL API.
+ * If you want to use a schema named `schema1` in the catalog named `catalog1`, you can use the code + * `SecurableObjects.ofSchema(catalog, "schema1")` to create the securable object, or you can use + * full name `catalog1.schema1` and type `SCHEMA` in the RESTFUL API.
+ * If you want to use a table named `table1` in the schema named `schema1`, you can use the code + * `SecurableObjects.ofTable(schema, "table1")` to create the securable object, or you can use full + * name `catalog1.schema1.table1` and type `TABLE` in the RESTFUL API.
+ * If you want to use a topic named `topic1` in the schema named `schema1`, you can use the code + * `SecurableObjects.ofTopic(schema, "topic1")` to create the securable object, or you can use full + * name `catalog1.schema1.topic1` and type `TOPIC` in the RESTFUL API.
+ * If you want to use a fileset named `fileset1` in the schema named `schema1`, you can use the code + * `SecurableObjects.ofFileset(schema, "fileset1)` to create the securable object, or you can use + * full name `catalog1.schema1.fileset1` and type `FILESET` in the RESTFUL API.
+ * If you want to use a metalake named `metalake1`, you can use the code + * `SecurableObjects.ofMetalake("metalake1")` to create the securable object, or you can use full + * name `metalake1` and type `METALAKE` in the RESTFUL API.
+ * If you want to use all the catalogs, you use the metalake to represent them. Likely, you can use + * their common parent to represent all securable objects.
* For example if you want to have read table privileges of all tables of `catalog1.schema1`, " you * can use add `read table` privilege for `catalog1.schema1` directly */ -@Evolving +@Unstable public interface SecurableObject { /** @@ -36,4 +57,49 @@ public interface SecurableObject { * @return The name of the securable object. */ String name(); + + /** + * The full name of th securable object. If the parent isn't null, the full name will join the + * parent full name and the name with `.`, otherwise will return the name. + * + * @return The name of the securable object. + */ + String fullName(); + + /** + * The type of securable object + * + * @return The type of securable object. + */ + Type type(); + + /** + * The type of securable object in the Gravitino system. Every type will map one kind of the + * entity of the underlying system. + */ + enum Type { + /** + * A catalog is a collection of metadata from a specific metadata source, like Apache Hive + * catalog, Apache Iceberg catalog, JDBC catalog, etc. + */ + CATALOG, + /** + * A schema is a sub collection of the catalog. The schema can contain filesets, tables, topics, + * etc. + */ + SCHEMA, + /** A fileset is mapped to a directory on a file system like HDFS, S3, ADLS, GCS, etc. */ + FILESET, + /** A table is mapped the table of relational data sources like Apache Hive, MySQL, etc. */ + TABLE, + /** + * A topic is mapped the topic of messaging data sources like Apache Kafka, Apache Pulsar, etc. + */ + TOPIC, + /** + * A metalake is a concept of tenant. It means an organization. A metalake contains many data + * sources. + */ + METALAKE + } } diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java index 80d68836b84..2c9d99bde72 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java @@ -6,6 +6,9 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; import java.util.Objects; import org.apache.commons.lang3.StringUtils; @@ -15,40 +18,15 @@ public class SecurableObjects { private static final Splitter DOT = Splitter.on('.'); /** - * Create the {@link SecurableObject} with the given names. + * Create the metalake {@link SecurableObject} with the given metalake name. * - * @param names The names of the securable object. - * @return The created {@link SecurableObject} + * @param metalake The metalake name + * @return The created metalake {@link SecurableObject} */ - public static SecurableObject of(String... names) { - if (names == null) { - throw new IllegalArgumentException("Cannot create a securable object with null names"); - } - - if (names.length == 0) { - throw new IllegalArgumentException("Cannot create a securable object with no names"); - } - - SecurableObject parent = null; - for (String name : names) { - if (name == null) { - throw new IllegalArgumentException("Cannot create a securable object with null name"); - } + public static SecurableObject ofMetalake(String metalake) { + checkName(metalake); - if (name.equals("*")) { - throw new IllegalArgumentException( - "Cannot create a securable object with `*` name. If you want to use a securable object which represents all catalogs," - + " you use the method `ofAllCatalogs`." - + " If you want to create an another securable object which represents all entities," - + " you can use its parent entity, For example," - + " if you want to have read table privileges of all tables of `catalog1.schema1`," - + " you can use add `read table` privilege for `catalog1.schema1` directly"); - } - - parent = new SecurableObjectImpl(parent, name); - } - - return parent; + return new SecurableObjectImpl(null, metalake, SecurableObject.Type.METALAKE); } /** @@ -58,7 +36,9 @@ public static SecurableObject of(String... names) { * @return The created catalog {@link SecurableObject} */ public static SecurableObject ofCatalog(String catalog) { - return of(catalog); + checkName(catalog); + + return new SecurableObjectImpl(null, catalog, SecurableObject.Type.CATALOG); } /** @@ -71,8 +51,9 @@ public static SecurableObject ofCatalog(String catalog) { */ public static SecurableObject ofSchema(SecurableObject catalog, String schema) { checkCatalog(catalog); + checkName(schema); - return of(catalog.name(), schema); + return new SecurableObjectImpl(catalog, schema, SecurableObject.Type.SCHEMA); } /** @@ -84,8 +65,9 @@ public static SecurableObject ofSchema(SecurableObject catalog, String schema) { */ public static SecurableObject ofTable(SecurableObject schema, String table) { checkSchema(schema); + checkName(table); - return of(schema.parent().name(), schema.name(), table); + return new SecurableObjectImpl(schema, table, SecurableObject.Type.TABLE); } /** @@ -97,8 +79,9 @@ public static SecurableObject ofTable(SecurableObject schema, String table) { */ public static SecurableObject ofTopic(SecurableObject schema, String topic) { checkSchema(schema); + checkName(topic); - return of(schema.parent().name(), schema.name(), topic); + return new SecurableObjectImpl(schema, topic, SecurableObject.Type.TOPIC); } /** @@ -111,25 +94,32 @@ public static SecurableObject ofTopic(SecurableObject schema, String topic) { */ public static SecurableObject ofFileset(SecurableObject schema, String fileset) { checkSchema(schema); + checkName(fileset); - return of(schema.parent().name(), schema.name(), fileset); + return new SecurableObjectImpl(schema, fileset, SecurableObject.Type.FILESET); } /** - * All catalogs is a special securable object .You can give the securable object the privileges - * `LOAD CATALOG`, `CREATE CATALOG`, etc. It means that you can load any catalog and create any - * which doesn't exist. + * All metalakes is a special securable object .You can give the securable object the privileges + * `CREATE METALAKE`, etc. It means that you can create any which doesn't exist. This securable + * object is only used for metalake admin. You can't grant any privilege to this securable object. + * You can't bind this securable object to any role, too. * * @return The created {@link SecurableObject} */ - public static SecurableObject ofAllCatalogs() { - return ALL_CATALOGS; + public static SecurableObject ofAllMetalakes() { + return ALL_METALAKES; } private static void checkSchema(SecurableObject schema) { if (schema == null) { throw new IllegalArgumentException("Securable schema object can't be null"); } + + if (schema.type() != SecurableObject.Type.SCHEMA) { + throw new IllegalArgumentException("Securable schema object type must be SCHEMA"); + } + checkCatalog(schema.parent()); } @@ -138,22 +128,29 @@ private static void checkCatalog(SecurableObject catalog) { throw new IllegalArgumentException("Securable catalog object can't be null"); } + if (catalog.type() != SecurableObject.Type.CATALOG) { + throw new IllegalArgumentException("Securable catalog type must be CATALOG"); + } + if (catalog.parent() != null) { throw new IllegalArgumentException( String.format("The parent of securable catalog object %s must be null", catalog.name())); } } - private static final SecurableObject ALL_CATALOGS = new SecurableObjectImpl(null, "*"); + private static final SecurableObject ALL_METALAKES = + new SecurableObjectImpl(null, "*", SecurableObject.Type.METALAKE); private static class SecurableObjectImpl implements SecurableObject { private final SecurableObject parent; private final String name; + private final Type type; - SecurableObjectImpl(SecurableObject parent, String name) { + SecurableObjectImpl(SecurableObject parent, String name, Type type) { this.parent = parent; this.name = name; + this.type = type; } @Override @@ -166,9 +163,19 @@ public String name() { return name; } + @Override + public String fullName() { + return toString(); + } + + @Override + public Type type() { + return type; + } + @Override public int hashCode() { - return Objects.hash(parent, name); + return Objects.hash(parent, name, type); } @Override @@ -188,26 +195,133 @@ public boolean equals(Object other) { SecurableObject otherSecurableObject = (SecurableObject) other; return Objects.equals(parent, otherSecurableObject.parent()) - && Objects.equals(name, otherSecurableObject.name()); + && Objects.equals(name, otherSecurableObject.name()) + && Objects.equals(type, otherSecurableObject.type()); } } /** - * Create a {@link SecurableObject} from the given identifier string. + * Create a {@link SecurableObject} from the given full name. * - * @param securableObjectIdentifier The identifier string + * @param fullName The full name of securable object. + * @param type The securable object type. * @return The created {@link SecurableObject} */ - public static SecurableObject parse(String securableObjectIdentifier) { - if ("*".equals(securableObjectIdentifier)) { - return SecurableObjects.ofAllCatalogs(); + public static SecurableObject parse(String fullName, SecurableObject.Type type) { + if ("*".equals(fullName)) { + if (type != SecurableObject.Type.METALAKE) { + throw new IllegalArgumentException("If securable object isn't metalake, it can't be `*`"); + } + return SecurableObjects.ofAllMetalakes(); } - if (StringUtils.isBlank(securableObjectIdentifier)) { + if (StringUtils.isBlank(fullName)) { throw new IllegalArgumentException("securable object identifier can't be blank"); } - Iterable parts = DOT.split(securableObjectIdentifier); - return SecurableObjects.of(Iterables.toArray(parts, String.class)); + Iterable parts = DOT.split(fullName); + return SecurableObjects.of(type, Iterables.toArray(parts, String.class)); + } + + /** + * Create the {@link SecurableObject} with the given names. + * + * @param type The securable object type. + * @param names The names of the securable object. + * @return The created {@link SecurableObject} + */ + static SecurableObject of(SecurableObject.Type type, String... names) { + + if (names == null) { + throw new IllegalArgumentException("Cannot create a securable object with null names"); + } + + if (names.length == 0) { + throw new IllegalArgumentException("Cannot create a securable object with no names"); + } + + if (type == null) { + throw new IllegalArgumentException("Cannot create a securable object with no type"); + } + + if (names.length > 3) { + throw new IllegalArgumentException( + "Cannot create a securable object with the name length which is greater than 3"); + } + + if (names.length == 1 + && type != SecurableObject.Type.CATALOG + && type != SecurableObject.Type.METALAKE) { + throw new IllegalArgumentException( + "If the length of names is 1, it must be the CATALOG or METALAKE type"); + } + + if (names.length == 2 && type != SecurableObject.Type.SCHEMA) { + throw new IllegalArgumentException("If the length of names is 2, it must be the SCHEMA type"); + } + + if (names.length == 3 + && type != SecurableObject.Type.FILESET + && type != SecurableObject.Type.TABLE + && type != SecurableObject.Type.TOPIC) { + throw new IllegalArgumentException( + "If the length of names is 3, it must be FILESET, TABLE or TOPIC"); + } + + List types = Lists.newArrayList(type); + + // Find all the types of the parent securable object. + SecurableObject.Type curType = type; + List reverseNames = Lists.newArrayList(names); + Collections.reverse(reverseNames); + for (String ignored : reverseNames) { + types.add(curType); + curType = getParentSecurableObjectType(curType); + } + Collections.reverse(types); + + SecurableObject parent = null; + int level = 0; + for (String name : names) { + checkName(name); + + parent = new SecurableObjectImpl(parent, name, types.get(level)); + + level++; + } + + return parent; + } + + private static SecurableObject.Type getParentSecurableObjectType(SecurableObject.Type type) { + switch (type) { + case FILESET: + return SecurableObject.Type.SCHEMA; + + case TOPIC: + return SecurableObject.Type.SCHEMA; + + case TABLE: + return SecurableObject.Type.SCHEMA; + + case SCHEMA: + return SecurableObject.Type.CATALOG; + + case CATALOG: + return SecurableObject.Type.METALAKE; + + default: + return null; + } + } + + private static void checkName(String name) { + if (name == null) { + throw new IllegalArgumentException("Cannot create a securable object with null name"); + } + + if ("*".equals(name)) { + throw new IllegalArgumentException("Cannot create a securable object with `*` name."); + } } } diff --git a/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java b/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java new file mode 100644 index 00000000000..7697c896f3d --- /dev/null +++ b/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java @@ -0,0 +1,94 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.authorization; + +import static com.datastrato.gravitino.authorization.SecurableObject.Type.CATALOG; +import static com.datastrato.gravitino.authorization.SecurableObject.Type.FILESET; +import static com.datastrato.gravitino.authorization.SecurableObject.Type.METALAKE; +import static com.datastrato.gravitino.authorization.SecurableObject.Type.SCHEMA; +import static com.datastrato.gravitino.authorization.SecurableObject.Type.TABLE; +import static com.datastrato.gravitino.authorization.SecurableObject.Type.TOPIC; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestSecurableObjects { + + @Test + public void testSecurableObjects() { + SecurableObject allMetalakes = SecurableObjects.ofAllMetalakes(); + Assertions.assertEquals("*", allMetalakes.fullName()); + Assertions.assertEquals(METALAKE, allMetalakes.type()); + Assertions.assertThrows( + IllegalArgumentException.class, () -> SecurableObjects.of(METALAKE, "*")); + + SecurableObject metalake = SecurableObjects.ofMetalake("metalake"); + Assertions.assertEquals("metalake", metalake.fullName()); + Assertions.assertEquals(METALAKE, metalake.type()); + SecurableObject anotherMetalake = SecurableObjects.of(METALAKE, "metalake"); + Assertions.assertEquals(metalake, anotherMetalake); + + SecurableObject catalog = SecurableObjects.ofCatalog("catalog"); + Assertions.assertEquals("catalog", catalog.fullName()); + Assertions.assertEquals(CATALOG, catalog.type()); + SecurableObject anotherCatalog = SecurableObjects.of(CATALOG, "catalog"); + Assertions.assertEquals(catalog, anotherCatalog); + + SecurableObject schema = SecurableObjects.ofSchema(catalog, "schema"); + Assertions.assertEquals("catalog.schema", schema.fullName()); + Assertions.assertEquals(SCHEMA, schema.type()); + SecurableObject anotherSchema = SecurableObjects.of(SCHEMA, "catalog", "schema"); + Assertions.assertEquals(schema, anotherSchema); + + SecurableObject table = SecurableObjects.ofTable(schema, "table"); + Assertions.assertEquals("catalog.schema.table", table.fullName()); + Assertions.assertEquals(TABLE, table.type()); + SecurableObject anotherTable = SecurableObjects.of(TABLE, "catalog", "schema", "table"); + Assertions.assertEquals(table, anotherTable); + + SecurableObject fileset = SecurableObjects.ofFileset(schema, "fileset"); + Assertions.assertEquals("catalog.schema.fileset", fileset.fullName()); + Assertions.assertEquals(FILESET, fileset.type()); + SecurableObject anotherFileset = SecurableObjects.of(FILESET, "catalog", "schema", "fileset"); + Assertions.assertEquals(fileset, anotherFileset); + + SecurableObject topic = SecurableObjects.ofTopic(schema, "topic"); + Assertions.assertEquals("catalog.schema.topic", topic.fullName()); + Assertions.assertEquals(TOPIC, topic.type()); + + SecurableObject anotherTopic = SecurableObjects.of(TOPIC, "catalog", "schema", "topic"); + Assertions.assertEquals(topic, anotherTopic); + + Exception e = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SecurableObjects.of(METALAKE, "metalake", "catalog")); + Assertions.assertTrue(e.getMessage().contains("length of names is 2")); + e = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SecurableObjects.of(CATALOG, "metalake", "catalog")); + Assertions.assertTrue(e.getMessage().contains("length of names is 2")); + + e = + Assertions.assertThrows( + IllegalArgumentException.class, () -> SecurableObjects.of(TABLE, "metalake")); + Assertions.assertTrue(e.getMessage().contains("the length of names is 1")); + e = + Assertions.assertThrows( + IllegalArgumentException.class, () -> SecurableObjects.of(TOPIC, "metalake")); + Assertions.assertTrue(e.getMessage().contains("the length of names is 1")); + e = + Assertions.assertThrows( + IllegalArgumentException.class, () -> SecurableObjects.of(FILESET, "metalake")); + Assertions.assertTrue(e.getMessage().contains("the length of names is 1")); + + e = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SecurableObjects.of(SCHEMA, "catalog", "schema", "table")); + Assertions.assertTrue(e.getMessage().contains("the length of names is 3")); + } +} diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java index c109c1a53e4..c6dc1651032 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java @@ -9,9 +9,11 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.MetalakeDTO; +import com.datastrato.gravitino.dto.authorization.SecurableObjectDTO; import com.datastrato.gravitino.dto.requests.CatalogUpdateRequest; import com.datastrato.gravitino.dto.requests.FilesetUpdateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdateRequest; @@ -271,4 +273,11 @@ private static TableUpdateRequest toColumnUpdateRequest(TableChange.ColumnChange "Unknown column change type: " + change.getClass().getSimpleName()); } } + + static SecurableObjectDTO toSecurableObject(SecurableObject securableObject) { + return SecurableObjectDTO.builder() + .withFullName(securableObject.fullName()) + .withType(securableObject.type()) + .build(); + } } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index 4cea0f062d8..aaa15cf41a9 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -451,7 +451,7 @@ public Role createRole( .map(Privilege::name) .map(Objects::toString) .collect(Collectors.toList()), - securableObject.toString()); + DTOConverters.toSecurableObject(securableObject)); req.validate(); RoleResponse resp = diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java index 560bb52e498..97bbcd01833 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java @@ -11,9 +11,11 @@ import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.authorization.RoleDTO; +import com.datastrato.gravitino.dto.authorization.SecurableObjectDTO; import com.datastrato.gravitino.dto.requests.RoleCreateRequest; import com.datastrato.gravitino.dto.responses.DeleteResponse; import com.datastrato.gravitino.dto.responses.ErrorResponse; @@ -45,7 +47,13 @@ public void testCreateRoles() throws Exception { String rolePath = withSlash(String.format(API_METALAKES_ROLES_PATH, metalakeName, "")); RoleCreateRequest request = new RoleCreateRequest( - roleName, ImmutableMap.of("k1", "v1"), Lists.newArrayList("LOAD_CATALOG"), "catalog"); + roleName, + ImmutableMap.of("k1", "v1"), + Lists.newArrayList("USE_CATALOG"), + SecurableObjectDTO.builder() + .withFullName("catalog") + .withType(SecurableObject.Type.CATALOG) + .build()); RoleDTO mockRole = mockRoleDTO(roleName); RoleResponse roleResponse = new RoleResponse(mockRole); @@ -57,7 +65,9 @@ public void testCreateRoles() throws Exception { roleName, ImmutableMap.of("k1", "v1"), SecurableObjects.ofCatalog("catalog"), - Lists.newArrayList(Privileges.LoadCatalog.get())); + Lists.newArrayList(Privileges.UseCatalog.get())); + Assertions.assertEquals(1L, Privileges.CreateCatalog.get().name().getLowBits()); + Assertions.assertEquals(0L, Privileges.CreateCatalog.get().name().getHighBits()); Assertions.assertNotNull(createdRole); assertRole(createdRole, mockRole); @@ -75,7 +85,7 @@ public void testCreateRoles() throws Exception { roleName, ImmutableMap.of("k1", "v1"), SecurableObjects.ofCatalog("catalog"), - Lists.newArrayList(Privileges.LoadCatalog.get()))); + Lists.newArrayList(Privileges.UseCatalog.get()))); Assertions.assertEquals("role already exists", ex.getMessage()); // test NoSuchMetalakeException @@ -91,7 +101,7 @@ public void testCreateRoles() throws Exception { roleName, ImmutableMap.of("k1", "v1"), SecurableObjects.ofCatalog("catalog"), - Lists.newArrayList(Privileges.LoadCatalog.get()))); + Lists.newArrayList(Privileges.UseCatalog.get()))); Assertions.assertEquals("metalake not found", ex.getMessage()); // test RuntimeException @@ -105,7 +115,7 @@ public void testCreateRoles() throws Exception { roleName, ImmutableMap.of("k1", "v1"), SecurableObjects.ofCatalog("catalog"), - Lists.newArrayList(Privileges.LoadCatalog.get())), + Lists.newArrayList(Privileges.UseCatalog.get())), "internal error"); } @@ -145,6 +155,15 @@ public void testGetRoles() throws Exception { buildMockResource(Method.GET, rolePath, null, errResp3, SC_SERVER_ERROR); Assertions.assertThrows( RuntimeException.class, () -> client.getRole(metalakeName, roleName), "internal error"); + + // test SecurableDTO use parent method + Role testParentRole = mockHasParentRoleDTO("test"); + Assertions.assertEquals("schema", testParentRole.securableObject().name()); + Assertions.assertEquals(SecurableObject.Type.SCHEMA, testParentRole.securableObject().type()); + Assertions.assertEquals("catalog", testParentRole.securableObject().parent().fullName()); + Assertions.assertEquals("catalog", testParentRole.securableObject().parent().name()); + Assertions.assertEquals( + SecurableObject.Type.CATALOG, testParentRole.securableObject().parent().type()); } @Test @@ -172,8 +191,20 @@ private RoleDTO mockRoleDTO(String name) { return RoleDTO.builder() .withName(name) .withProperties(ImmutableMap.of("k1", "v1")) - .withSecurableObject(SecurableObjects.of("catalog")) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(DTOConverters.toSecurableObject(SecurableObjects.ofCatalog("catalog"))) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) + .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .build(); + } + + private RoleDTO mockHasParentRoleDTO(String name) { + SecurableObject catalog = SecurableObjects.ofCatalog("catalog"); + return RoleDTO.builder() + .withName(name) + .withProperties(ImmutableMap.of("k1", "v1")) + .withSecurableObject( + DTOConverters.toSecurableObject(SecurableObjects.ofSchema(catalog, "schema"))) + .withPrivileges(Lists.newArrayList(Privileges.UseSchema.get())) .withAudit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); } @@ -182,6 +213,6 @@ private void assertRole(Role expected, Role actual) { Assertions.assertEquals(expected.name(), actual.name()); Assertions.assertEquals(expected.privileges(), actual.privileges()); Assertions.assertEquals( - expected.securableObject().toString(), actual.securableObject().toString()); + expected.securableObject().fullName(), actual.securableObject().fullName()); } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java index 2f50da3a353..6a9c24fa3a1 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.Role; import com.datastrato.gravitino.authorization.SecurableObject; -import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.dto.AuditDTO; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; @@ -39,7 +38,7 @@ public class RoleDTO implements Role { private List privileges; @JsonProperty("securableObject") - private String securableObject; + private SecurableObjectDTO securableObject; /** Default constructor for Jackson deserialization. */ protected RoleDTO() {} @@ -57,7 +56,7 @@ protected RoleDTO( String name, Map properties, List privileges, - String securableObject, + SecurableObjectDTO securableObject, AuditDTO audit) { this.name = name; this.audit = audit; @@ -103,7 +102,7 @@ public List privileges() { */ @Override public SecurableObject securableObject() { - return SecurableObjects.parse(securableObject); + return securableObject; } /** @return The audit information of the Role DTO. */ @@ -141,7 +140,7 @@ public static class Builder { protected Map properties; /** The securable object of the role. */ - protected SecurableObject securableObject; + protected SecurableObjectDTO securableObject; /** * Sets the name of the role. @@ -188,7 +187,7 @@ public S withProperties(Map properties) { * @param securableObject The securableObject of the role. * @return The builder instance. */ - public S withSecurableObject(SecurableObject securableObject) { + public S withSecurableObject(SecurableObjectDTO securableObject) { this.securableObject = securableObject; return (S) this; } @@ -224,7 +223,7 @@ public RoleDTO build() { .map(Privilege::name) .map(Objects::toString) .collect(Collectors.toList()), - securableObject.toString(), + securableObject, audit); } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java new file mode 100644 index 00000000000..abde242dd77 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java @@ -0,0 +1,111 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.authorization; + +import com.datastrato.gravitino.authorization.SecurableObject; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; + +/** Data transfer object representing a securable object. */ +public class SecurableObjectDTO implements SecurableObject { + + @JsonProperty("fullName") + private String fullName; + + @JsonProperty("type") + private Type type; + + private SecurableObject parent; + private String name; + + /** Default constructor for Jackson deserialization. */ + protected SecurableObjectDTO() {} + + /** + * Creates a new instance of RoleDTO. + * + * @param fullName The name of the Role DTO. + * @param type The type of the securable object. + */ + protected SecurableObjectDTO(String fullName, Type type) { + SecurableObject securableObject = SecurableObjects.parse(fullName, type); + this.type = type; + this.fullName = fullName; + this.parent = securableObject.parent(); + this.name = securableObject.name(); + } + + @Nullable + @Override + public SecurableObject parent() { + return parent; + } + + @Override + public String name() { + return name; + } + + @Override + public String fullName() { + return fullName; + } + + @Override + public Type type() { + return type; + } + + /** @return the builder for creating a new instance of SecurableObjectDTO. */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link SecurableObjectDTO}. */ + public static class Builder { + private String fullName; + private Type type; + + /** + * Sets the full name of the securable object. + * + * @param fullName The full name of the securable object. + * @return The builder instance. + */ + public Builder withFullName(String fullName) { + this.fullName = fullName; + return this; + } + + /** + * Sets the type of the securable object. + * + * @param type The type of the securable object. + * @return The builder instance. + */ + public Builder withType(Type type) { + this.type = type; + return this; + } + + /** + * Builds an instance of SecurableObjectDTO using the builder's properties. + * + * @return An instance of SecurableObjectDTO. + * @throws IllegalArgumentException If the full name or type are not set. + */ + public SecurableObjectDTO build() { + Preconditions.checkArgument( + StringUtils.isNotBlank(fullName), "full name cannot be null or empty"); + + Preconditions.checkArgument(type != null, "type cannot be null"); + + return new SecurableObjectDTO(fullName, type); + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java index f2deaf72f9a..89b11de658f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.dto.requests; +import com.datastrato.gravitino.dto.authorization.SecurableObjectDTO; import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; @@ -37,7 +38,7 @@ public class RoleCreateRequest implements RESTRequest { private List privileges; @JsonProperty("securableObject") - private String securableObject; + private SecurableObjectDTO securableObject; /** Default constructor for RoleCreateRequest. (Used for Jackson deserialization.) */ public RoleCreateRequest() { @@ -56,7 +57,7 @@ public RoleCreateRequest( String name, Map properties, List privileges, - String securableObject) { + SecurableObjectDTO securableObject) { super(); this.name = name; this.properties = properties; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index ad93bfbf566..be7a6691cc6 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -11,12 +11,14 @@ import com.datastrato.gravitino.Metalake; import com.datastrato.gravitino.authorization.Group; import com.datastrato.gravitino.authorization.Role; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.authorization.User; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.MetalakeDTO; import com.datastrato.gravitino.dto.authorization.GroupDTO; import com.datastrato.gravitino.dto.authorization.RoleDTO; +import com.datastrato.gravitino.dto.authorization.SecurableObjectDTO; import com.datastrato.gravitino.dto.authorization.UserDTO; import com.datastrato.gravitino.dto.file.FilesetDTO; import com.datastrato.gravitino.dto.messaging.TopicDTO; @@ -384,13 +386,30 @@ public static RoleDTO toDTO(Role role) { return RoleDTO.builder() .withName(role.name()) - .withSecurableObject(role.securableObject()) + .withSecurableObject(toDTO(role.securableObject())) .withPrivileges(role.privileges()) .withProperties(role.properties()) .withAudit(toDTO(role.auditInfo())) .build(); } + /** + * Converts a securable object implementation to a SecurableObjectDTO. + * + * @param securableObject The securable object implementation. + * @return The securable object DTO. + */ + public static SecurableObjectDTO toDTO(SecurableObject securableObject) { + if (securableObject instanceof SecurableObjectDTO) { + return (SecurableObjectDTO) securableObject; + } + + return SecurableObjectDTO.builder() + .withFullName(securableObject.fullName()) + .withType(securableObject.type()) + .build(); + } + /** * Converts a Expression to an FunctionArg DTO. * diff --git a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java index 96b27b0e851..f4515c726b0 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java @@ -23,6 +23,7 @@ import com.datastrato.gravitino.dto.rel.SchemaDTO; import com.datastrato.gravitino.dto.rel.TableDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; +import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Lists; import java.time.Instant; @@ -267,8 +268,8 @@ void testRoleResponse() throws IllegalArgumentException { RoleDTO role = RoleDTO.builder() .withName("role1") - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) - .withSecurableObject(SecurableObjects.ofCatalog("catalog")) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) + .withSecurableObject(DTOConverters.toDTO(SecurableObjects.ofCatalog("catalog"))) .withAudit(audit) .build(); RoleResponse response = new RoleResponse(role); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index 841a8f9c15e..92d7623b13b 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.RoleEntity; import java.util.stream.Collectors; @@ -29,7 +30,8 @@ public Role serialize(RoleEntity roleEntity) { roleEntity.privileges().stream() .map(privilege -> privilege.name().toString()) .collect(Collectors.toList())) - .setSecurableObject(roleEntity.securableObject().toString()); + .setSecurableObjectFullName(roleEntity.securableObject().fullName()) + .setSecurableObjectType(roleEntity.securableObject().type().name()); if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { builder.putAllProperties(roleEntity.properties()); @@ -55,7 +57,10 @@ public RoleEntity deserialize(Role role, Namespace namespace) { role.getPrivilegesList().stream() .map(Privileges::fromString) .collect(Collectors.toList())) - .withSecurableObject(SecurableObjects.parse(role.getSecurableObject())) + .withSecurableObject( + SecurableObjects.parse( + role.getSecurableObjectFullName(), + SecurableObject.Type.valueOf(role.getSecurableObjectType()))) .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); if (!role.getPropertiesMap().isEmpty()) { diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java index 6b69395be53..8e4d88e4520 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java @@ -222,7 +222,11 @@ public void testCreateRole() { Role role = accessControlManager.createRole( - "metalake", "create", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + "metalake", + "create", + props, + SecurableObjects.ofCatalog("catalog"), + Lists.newArrayList()); Assertions.assertEquals("create", role.name()); testProperties(props, role.properties()); @@ -234,7 +238,7 @@ public void testCreateRole() { "metalake", "create", props, - SecurableObjects.ofAllCatalogs(), + SecurableObjects.ofCatalog("catalog"), Lists.newArrayList())); } @@ -243,7 +247,7 @@ public void testLoadRole() { Map props = ImmutableMap.of("k1", "v1"); accessControlManager.createRole( - "metalake", "loadRole", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + "metalake", "loadRole", props, SecurableObjects.ofCatalog("catalog"), Lists.newArrayList()); Role cachedRole = accessControlManager.getRole("metalake", "loadRole"); accessControlManager.getRoleManager().getCache().invalidateAll(); @@ -267,7 +271,7 @@ public void testDropRole() { Map props = ImmutableMap.of("k1", "v1"); accessControlManager.createRole( - "metalake", "testDrop", props, SecurableObjects.ofAllCatalogs(), Lists.newArrayList()); + "metalake", "testDrop", props, SecurableObjects.ofCatalog("catalog"), Lists.newArrayList()); // Test drop role boolean dropped = accessControlManager.deleteRole("metalake", "testDrop"); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java index 4d1080649a7..8fab7127f09 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -84,8 +84,8 @@ public class TestAccessControlManagerForPermissions { .withId(1L) .withName("role") .withProperties(Maps.newHashMap()) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) - .withSecurableObject(SecurableObjects.of(CATALOG)) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(CATALOG)) .withAuditInfo(auditInfo) .build(); @@ -268,7 +268,7 @@ public void testDropRole() throws IOException { .withId(1L) .withName(anotherRole) .withProperties(Maps.newHashMap()) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .withSecurableObject(SecurableObjects.ofCatalog(CATALOG)) .withAuditInfo(auditInfo) .build(); diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java index 7108f33ef8d..109254af7fc 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java @@ -266,8 +266,8 @@ public void testRole() { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.of(catalogName)) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .withProperties(map) .build(); @@ -277,17 +277,17 @@ public void testRole() { Assertions.assertEquals(auditInfo, fields.get(RoleEntity.AUDIT_INFO)); Assertions.assertEquals(map, fields.get(RoleEntity.PROPERTIES)); Assertions.assertEquals( - Lists.newArrayList(Privileges.LoadCatalog.get()), fields.get(RoleEntity.PRIVILEGES)); + Lists.newArrayList(Privileges.UseCatalog.get()), fields.get(RoleEntity.PRIVILEGES)); Assertions.assertEquals( - SecurableObjects.of(catalogName), fields.get(RoleEntity.SECURABLE_OBJECT)); + SecurableObjects.ofCatalog(catalogName), fields.get(RoleEntity.SECURABLE_OBJECT)); RoleEntity roleWithoutFields = RoleEntity.builder() .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.of(catalogName)) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .build(); Assertions.assertNull(roleWithoutFields.properties()); } diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index a314a3f1c73..3f88105e21c 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -386,8 +386,8 @@ public void testEntitiesSerDe() throws IOException { .withName(roleName) .withNamespace(roleNamespace) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.of(catalogName)) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .withProperties(props) .build(); byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); @@ -401,8 +401,8 @@ public void testEntitiesSerDe() throws IOException { .withName(roleName) .withNamespace(roleNamespace) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.of(catalogName)) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog(catalogName)) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .build(); roleBytes = protoEntitySerDe.serialize(roleWithoutFields); roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 42fad98d86a..bd49c196b0a 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -1239,8 +1239,8 @@ private static RoleEntity createRole(String metalake, String name, AuditInfo aud .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.of("catalog")) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withSecurableObject(SecurableObjects.ofCatalog("catalog")) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .withProperties(Collections.emptyMap()) .build(); } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 5c6c4aa2e51..64517c33549 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -770,8 +770,8 @@ public static RoleEntity createRoleEntity( .withNamespace(namespace) .withProperties(null) .withAuditInfo(auditInfo) - .withSecurableObject(SecurableObjects.ofAllCatalogs()) - .withPrivileges(Lists.newArrayList(Privileges.fromName(Privilege.Name.LOAD_CATALOG))) + .withSecurableObject(SecurableObjects.ofCatalog("catalog")) + .withPrivileges(Lists.newArrayList(Privileges.fromName(Privilege.Name.USE_CATALOG))) .build(); } diff --git a/meta/src/main/proto/gravitino_meta.proto b/meta/src/main/proto/gravitino_meta.proto index e160c8eaf02..98162c3ba98 100644 --- a/meta/src/main/proto/gravitino_meta.proto +++ b/meta/src/main/proto/gravitino_meta.proto @@ -122,7 +122,8 @@ message Role { uint64 id = 1; string name = 2; repeated string privileges = 3; - string securable_object = 4; - map properties = 5; - AuditInfo audit_info = 6; + string securable_object_full_name = 4; + string securable_object_type = 5; + map properties = 6; + AuditInfo audit_info = 7; } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java index 14b8d331179..a5fc49e9057 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java @@ -65,6 +65,7 @@ public Response getRole(@PathParam("metalake") String metalake, @PathParam("role @ResponseMetered(name = "create-role", absolute = true) public Response createRole(@PathParam("metalake") String metalake, RoleCreateRequest request) { try { + return Utils.doAs( httpRequest, () -> @@ -75,7 +76,9 @@ public Response createRole(@PathParam("metalake") String metalake, RoleCreateReq metalake, request.getName(), request.getProperties(), - SecurableObjects.parse(request.getSecurableObject()), + SecurableObjects.parse( + request.getSecurableObject().fullName(), + request.getSecurableObject().type()), request.getPrivileges().stream() .map(Privileges::fromString) .collect(Collectors.toList())))))); diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java index 59397bce665..2c6d88468c8 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java @@ -24,6 +24,7 @@ import com.datastrato.gravitino.dto.responses.ErrorConstants; import com.datastrato.gravitino.dto.responses.ErrorResponse; import com.datastrato.gravitino.dto.responses.RoleResponse; +import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchRoleException; import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; @@ -100,8 +101,8 @@ public void testCreateRole() { new RoleCreateRequest( "role", Collections.emptyMap(), - Lists.newArrayList(Privileges.LoadCatalog.get().name().toString()), - SecurableObjects.of("catalog").toString()); + Lists.newArrayList(Privileges.UseCatalog.get().name().toString()), + DTOConverters.toDTO(SecurableObjects.ofCatalog("catalog"))); Role role = buildRole("role1"); when(manager.createRole(any(), any(), any(), any(), any())).thenReturn(role); @@ -120,8 +121,9 @@ public void testCreateRole() { RoleDTO roleDTO = roleResponse.getRole(); Assertions.assertEquals("role1", roleDTO.name()); - Assertions.assertEquals(SecurableObjects.of("catalog"), roleDTO.securableObject()); - Assertions.assertEquals(Lists.newArrayList(Privileges.LoadCatalog.get()), roleDTO.privileges()); + Assertions.assertEquals( + SecurableObjects.ofCatalog("catalog").fullName(), roleDTO.securableObject().fullName()); + Assertions.assertEquals(Lists.newArrayList(Privileges.UseCatalog.get()), roleDTO.privileges()); // Test to throw NoSuchMetalakeException doThrow(new NoSuchMetalakeException("mock error")) @@ -194,8 +196,9 @@ public void testGetRole() { RoleDTO roleDTO = roleResponse.getRole(); Assertions.assertEquals("role1", roleDTO.name()); Assertions.assertTrue(role.properties().isEmpty()); - Assertions.assertEquals(SecurableObjects.of("catalog"), roleDTO.securableObject()); - Assertions.assertEquals(Lists.newArrayList(Privileges.LoadCatalog.get()), roleDTO.privileges()); + Assertions.assertEquals( + SecurableObjects.ofCatalog("catalog").fullName(), roleDTO.securableObject().fullName()); + Assertions.assertEquals(Lists.newArrayList(Privileges.UseCatalog.get()), roleDTO.privileges()); // Test to throw NoSuchMetalakeException doThrow(new NoSuchMetalakeException("mock error")).when(manager).getRole(any(), any()); @@ -245,9 +248,9 @@ private Role buildRole(String role) { return RoleEntity.builder() .withId(1L) .withName(role) - .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) + .withPrivileges(Lists.newArrayList(Privileges.UseCatalog.get())) .withProperties(Collections.emptyMap()) - .withSecurableObject(SecurableObjects.of("catalog")) + .withSecurableObject(SecurableObjects.ofCatalog("catalog")) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); From d2d1450a25ac3cd2efa1703c1bf085573b51d9a3 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 22 Apr 2024 21:51:38 +0800 Subject: [PATCH 099/106] [#3075] fix(core): Fix issues schema entity cannot delete if the schema is not found from underlying sources (#3095) ### What changes were proposed in this pull request? Ignore the return value of dropSchema from underlying sources, to drop the entity from store mandatorily. ### Why are the changes needed? If the return value of `dropSchema` is false from underlying sources, the current implementation will ignore deleting it from entity store, so it cannot drop the catalog. Fix: #3075 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Adding new ITs. --- .../gravitino/rel/SupportsSchemas.java | 3 +- .../integration/test/HadoopCatalogIT.java | 41 ++++++++++++ .../hive/integration/test/CatalogHiveIT.java | 66 +++++++++++++++++++ .../integration/test/CatalogKafkaIT.java | 26 +++++++- .../catalog/SchemaOperationDispatcher.java | 35 +++++++--- .../catalog/TableOperationDispatcher.java | 66 +++++++++++++------ .../catalog/TopicOperationDispatcher.java | 33 +++++++--- .../server/web/rest/RoleOperations.java | 6 +- 8 files changed, 232 insertions(+), 44 deletions(-) diff --git a/api/src/main/java/com/datastrato/gravitino/rel/SupportsSchemas.java b/api/src/main/java/com/datastrato/gravitino/rel/SupportsSchemas.java index e490aa76487..2c47c304114 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/SupportsSchemas.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/SupportsSchemas.java @@ -108,7 +108,8 @@ Schema createSchema(NameIdentifier ident, String comment, Map pr * * @param ident The name identifier of the schema. * @param cascade If true, recursively drop all objects within the schema. - * @return True if the schema exists and is dropped successfully, false otherwise. + * @return True if the schema exists and is dropped successfully, false if the schema doesn't + * exist. * @throws NonEmptySchemaException If the schema is not empty and cascade is false. */ boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmptySchemaException; diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index 22bd93f18f2..c9c61cb6166 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -502,6 +502,47 @@ public void testFilesetRemoveProperties() throws IOException { Assertions.assertEquals(0, newFileset.properties().size(), "properties should be removed"); } + @Test + public void testDropCatalogWithEmptySchema() { + String catalogName = + GravitinoITUtils.genRandomName("test_drop_catalog_with_empty_schema_catalog"); + // Create a catalog without specifying location. + Catalog filesetCatalog = + metalake.createCatalog( + NameIdentifier.of(metalakeName, catalogName), + Catalog.Type.FILESET, + provider, + "comment", + ImmutableMap.of()); + + // Create a schema without specifying location. + String schemaName = + GravitinoITUtils.genRandomName("test_drop_catalog_with_empty_schema_schema"); + filesetCatalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), "comment", ImmutableMap.of()); + + // Drop the empty schema. + boolean dropped = + filesetCatalog + .asSchemas() + .dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), true); + Assertions.assertTrue(dropped, "schema should be dropped"); + Assertions.assertFalse( + filesetCatalog + .asSchemas() + .schemaExists(NameIdentifier.of(metalakeName, catalogName, schemaName)), + "schema should not be exists"); + + // Drop the catalog. + dropped = metalake.dropCatalog(NameIdentifier.of(metalakeName, catalogName)); + Assertions.assertTrue(dropped, "catalog should be dropped"); + Assertions.assertFalse( + metalake.catalogExists(NameIdentifier.of(metalakeName, catalogName)), + "catalog should not be exists"); + } + private Fileset createFileset( String filesetName, String comment, diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 46c3a105643..bd12d29b12e 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -1563,6 +1563,72 @@ public void testPurgeHiveExternalTable() throws TException, InterruptedException hdfs.listStatus(tableDirectory).length > 0, "The table should not be empty"); } + @Test + public void testRemoveNonExistTable() throws TException, InterruptedException { + Column[] columns = createColumns(); + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + columns, + TABLE_COMMENT, + ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), + new Transform[] {Transforms.identity(columns[2].name())}); + + // Directly drop table from hive metastore. + hiveClientPool.run( + client -> { + client.dropTable(schemaName, tableName, true, false, false); + return null; + }); + + // Drop table from catalog, drop non-exist table should return false; + Assertions.assertFalse( + catalog + .asTableCatalog() + .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + "The table should not be found in the catalog"); + + Assertions.assertFalse( + catalog + .asTableCatalog() + .tableExists(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + "The table should not be found in the catalog"); + } + + @Test + public void testPurgeNonExistTable() throws TException, InterruptedException { + Column[] columns = createColumns(); + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + columns, + TABLE_COMMENT, + ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), + new Transform[] {Transforms.identity(columns[2].name())}); + + // Directly drop table from hive metastore. + hiveClientPool.run( + client -> { + client.dropTable(schemaName, tableName, true, false, true); + return null; + }); + + // Drop table from catalog, drop non-exist table should return false; + Assertions.assertFalse( + catalog + .asTableCatalog() + .purgeTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + "The table should not be found in the catalog"); + + Assertions.assertFalse( + catalog + .asTableCatalog() + .tableExists(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + "The table should not be found in the catalog"); + } + @Test void testCustomCatalogOperations() { String catalogName = "custom_catalog"; diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index e98b918cf78..e7854705b84 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -309,7 +309,7 @@ public void testAlterTopic() { } @Test - public void testDropTopic() { + public void testDropTopic() throws ExecutionException, InterruptedException { String topicName = GravitinoITUtils.genRandomName("test-topic"); Topic createdTopic = catalog @@ -332,6 +332,30 @@ public void testDropTopic() { Assertions.assertThrows(ExecutionException.class, () -> getTopicDesc(createdTopic.name())); Assertions.assertTrue( ex.getMessage().contains("This server does not host this topic-partition")); + + // verify dropping non-exist topic + String topicName1 = GravitinoITUtils.genRandomName("test-topic"); + catalog + .asTopicCatalog() + .createTopic( + NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1), + "comment", + null, + Collections.emptyMap()); + + adminClient.deleteTopics(Collections.singleton(topicName1)).all().get(); + boolean dropped1 = + catalog + .asTopicCatalog() + .dropTopic( + NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1)); + Assertions.assertFalse(dropped1, "Should return false when dropping non-exist topic"); + Assertions.assertFalse( + catalog + .asTopicCatalog() + .topicExists( + NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1)), + "Topic should not exist after dropping"); } private void assertTopicWithKafka(Topic createdTopic) diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index 986de6bfa00..57c75681e0e 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -14,6 +14,7 @@ import com.datastrato.gravitino.connector.HasPropertyMetadata; import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; @@ -292,25 +293,41 @@ public Schema alterSchema(NameIdentifier ident, SchemaChange... changes) * * @param ident The identifier of the schema to be dropped. * @param cascade If true, drops all tables within the schema as well. - * @return True if the schema was successfully dropped, false otherwise. + * @return True if the schema was successfully dropped, false if the schema doesn't exist. * @throws NonEmptySchemaException If the schema contains tables and cascade is set to false. + * @throws RuntimeException If an error occurs while dropping the schema. */ @Override public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmptySchemaException { - boolean dropped = + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + boolean droppedFromCatalog = doWithCatalog( - getCatalogIdentifier(ident), + catalogIdent, c -> c.doWithSchemaOps(s -> s.dropSchema(ident, cascade)), - NonEmptySchemaException.class); - - if (!dropped) { - return false; - } + NonEmptySchemaException.class, + RuntimeException.class); + // For unmanaged schema, it could happen that the schema: + // 1. Is not found in the catalog (dropped directly from underlying sources) + // 2. Is found in the catalog but not in the store (not managed by Gravitino) + // 3. Is found in the catalog and the store (managed by Gravitino) + // 4. Neither found in the catalog nor in the store. + // In all situations, we try to delete the schema from the store, but we don't take the + // return value of the store operation into account. We only take the return value of the + // catalog into account. + // + // For managed schema, we should take the return value of the store operation into account. + boolean droppedFromStore = false; try { - return store.delete(ident, SCHEMA, cascade); + droppedFromStore = store.delete(ident, SCHEMA, cascade); + } catch (NoSuchEntityException e) { + LOG.warn("The schema to be dropped does not exist in the store: {}", ident, e); } catch (Exception e) { throw new RuntimeException(e); } + + return isManagedEntity(catalogIdent, Capability.Scope.SCHEMA) + ? droppedFromStore + : droppedFromCatalog; } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index 5ba444e7770..df41cdc2634 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -14,6 +14,8 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; @@ -296,27 +298,37 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) * @param ident The identifier of the table to drop. * @return {@code true} if the table was successfully dropped, {@code false} if the table does not * exist. - * @throws NoSuchTableException If the table to drop does not exist. + * @throws RuntimeException If an error occurs while dropping the table. */ @Override public boolean dropTable(NameIdentifier ident) { - boolean dropped = + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + boolean droppedFromCatalog = doWithCatalog( - getCatalogIdentifier(ident), - c -> c.doWithTableOps(t -> t.dropTable(ident)), - NoSuchTableException.class); - - if (!dropped) { - return false; - } + catalogIdent, c -> c.doWithTableOps(t -> t.dropTable(ident)), RuntimeException.class); + // For unmanaged table, it could happen that the table: + // 1. Is not found in the catalog (dropped directly from underlying sources) + // 2. Is found in the catalog but not in the store (not managed by Gravitino) + // 3. Is found in the catalog and the store (managed by Gravitino) + // 4. Neither found in the catalog nor in the store. + // In all situations, we try to delete the schema from the store, but we don't take the + // return value of the store operation into account. We only take the return value of the + // catalog into account. + // + // For managed table, we should take the return value of the store operation into account. + boolean droppedFromStore = false; try { - store.delete(ident, TABLE); + droppedFromStore = store.delete(ident, TABLE); + } catch (NoSuchEntityException e) { + LOG.warn("The table to be dropped does not exist in the store: {}", ident, e); } catch (Exception e) { throw new RuntimeException(e); } - return true; + return isManagedEntity(catalogIdent, Capability.Scope.TABLE) + ? droppedFromStore + : droppedFromCatalog; } /** @@ -331,26 +343,40 @@ public boolean dropTable(NameIdentifier ident) { * @param ident A table identifier. * @return True if the table was purged, false if the table did not exist. * @throws UnsupportedOperationException If the catalog does not support to purge a table. + * @throws RuntimeException If an error occurs while purging the table. */ @Override public boolean purgeTable(NameIdentifier ident) throws UnsupportedOperationException { - boolean purged = + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + boolean droppedFromCatalog = doWithCatalog( - getCatalogIdentifier(ident), + catalogIdent, c -> c.doWithTableOps(t -> t.purgeTable(ident)), - NoSuchTableException.class, + RuntimeException.class, UnsupportedOperationException.class); - if (!purged) { - return false; - } - + // For unmanaged table, it could happen that the table: + // 1. Is not found in the catalog (dropped directly from underlying sources) + // 2. Is found in the catalog but not in the store (not managed by Gravitino) + // 3. Is found in the catalog and the store (managed by Gravitino) + // 4. Neither found in the catalog nor in the store. + // In all situations, we try to delete the schema from the store, but we don't take the + // return value of the store operation into account. We only take the return value of the + // catalog into account. + // + // For managed table, we should take the return value of the store operation into account. + boolean droppedFromStore = false; try { - store.delete(ident, TABLE); + droppedFromStore = store.delete(ident, TABLE); + } catch (NoSuchEntityException e) { + LOG.warn("The table to be purged does not exist in the store: {}", ident, e); + return false; } catch (Exception e) { throw new RuntimeException(e); } - return true; + return isManagedEntity(catalogIdent, Capability.Scope.TABLE) + ? droppedFromStore + : droppedFromCatalog; } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java index c9188093d69..6d52e92d75a 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java @@ -13,6 +13,8 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTopicException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; @@ -248,25 +250,36 @@ public Topic alterTopic(NameIdentifier ident, TopicChange... changes) * * @param ident A topic identifier. * @return true If the topic is dropped, false if the topic does not exist. + * @throws RuntimeException If an error occurs while dropping the topic. */ @Override public boolean dropTopic(NameIdentifier ident) { - boolean dropped = + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + boolean droppedFromCatalog = doWithCatalog( - getCatalogIdentifier(ident), - c -> c.doWithTopicOps(t -> t.dropTopic(ident)), - NoSuchTopicException.class); - - if (!dropped) { - return false; - } + catalogIdent, c -> c.doWithTopicOps(t -> t.dropTopic(ident)), RuntimeException.class); + // For unmanaged topic, it could happen that the topic: + // 1. Is not found in the catalog (dropped directly from underlying sources) + // 2. Is found in the catalog but not in the store (not managed by Gravitino) + // 3. Is found in the catalog and the store (managed by Gravitino) + // 4. Neither found in the catalog nor in the store. + // In all situations, we try to delete the schema from the store, but we don't take the + // return value of the store operation into account. We only take the return value of the + // catalog into account. + // + // For managed topic, we should take the return value of the store operation into account. + boolean droppedFromStore = false; try { - store.delete(ident, TOPIC); + droppedFromStore = store.delete(ident, TOPIC); + } catch (NoSuchEntityException e) { + LOG.warn("The topic to be dropped does not exist in the store: {}", ident, e); } catch (Exception e) { throw new RuntimeException(e); } - return true; + return isManagedEntity(catalogIdent, Capability.Scope.TOPIC) + ? droppedFromStore + : droppedFromCatalog; } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java index a5fc49e9057..811553e1938 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java @@ -99,11 +99,11 @@ public Response deleteRole( return Utils.doAs( httpRequest, () -> { - boolean deteted = accessControlManager.deleteRole(metalake, role); - if (!deteted) { + boolean deleted = accessControlManager.deleteRole(metalake, role); + if (!deleted) { LOG.warn("Failed to delete role {} under metalake {}", role, metalake); } - return Utils.ok(new DeleteResponse(deteted)); + return Utils.ok(new DeleteResponse(deleted)); }); } catch (Exception e) { return ExceptionHandlers.handleRoleException(OperationType.DELETE, role, metalake, e); From 841c90254c3a7b56899512fb347593c64d6a2175 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Mon, 22 Apr 2024 22:20:31 +0800 Subject: [PATCH 100/106] [#3058] fix(web): fix value displaying and enhance the styles (#3102) ### What changes were proposed in this pull request? Fix value displaying and enhance the styles. ### Why are the changes needed? Fix: #3058 ### Does this PR introduce _any_ user-facing change? `[['id', 'b'], ['name'], ['d', 'e'], ['f'], ['g']]` header: image column: image others: image image image image image image ### How was this patch tested? local --- .../rightContent/tabsContent/TabsContent.js | 57 +++++++++++++++---- .../tabsContent/tableView/TableView.js | 31 ++++++---- web/src/lib/store/metalakes/index.js | 2 +- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js index bdea15e7763..eb46897bf47 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js @@ -13,6 +13,8 @@ import { styled, Box, Divider, List, ListItem, ListItemText, Stack, Tab, Typogra import Tooltip, { tooltipClasses } from '@mui/material/Tooltip' import { TabContext, TabList, TabPanel } from '@mui/lab' +import clsx from 'clsx' + import { useAppSelector } from '@/lib/hooks/useStore' import { useSearchParams } from 'next/navigation' @@ -165,15 +167,46 @@ const TabsContent = () => { {item.items.map((it, idx) => { return ( - - {item.type === 'sortOrders' ? it.text : it.fields.join('.')} - + + + {item.type === 'sortOrders' + ? it.text + : it.fields.map((v, vi) => { + return ( + + + {Array.isArray(v) ? v.join('.') : v} + + {vi < it.fields.length - 1 && ( + `1px solid ${theme.palette.grey[800]}` + }} + > + )} + + ) + })} + + {idx < item.items.length - 1 && ( + `1px solid ${theme.palette.grey[800]}` + }} + > + )} + ) })} @@ -220,9 +253,11 @@ const TabsContent = () => { return ( - {it.fields.join('.')} + {it.fields.map(v => (Array.isArray(v) ? v.join('.') : v)).join(',')} - {idx < item.items.length - 1 && , } + {idx < item.items.length - 1 && ( + , + )} ) })} diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js index 71a2ea1464f..2549b4ed308 100644 --- a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js +++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js @@ -7,7 +7,7 @@ import { Inconsolata } from 'next/font/google' -import { useState, useEffect } from 'react' +import { useState, useEffect, Fragment } from 'react' import Link from 'next/link' @@ -113,15 +113,26 @@ const TableView = () => { {items.map((it, idx) => { return ( - - {it.text || it.fields} - + + + {it.text || it.fields} + + {idx < items.length - 1 && ( + `1px solid ${theme.palette.grey[800]}` + }} + > + )} + ) })} diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js index 30742dea554..6b4f07acf4c 100644 --- a/web/src/lib/store/metalakes/index.js +++ b/web/src/lib/store/metalakes/index.js @@ -672,7 +672,7 @@ export const getTableDetails = createAsyncThunk( fields: i.fieldNames, name: i.name, indexType: i.indexType, - text: `${i.name}(${i.fieldNames.join('.')})` + text: `${i.name}(${i.fieldNames.map(v => v.join('.')).join(',')})` } }) } From 4cf2ed266e4058cff376786442bc766784bc45ec Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 22 Apr 2024 22:46:48 +0800 Subject: [PATCH 101/106] [#3001] docs(spark-connector): add spark connector document (#3018) ### What changes were proposed in this pull request? add spark connector document ### Why are the changes needed? Fix: #3001 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? document --- docs/spark-connector/spark-catalog-hive.md | 53 +++++++++++ docs/spark-connector/spark-catalog-iceberg.md | 60 ++++++++++++ docs/spark-connector/spark-connector.md | 93 +++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 docs/spark-connector/spark-catalog-hive.md create mode 100644 docs/spark-connector/spark-catalog-iceberg.md create mode 100644 docs/spark-connector/spark-connector.md diff --git a/docs/spark-connector/spark-catalog-hive.md b/docs/spark-connector/spark-catalog-hive.md new file mode 100644 index 00000000000..5e38056cac9 --- /dev/null +++ b/docs/spark-connector/spark-catalog-hive.md @@ -0,0 +1,53 @@ +--- +title: "Spark connector hive catalog" +slug: /spark-connector/spark-catalog-hive +keyword: spark connector hive catalog +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +With the Gravitino Spark connector, accessing data or managing metadata in Hive catalogs becomes straightforward, enabling seamless federation queries across different Hive catalogs. + +## Capabilities + +Supports most DDL and DML operations in SparkSQL, except such operations: + +- Function operations +- Partition operations +- View operations +- Querying UDF +- `LOAD` clause +- `CREATE TABLE LIKE` clause +- `TRUCATE TABLE` clause + +## Requirement + +* Hive metastore 2.x +* HDFS 2.x or 3.x + +## SQL example + + +```sql + +// Suppose hive_a is the Hive catalog name managed by Gravitino +USE hive_a; + +CREATE DATABASE IF NOT EXISTS mydatabase; +USE mydatabase; + +// Create table +CREATE TABLE IF NOT EXISTS employees ( + id INT, + name STRING, + age INT +) +PARTITIONED BY (department STRING) +STORED AS PARQUET; +DESC TABLE EXTENDED employees; + +INSERT OVERWRITE TABLE employees PARTITION(department='Engineering') VALUES (1, 'John Doe', 30), (2, 'Jane Smith', 28); +INSERT OVERWRITE TABLE employees PARTITION(department='Marketing') VALUES (3, 'Mike Brown', 32); + +SELECT * FROM employees WHERE department = 'Engineering'; +``` diff --git a/docs/spark-connector/spark-catalog-iceberg.md b/docs/spark-connector/spark-catalog-iceberg.md new file mode 100644 index 00000000000..a50ed4fffe5 --- /dev/null +++ b/docs/spark-connector/spark-catalog-iceberg.md @@ -0,0 +1,60 @@ +--- +title: "Spark connector Iceberg catalog" +slug: /spark-connector/spark-catalog-iceberg +keyword: spark connector iceberg catalog +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +## Capabilities + +#### Support basic DML and DDL operations: + +- `CREATE TABLE` + +Supports basic create table clause including table schema, properties, partition, does not support distribution and sort orders. + +- `DROP TABLE` +- `ALTER TABLE` +- `INSERT INTO&OVERWRITE` +- `SELECT` +- `DELETE` + +Supports file delete only. + +#### Not supported operations: + +- Row level operations. like `MERGE INOT`, `DELETE FROM`, `UPDATE` +- View operations. +- Branching and tagging operations. +- Spark procedures. +- Other Iceberg extension SQL, like: + - `ALTER TABLE prod.db.sample ADD PARTITION FIELD xx` + - `ALTER TABLE ... WRITE ORDERED BY` + +## SQL example + +```sql +// Suppose iceberg_a is the Iceberg catalog name managed by Gravitino +USE iceberg_a; + +CREATE DATABASE IF NOT EXISTS mydatabase; +USE mydatabase; + +CREATE TABLE IF NOT EXISTS employee ( + id bigint, + name string, + department string, + hire_date timestamp +) USING iceberg +PARTITIONED BY (days(hire_date)); +DESC TABLE EXTENDED employee; + +INSERT INTO employee +VALUES +(1, 'Alice', 'Engineering', TIMESTAMP '2021-01-01 09:00:00'), +(2, 'Bob', 'Marketing', TIMESTAMP '2021-02-01 10:30:00'), +(3, 'Charlie', 'Sales', TIMESTAMP '2021-03-01 08:45:00'); + +SELECT * FROM employee WHERE date(hire_date) = '2021-01-01' +``` diff --git a/docs/spark-connector/spark-connector.md b/docs/spark-connector/spark-connector.md new file mode 100644 index 00000000000..246b566ce1b --- /dev/null +++ b/docs/spark-connector/spark-connector.md @@ -0,0 +1,93 @@ +--- +title: "Gravitino Spark connector" +slug: /spark-connector/spark-connector +keyword: spark connector federation query +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +## Overview + +The Gravitino Spark connector leverages the Spark DataSourceV2 interface to facilitate the management of diverse catalogs under Gravitino. This capability allows users to perform federation queries, accessing data from various catalogs through a unified interface and consistent access control. + +## Capabilities + +1. Supports [Hive catalog](spark-catalog-hive.md) and [Iceberg catalog](spark-catalog-iceberg.md). +2. Supports federation query. +3. Supports most DDL and DML SQLs. + +## Requirement + +* Spark 3.4 +* Scala 2.12 +* JDK 8,11,17 + +## How to use it + +1. [Build](../how-to-build.md) or download the Gravitino spark connector jar, and place it to the classpath of Spark. +2. Configure the Spark session to use the Gravitino spark connector. + +| Property | Type | Default Value | Description | Required | Since Version | +|------------------------------|--------|---------------|-----------------------------------------------------------------------------------------------------|----------|---------------| +| spark.plugins | string | (none) | Gravitino spark plugin name, `com.datastrato.gravitino.spark.connector.plugin.GravitinoSparkPlugin` | Yes | 0.5.0 | +| spark.sql.gravitino.metalake | string | (none) | The metalake name that spark connector used to request to Gravitino. | Yes | 0.5.0 | +| spark.sql.gravitino.uri | string | (none) | The uri of Gravitino server address. | Yes | 0.5.0 | + +```shell +./bin/spark-sql -v \ +--conf spark.plugins="com.datastrato.gravitino.spark.connector.plugin.GravitinoSparkPlugin" \ +--conf spark.sql.gravitino.uri=http://127.0.0.1:8090 \ +--conf spark.sql.gravitino.metalake=test \ +--conf spark.sql.warehouse.dir=hdfs://127.0.0.1:9000/user/hive/warehouse-hive +``` + +3. Execute the Spark SQL query. + +Suppose there are two catalogs in the metalake `test`, `hive` for Hive catalog and `iceberg` for Iceberg catalog. + +```sql +// use hive catalog +USE hive; +CREATE DATABASE db; +USE db; +CREATE TABLE hive_students (id INT, name STRING); +INSERT INTO hive_students VALUES (1, 'Alice'), (2, 'Bob'); + +// use Iceberg catalog +USE iceberg; +USE db; +CREATE TABLE IF NOT EXISTS iceberg_scores (id INT, score INT) USING iceberg; +INSERT INTO iceberg_scores VALUES (1, 95), (2, 88); + +// execute federation query between hive table and iceberg table +SELECT hs.name, is.score FROM hive.db.hive_students hs JOIN iceberg_scores is ON hs.id = is.id; +``` + +:::info +The command `SHOW CATALOGS` will only display the Spark default catalog, named spark_catalog, due to limitations within the Spark catalog manager. It does not list the catalogs present in the metalake. However, after explicitly using the `USE` command with a specific catalog name, that catalog name then becomes visible in the output of `SHOW CATALOGS`. +::: + +## Datatype mapping + +Gravitino spark connector support the following datatype mapping between Spark and Gravitino. + +| Spark Data Type | Gravitino Data Type | Since Version | +|-----------------|---------------------|---------------| +| `BooleanType` | `boolean` | 0.5.0 | +| `ByteType` | `byte` | 0.5.0 | +| `ShortType` | `short` | 0.5.0 | +| `IntegerType` | `integer` | 0.5.0 | +| `LongType` | `long` | 0.5.0 | +| `FloatType` | `float` | 0.5.0 | +| `DoubleType` | `double` | 0.5.0 | +| `DecimalType` | `decimal` | 0.5.0 | +| `StringType` | `string` | 0.5.0 | +| `CharType` | `char` | 0.5.0 | +| `VarcharType` | `varchar` | 0.5.0 | +| `TimestampType` | `timestamp` | 0.5.0 | +| `TimestampType` | `timestamp` | 0.5.0 | +| `DateType` | `date` | 0.5.0 | +| `BinaryType` | `binary` | 0.5.0 | +| `ArrayType` | `array` | 0.5.0 | +| `MapType` | `map` | 0.5.0 | +| `StructType` | `struct` | 0.5.0 | \ No newline at end of file From 59dc495146126230094dcffb3ef4936ff81efc3c Mon Sep 17 00:00:00 2001 From: Yuhui Date: Mon, 22 Apr 2024 23:26:13 +0800 Subject: [PATCH 102/106] [#1785] improve (trino-connector): Gravitino Trino connector support origin Trino connector configuration in the catalog properties (#3054) ### What changes were proposed in this pull request? Gravitino Trino connector Support passing Trino connector configuration in the catalog properties. the configuration key should start with `trino.bypass`. For example using `trino.bypass.hive.config.resources` to passing the `hive.config.resources` to the Gravitino Hive catalog in Trino runtime. ### Why are the changes needed? Fix: #1785 ### Does this PR introduce _any_ user-facing change? Add documents about : Passing Trino connector properties in `trino-connector/supported-catalog.md` ### How was this patch tested? UT IT --------- Co-authored-by: Qi Yu --- .../catalog/property/PropertyConverter.java | 2 + docs/apache-hive-catalog.md | 2 + docs/jdbc-mysql-catalog.md | 2 + docs/jdbc-postgresql-catalog.md | 2 + docs/lakehouse-iceberg-catalog.md | 2 + docs/trino-connector/supported-catalog.md | 24 ++ .../test/trino/TrinoConnectorIT.java | 40 ++- .../jdbc-mysql/00008_alter_catalog.sql | 6 +- .../jdbc-mysql/00008_alter_catalog.txt | 6 +- .../jdbc-mysql/catalog_mysql_prepare.sql | 2 +- .../jdbc-postgresql/catalog_pg_prepare.sql | 2 +- .../catalog/CatalogConnectorManager.java | 3 +- .../hive/HiveCatalogPropertyConverter.java | 261 +++++++++----- .../catalog/hive/HiveConnectorAdapter.java | 14 +- .../IcebergCatalogPropertyConverter.java | 328 ++++++++++++------ .../jdbc/JDBCCatalogPropertyConverter.java | 77 ++-- .../connector/metadata/GravitinoCatalog.java | 2 +- .../trino/connector/GravitinoMockServer.java | 7 +- .../TestCreateGravitinoConnector.java | 4 +- .../connector/TestGravitinoConnector.java | 2 +- ...itinoConnectorWithMetalakeCatalogName.java | 2 +- .../TestHiveCatalogPropertyConverter.java | 47 ++- .../TestIcebergCatalogPropertyConverter.java | 88 +++++ .../TestJDBCCatalogPropertyConverter.java | 84 +++++ .../metadata/TestGravitinoCatalog.java | 2 +- 25 files changed, 737 insertions(+), 274 deletions(-) diff --git a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java index 7125a044002..30619f832cf 100644 --- a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java +++ b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java @@ -15,6 +15,8 @@ /** Transforming between gravitino schema/table/column property and engine property. */ public abstract class PropertyConverter { + protected static final String TRINO_PROPERTIES_PREFIX = "trino.bypass."; + private static final Logger LOG = LoggerFactory.getLogger(PropertyConverter.class); /** * Mapping that maps engine properties to Gravitino properties. It will return a map that holds diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index 2fe3da52748..6743dad5b93 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -40,6 +40,8 @@ The Hive catalog supports creating, updating, and deleting databases and tables | `kerberos.check-interval-sec` | The interval to check validness of the principal | 60 | No | 0.4.0 | | `kerberos.keytab-fetch-timeout-sec` | The timeout to fetch key tab | 60 | No | 0.4.0 | +When you use the Gravitino with Trino. You can pass the Trino Hive connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.hive.config.resources` to pass the `hive.config.resources` to the Gravitino Hive catalog in Trino runtime. + ### Catalog operations Refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations) for more details. diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index aba936b28c6..fa0569f4415 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -37,6 +37,8 @@ You can pass to a MySQL data source any property that isn't defined by Gravitino Check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) +When you use the Gravitino with Trino. You can pass the Trino MySQL connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.join-pushdown.strategy` to pass the `join-pushdown.strategy` to the Gravitino MySQL catalog in Trino runtime. + If you use a JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-user` and `jdbc-password` to catalog properties. | Configuration item | Description | Default value | Required | Since Version | diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 5f673ecac68..69b7acd0350 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -35,6 +35,8 @@ Gravitino saves some system information in schema and table comment, like `(From Any property that isn't defined by Gravitino can pass to MySQL data source by adding `gravitino.bypass.` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. You can check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) +When you use the Gravitino with Trino. You can pass the Trino PostgreSQL connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.join-pushdown.strategy` to pass the `join-pushdown.strategy` to the Gravitino PostgreSQL catalog in Trino runtime. + If you use JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-database`, `jdbc-user` and `jdbc-password` to catalog properties. | Configuration item | Description | Default value | Required | Since Version | diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index a5d1b67b4b2..f33f32b97eb 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -41,6 +41,8 @@ Builds with Hadoop 2.10.x, there may be compatibility issues when accessing Hado Any properties not defined by Gravitino with `gravitino.bypass.` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. +When you use the Gravitino with Trino. You can pass the Trino Iceberg connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.iceberg.table-statistics-enabled` to pass the `iceberg.table-statistics-enabled` to the Gravitino Iceberg catalog in Trino runtime. + #### JDBC catalog If you are using JDBC catalog, you must provide `jdbc-user`, `jdbc-password` and `jdbc-driver` to catalog properties. diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index 158002c469f..5c9458e7449 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -122,6 +122,30 @@ call gravitino.system.alter_catalog( if you need more information about catalog, please refer to: [Create a Catalog](../manage-relational-metadata-using-gravitino.md#create-a-catalog). +## Passing Trino connector configuration +A Gravitino catalog is implemented by the Trino connector, so you can pass the Trino connector configuration to the Gravitino catalog. +For example, you want to set the `hive.config.resources` configuration for the Hive catalog, you can pass the configuration to the +Gravitino catalog like this: + +```sql +call gravitino.system.create_catalog( + 'gt_hive', + 'hive', + map( + array['metastore.uris', 'trino.bypass.hive.config.resources'], + array['thrift://trino-ci-hive:9083', "/tmp/hive-site.xml,/tmp/core-site.xml"] + ) +); +``` + +A prefix with `trino.bypass.` in the configuration key is used to indicate Gravitino connector to pass the Trino connector configuration to the Gravitino catalog in the Trino runtime. + +More trino connector configurations can refer to: +- [Hive catalog](https://trino.io/docs/current/connector/hive.html#hive-general-configuration-properties) +- [Iceberg catalog](https://trino.io/docs/current/connector/iceberg.html#general-configuration) +- [MySQL catalog](https://trino.io/docs/current/connector/mysql.html#general-configuration-properties) +- [PostgreSQL catalog](https://trino.io/docs/current/connector/postgresql.html#general-configuration-properties) + ## Data type mapping between Trino and Gravitino Gravitino connector supports the following data type conversions between Trino and Gravitino currently. Depending on the detailed catalog, Gravitino may not support some data types conversion for this specific catalog, for example, diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 1f8c90ce769..57f64493d1d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -825,16 +825,20 @@ void testHiveCatalogCreatedByGravitino() throws InterruptedException { "thrift://%s:%s", containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) - .put("hive.immutable-partitions", "true") - .put("hive.target-max-file-size", "1GB") - .put("hive.create-empty-bucket-files", "true") - .put("hive.validate-bucketing", "true") + .put("trino.bypass.hive.immutable-partitions", "true") + .put("trino.bypass.hive.target-max-file-size", "1GB") + .put("trino.bypass.hive.create-empty-bucket-files", "true") + .put("trino.bypass.hive.validate-bucketing", "true") .build()); Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); - Assertions.assertEquals("true", catalog.properties().get("hive.immutable-partitions")); - Assertions.assertEquals("1GB", catalog.properties().get("hive.target-max-file-size")); - Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); - Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.immutable-partitions")); + Assertions.assertEquals( + "1GB", catalog.properties().get("trino.bypass.hive.target-max-file-size")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.create-empty-bucket-files")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.validate-bucketing")); String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); @@ -863,17 +867,21 @@ void testWrongHiveCatalogProperty() throws InterruptedException { "thrift://%s:%s", containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) - .put("hive.immutable-partitions", "true") + .put("trino.bypass.hive.immutable-partitions", "true") // it should be like '1GB, 1MB', we make it wrong purposely. - .put("hive.target-max-file-size", "xxxx") - .put("hive.create-empty-bucket-files", "true") - .put("hive.validate-bucketing", "true") + .put("trino.bypass.hive.target-max-file-size", "xxxx") + .put("trino.bypass.hive.create-empty-bucket-files", "true") + .put("trino.bypass.hive.validate-bucketing", "true") .build()); Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); - Assertions.assertEquals("true", catalog.properties().get("hive.immutable-partitions")); - Assertions.assertEquals("xxxx", catalog.properties().get("hive.target-max-file-size")); - Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); - Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.immutable-partitions")); + Assertions.assertEquals( + "xxxx", catalog.properties().get("trino.bypass.hive.target-max-file-size")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.create-empty-bucket-files")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.validate-bucketing")); String sql = String.format("show catalogs like '%s'", catalogName); checkTrinoHasLoaded(sql, 6); diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql index 46d8b8c8034..7fe2ab1ec39 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql @@ -12,7 +12,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', map( - array['join-pushdown.strategy', 'test_key'], + array['trino.bypass.join-pushdown.strategy', 'test_key'], array['EAGER', 'test_value'] ) ); @@ -22,7 +22,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', map(), - array['join-pushdown.strategy'] + array['trino.bypass.join-pushdown.strategy'] ); select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; @@ -30,7 +30,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( catalog => 'gt_mysql_xxx1', set_properties => map( - array['join-pushdown.strategy'], + array['trino.bypass.join-pushdown.strategy'], array['EAGER'] ), remove_properties => array['test_key'] diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt index b0e3aac0dcc..35dab129f61 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt @@ -4,7 +4,7 @@ CALL CALL -"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL @@ -12,6 +12,6 @@ CALL CALL -"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" -CALL \ No newline at end of file +CALL diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql index eacf543b4ab..19c42e58ed3 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql @@ -2,7 +2,7 @@ call gravitino.system.create_catalog( 'gt_mysql', 'jdbc-mysql', map( - array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver', 'join-pushdown.strategy'], + array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver', 'trino.bypass.join-pushdown.strategy'], array['${mysql_uri}/?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver', 'EAGER'] ) ); \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql index d99ff9d8081..4872e799bd4 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql @@ -2,7 +2,7 @@ call gravitino.system.create_catalog( 'gt_postgresql', 'jdbc-postgresql', map( - array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-database', 'jdbc-driver', 'join-pushdown.strategy'], + array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-database', 'jdbc-driver', 'trino.bypass.join-pushdown.strategy'], array['${postgresql_uri}/gt_db', 'trino', 'ds123', 'gt_db', 'org.postgresql.Driver', 'EAGER'] ) ); \ No newline at end of file diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index ac1da69bfbe..d5541c9aed2 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -167,8 +167,7 @@ public void loadCatalogs(GravitinoMetalake metalake) { (NameIdentifier nameIdentifier) -> { try { Catalog catalog = metalake.loadCatalog(nameIdentifier); - GravitinoCatalog gravitinoCatalog = - new GravitinoCatalog(metalake.name(), catalog, config.simplifyCatalogNames()); + GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(metalake.name(), catalog); if (catalogConnectors.containsKey(getTrinoCatalogName(gravitinoCatalog))) { // Reload catalogs that have been updated in Gravitino server. reloadCatalog(metalake, gravitinoCatalog); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java index fec7317f523..4d675b977df 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java @@ -23,117 +23,204 @@ public class HiveCatalogPropertyConverter extends PropertyConverter { new ImmutableMap.Builder() // Key is the Trino property, value is the Gravitino property // General configuration - .put("hive.config.resources", "hive.config.resources") - .put("hive.recursive-directories", "hive.recursive-directories") - .put("hive.ignore-absent-partitions", "hive.ignore-absent-partitions") - .put("hive.storage-format", "hive.storage-format") - .put("hive.compression-codec", "hive.compression-codec") - .put("hive.force-local-scheduling", "hive.force-local-scheduling") - .put("hive.respect-table-format", "hive.respect-table-format") - .put("hive.immutable-partitions", "hive.immutable-partitions") + .put("hive.config.resources", TRINO_PROPERTIES_PREFIX + "hive.config.resources") + .put( + "hive.recursive-directories", + TRINO_PROPERTIES_PREFIX + "hive.recursive-directories") + .put( + "hive.ignore-absent-partitions", + TRINO_PROPERTIES_PREFIX + "hive.ignore-absent-partitions") + .put("hive.storage-format", TRINO_PROPERTIES_PREFIX + "hive.storage-format") + .put("hive.compression-codec", TRINO_PROPERTIES_PREFIX + "hive.compression-codec") + .put( + "hive.force-local-scheduling", + TRINO_PROPERTIES_PREFIX + "hive.force-local-scheduling") + .put( + "hive.respect-table-format", + TRINO_PROPERTIES_PREFIX + "hive.respect-table-format") + .put( + "hive.immutable-partitions", + TRINO_PROPERTIES_PREFIX + "hive.immutable-partitions") .put( "hive.insert-existing-partitions-behavior", - "hive.insert-existing-partitions-behavior") - .put("hive.target-max-file-size", "hive.target-max-file-size") - .put("hive.create-empty-bucket-files", "hive.create-empty-bucket-files") - .put("hive.validate-bucketing", "hive.validate-bucketing") - .put("hive.partition-statistics-sample-size", "hive.partition-statistics-sample-size") - .put("hive.max-partitions-per-writers", "hive.max-partitions-per-writers") - .put("hive.max-partitions-for-eager-load", "hive.max-partitions-for-eager-load") - .put("hive.max-partitions-per-scan", "hive.max-partitions-per-scan") - .put("hive.dfs.replication", "hive.dfs.replication") - .put("hive.security", "hive.security") - .put("security.config-file", "security.config-file") - .put("hive.non-managed-table-writes-enabled", "hive.non-managed-table-writes-enabled") + TRINO_PROPERTIES_PREFIX + "hive.insert-existing-partitions-behavior") + .put( + "hive.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "hive.target-max-file-size") + .put( + "hive.create-empty-bucket-files", + TRINO_PROPERTIES_PREFIX + "hive.create-empty-bucket-files") + .put("hive.validate-bucketing", TRINO_PROPERTIES_PREFIX + "hive.validate-bucketing") + .put( + "hive.partition-statistics-sample-size", + TRINO_PROPERTIES_PREFIX + "hive.partition-statistics-sample-size") + .put( + "hive.max-partitions-per-writers", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-writers") + .put( + "hive.max-partitions-for-eager-load", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-for-eager-load") + .put( + "hive.max-partitions-per-scan", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-scan") + .put("hive.dfs.replication", TRINO_PROPERTIES_PREFIX + "hive.dfs.replication") + .put("hive.security", TRINO_PROPERTIES_PREFIX + "hive.security") + .put("security.config-file", TRINO_PROPERTIES_PREFIX + "security.config-file") + .put( + "hive.non-managed-table-writes-enabled", + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-writes-enabled") .put( "hive.non-managed-table-creates-enabled", - "hive.non-managed-table-creates-enabled") + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-creates-enabled") .put( "hive.collect-column-statistics-on-write", - "hive.collect-column-statistics-on-write") - .put("hive.file-status-cache-tables", "hive.file-status-cache-tables") + TRINO_PROPERTIES_PREFIX + "hive.collect-column-statistics-on-write") + .put( + "hive.file-status-cache-tables", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-tables") .put( "hive.file-status-cache.max-retained-size", - "hive.file-status-cache.max-retained-size") - .put("hive.file-status-cache-expire-time", "hive.file-status-cache-expire-time") + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache.max-retained-size") + .put( + "hive.file-status-cache-expire-time", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-expire-time") .put( "hive.per-transaction-file-status-cache.max-retained-size", - "hive.per-transaction-file-status-cache.max-retained-size") - .put("hive.timestamp-precision", "hive.timestamp-precision") + TRINO_PROPERTIES_PREFIX + + "hive.per-transaction-file-status-cache.max-retained-size") + .put("hive.timestamp-precision", TRINO_PROPERTIES_PREFIX + "hive.timestamp-precision") .put( "hive.temporary-staging-directory-enabled", - "hive.temporary-staging-directory-enabled") - .put("hive.temporary-staging-directory-path", "hive.temporary-staging-directory-path") - .put("hive.hive-views.enabled", "hive.hive-views.enabled") - .put("hive.hive-views.legacy-translation", "hive.hive-views.legacy-translation") + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-enabled") + .put( + "hive.temporary-staging-directory-path", + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-path") + .put("hive.hive-views.enabled", TRINO_PROPERTIES_PREFIX + "hive.hive-views.enabled") + .put( + "hive.hive-views.legacy-translation", + TRINO_PROPERTIES_PREFIX + "hive.hive-views.legacy-translation") .put( "hive.parallel-partitioned-bucketed-writes", - "hive.parallel-partitioned-bucketed-writes") - .put("hive.fs.new-directory-permissions", "hive.fs.new-directory-permissions") - .put("hive.fs.cache.max-size", "hive.fs.cache.max-size") - .put("hive.query-partition-filter-required", "hive.query-partition-filter-required") - .put("hive.table-statistics-enabled", "hive.table-statistics-enabled") - .put("hive.auto-purge", "hive.auto-purge") - .put("hive.partition-projection-enabled", "hive.partition-projection-enabled") - .put("hive.max-partition-drops-per-query", "hive.max-partition-drops-per-query") - .put("hive.single-statement-writes", "hive.single-statement-writes") + TRINO_PROPERTIES_PREFIX + "hive.parallel-partitioned-bucketed-writes") + .put( + "hive.fs.new-directory-permissions", + TRINO_PROPERTIES_PREFIX + "hive.fs.new-directory-permissions") + .put("hive.fs.cache.max-size", TRINO_PROPERTIES_PREFIX + "hive.fs.cache.max-size") + .put( + "hive.query-partition-filter-required", + TRINO_PROPERTIES_PREFIX + "hive.query-partition-filter-required") + .put( + "hive.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "hive.table-statistics-enabled") + .put("hive.auto-purge", TRINO_PROPERTIES_PREFIX + "hive.auto-purge") + .put( + "hive.partition-projection-enabled", + TRINO_PROPERTIES_PREFIX + "hive.partition-projection-enabled") + .put( + "hive.max-partition-drops-per-query", + TRINO_PROPERTIES_PREFIX + "hive.max-partition-drops-per-query") + .put( + "hive.single-statement-writes", + TRINO_PROPERTIES_PREFIX + "hive.single-statement-writes") // Performance - .put("hive.max-outstanding-splits", "hive.max-outstanding-splits") - .put("hive.max-outstanding-splits-size", "hive.max-outstanding-splits-size") - .put("hive.max-splits-per-second", "hive.max-splits-per-second") - .put("hive.max-initial-splits", "hive.max-initial-splits") - .put("hive.max-initial-split-size", "hive.max-initial-split-size") - .put("hive.max-split-size", "hive.max-split-size") + .put( + "hive.max-outstanding-splits", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits") + .put( + "hive.max-outstanding-splits-size", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits-size") + .put( + "hive.max-splits-per-second", + TRINO_PROPERTIES_PREFIX + "hive.max-splits-per-second") + .put("hive.max-initial-splits", TRINO_PROPERTIES_PREFIX + "hive.max-initial-splits") + .put( + "hive.max-initial-split-size", + TRINO_PROPERTIES_PREFIX + "hive.max-initial-split-size") + .put("hive.max-split-size", TRINO_PROPERTIES_PREFIX + "hive.max-split-size") // S3 - .put("hive.s3.aws-access-key", "hive.s3.aws-access-key") - .put("hive.s3.aws-secret-key", "hive.s3.aws-secret-key") - .put("hive.s3.iam-role", "hive.s3.iam-role") - .put("hive.s3.external-id", "hive.s3.external-id") - .put("hive.s3.endpoint", "hive.s3.endpoint") - .put("hive.s3.region", "hive.s3.region") - .put("hive.s3.storage-class", "hive.s3.storage-class") - .put("hive.s3.signer-type", "hive.s3.signer-type") - .put("hive.s3.signer-class", "hive.s3.signer-class") - .put("hive.s3.path-style-access", "hive.s3.path-style-access") - .put("hive.s3.staging-directory", "hive.s3.staging-directory") - .put("hive.s3.pin-client-to-current-region", "hive.s3.pin-client-to-current-region") - .put("hive.s3.ssl.enabled", "hive.s3.ssl.enabled") - .put("hive.s3.sse.enabled", "hive.s3.sse.enabled") - .put("hive.s3.sse.type", "hive.s3.sse.type") - .put("hive.s3.sse.kms-key-id", "hive.s3.sse.kms-key-id") - .put("hive.s3.kms-key-id", "hive.s3.kms-key-id") - .put("hive.s3.encryption-materials-provider", "hive.s3.encryption-materials-provider") - .put("hive.s3.upload-acl-type", "hive.s3.upload-acl-type") - .put("hive.s3.skip-glacier-objects", "hive.s3.skip-glacier-objects") - .put("hive.s3.streaming.enabled", "hive.s3.streaming.enabled") - .put("hive.s3.streaming.part-size", "hive.s3.streaming.part-size") - .put("hive.s3.proxy.host", "hive.s3.proxy.host") - .put("hive.s3.proxy.port", "hive.s3.proxy.port") - .put("hive.s3.proxy.protocol", "hive.s3.proxy.protocol") - .put("hive.s3.proxy.non-proxy-hosts", "hive.s3.proxy.non-proxy-hosts") - .put("hive.s3.proxy.username", "hive.s3.proxy.username") - .put("hive.s3.proxy.password", "hive.s3.proxy.password") - .put("hive.s3.proxy.preemptive-basic-auth", "hive.s3.proxy.preemptive-basic-auth") - .put("hive.s3.sts.endpoint", "hive.s3.sts.endpoint") - .put("hive.s3.sts.region", "hive.s3.sts.region") + .put("hive.s3.aws-access-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-access-key") + .put("hive.s3.aws-secret-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-secret-key") + .put("hive.s3.iam-role", TRINO_PROPERTIES_PREFIX + "hive.s3.iam-role") + .put("hive.s3.external-id", TRINO_PROPERTIES_PREFIX + "hive.s3.external-id") + .put("hive.s3.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.endpoint") + .put("hive.s3.region", TRINO_PROPERTIES_PREFIX + "hive.s3.region") + .put("hive.s3.storage-class", TRINO_PROPERTIES_PREFIX + "hive.s3.storage-class") + .put("hive.s3.signer-type", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-type") + .put("hive.s3.signer-class", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-class") + .put( + "hive.s3.path-style-access", + TRINO_PROPERTIES_PREFIX + "hive.s3.path-style-access") + .put( + "hive.s3.staging-directory", + TRINO_PROPERTIES_PREFIX + "hive.s3.staging-directory") + .put( + "hive.s3.pin-client-to-current-region", + TRINO_PROPERTIES_PREFIX + "hive.s3.pin-client-to-current-region") + .put("hive.s3.ssl.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.ssl.enabled") + .put("hive.s3.sse.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.enabled") + .put("hive.s3.sse.type", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.type") + .put("hive.s3.sse.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.kms-key-id") + .put("hive.s3.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.kms-key-id") + .put( + "hive.s3.encryption-materials-provider", + TRINO_PROPERTIES_PREFIX + "hive.s3.encryption-materials-provider") + .put("hive.s3.upload-acl-type", TRINO_PROPERTIES_PREFIX + "hive.s3.upload-acl-type") + .put( + "hive.s3.skip-glacier-objects", + TRINO_PROPERTIES_PREFIX + "hive.s3.skip-glacier-objects") + .put( + "hive.s3.streaming.enabled", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.enabled") + .put( + "hive.s3.streaming.part-size", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.part-size") + .put("hive.s3.proxy.host", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.host") + .put("hive.s3.proxy.port", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.port") + .put("hive.s3.proxy.protocol", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.protocol") + .put( + "hive.s3.proxy.non-proxy-hosts", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.non-proxy-hosts") + .put("hive.s3.proxy.username", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.username") + .put("hive.s3.proxy.password", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.password") + .put( + "hive.s3.proxy.preemptive-basic-auth", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.preemptive-basic-auth") + .put("hive.s3.sts.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.endpoint") + .put("hive.s3.sts.region", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.region") // Hive metastore Thrift service authentication - .put("hive.metastore.authentication.type", "hive.metastore.authentication.type") + .put( + "hive.metastore.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.metastore.authentication.type") .put( "hive.metastore.thrift.impersonation.enabled", - "hive.metastore.thrift.impersonation.enabled") - .put("hive.metastore.service.principal", "hive.metastore.service.principal") - .put("hive.metastore.client.principal", "hive.metastore.client.principal") - .put("hive.metastore.client.keytab", "hive.metastore.client.keytab") + TRINO_PROPERTIES_PREFIX + "hive.metastore.thrift.impersonation.enabled") + .put( + "hive.metastore.service.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.service.principal") + .put( + "hive.metastore.client.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.principal") + .put( + "hive.metastore.client.keytab", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.keytab") // HDFS authentication - .put("hive.hdfs.authentication.type", "hive.hdfs.authentication.type") - .put("hive.hdfs.impersonation.enabled", "hive.hdfs.impersonation.enabled") - .put("hive.hdfs.trino.principal", "hive.hdfs.trino.principal") - .put("hive.hdfs.trino.keytab", "hive.hdfs.trino.keytab") - .put("hive.hdfs.wire-encryption.enabled", "hive.hdfs.wire-encryption.enabled") + .put( + "hive.hdfs.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.authentication.type") + .put( + "hive.hdfs.impersonation.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.impersonation.enabled") + .put( + "hive.hdfs.trino.principal", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.principal") + .put("hive.hdfs.trino.keytab", TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.keytab") + .put( + "hive.hdfs.wire-encryption.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.wire-encryption.enabled") .build()); @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 3ada2983b94..9d42ce80342 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.HasPropertyMeta; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; -import com.google.common.collect.Maps; import io.trino.spi.session.PropertyMetadata; import java.util.Collections; import java.util.HashMap; @@ -44,18 +43,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog properties.put("hive.security", "allow-all"); Map trinoProperty = catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - - // Trino only supports properties that define in catalogPropertyMeta, the name of entries in - // catalogPropertyMeta is in the format of "catalogName_propertyName", so we need to replace - // '_' with '.' to align with the name in trino. - Map> catalogPropertyMeta = - Maps.uniqueIndex( - propertyMetadata.getCatalogPropertyMeta(), - propertyMetadata -> propertyMetadata.getName().replace("_", ".")); - - trinoProperty.entrySet().stream() - .filter(entry -> catalogPropertyMeta.containsKey(entry.getKey())) - .forEach(entry -> properties.put(entry.getKey(), entry.getValue())); + properties.putAll(trinoProperty); config.put("properties", properties); return config; diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java index 4be199546f9..74397799a40 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java @@ -22,152 +22,259 @@ public class IcebergCatalogPropertyConverter extends PropertyConverter { new TreeBidiMap<>( new ImmutableMap.Builder() // General configuration - .put("iceberg.catalog.type", "iceberg.catalog.type") - .put("iceberg.file-format", "iceberg.file-format") - .put("iceberg.compression-codec", "iceberg.compression-codec") - .put("iceberg.use-file-size-from-metadata", "iceberg.use-file-size-from-metadata") - .put("iceberg.max-partitions-per-writer", "iceberg.max-partitions-per-writer") - .put("iceberg.target-max-file-size", "iceberg.target-max-file-size") - .put("iceberg.unique-table-location", "iceberg.unique-table-location") + .put("iceberg.catalog.type", TRINO_PROPERTIES_PREFIX + "iceberg.catalog.type") + .put("iceberg.file-format", TRINO_PROPERTIES_PREFIX + "iceberg.file-format") + .put( + "iceberg.compression-codec", + TRINO_PROPERTIES_PREFIX + "iceberg.compression-codec") + .put( + "iceberg.use-file-size-from-metadata", + TRINO_PROPERTIES_PREFIX + "iceberg.use-file-size-from-metadata") + .put( + "iceberg.max-partitions-per-writer", + TRINO_PROPERTIES_PREFIX + "iceberg.max-partitions-per-writer") + .put( + "iceberg.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "iceberg.target-max-file-size") + .put( + "iceberg.unique-table-location", + TRINO_PROPERTIES_PREFIX + "iceberg.unique-table-location") .put( "iceberg.dynamic-filtering.wait-timeout", - "iceberg.dynamic-filtering.wait-timeout") + TRINO_PROPERTIES_PREFIX + "iceberg.dynamic-filtering.wait-timeout") .put( "iceberg.delete-schema-locations-fallback", - "iceberg.delete-schema-locations-fallback") - .put("iceberg.minimum-assigned-split-weight", "iceberg.minimum-assigned-split-weight") - .put("iceberg.table-statistics-enabled", "iceberg.table-statistics-enabled") - .put("iceberg.extended-statistics.enabled", "iceberg.extended-statistics.enabled") + TRINO_PROPERTIES_PREFIX + "iceberg.delete-schema-locations-fallback") + .put( + "iceberg.minimum-assigned-split-weight", + TRINO_PROPERTIES_PREFIX + "iceberg.minimum-assigned-split-weight") + .put( + "iceberg.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.table-statistics-enabled") + .put( + "iceberg.extended-statistics.enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.extended-statistics.enabled") .put( "iceberg.extended-statistics.collect-on-write", - "iceberg.extended-statistics.collect-on-write") - .put("iceberg.projection-pushdown-enabled", "iceberg.projection-pushdown-enabled") - .put("iceberg.hive-catalog-name", "iceberg.hive-catalog-name") + TRINO_PROPERTIES_PREFIX + "iceberg.extended-statistics.collect-on-write") + .put( + "iceberg.projection-pushdown-enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.projection-pushdown-enabled") + .put( + "iceberg.hive-catalog-name", + TRINO_PROPERTIES_PREFIX + "iceberg.hive-catalog-name") .put( "iceberg.materialized-views.storage-schema", - "iceberg.materialized-views.storage-schema") + TRINO_PROPERTIES_PREFIX + "iceberg.materialized-views.storage-schema") .put( "iceberg.materialized-views.hide-storage-table", - "iceberg.materialized-views.hide-storage-table") + TRINO_PROPERTIES_PREFIX + "iceberg.materialized-views.hide-storage-table") .put( "iceberg.register-table-procedure.enabled", - "iceberg.register-table-procedure.enabled") + TRINO_PROPERTIES_PREFIX + "iceberg.register-table-procedure.enabled") .put( "iceberg.query-partition-filter-required", - "iceberg.query-partition-filter-required") + TRINO_PROPERTIES_PREFIX + "iceberg.query-partition-filter-required") // Hive - .put("hive.config.resources", "hive.config.resources") - .put("hive.recursive-directories", "hive.recursive-directories") - .put("hive.ignore-absent-partitions", "hive.ignore-absent-partitions") - .put("hive.storage-format", "hive.storage-format") - .put("hive.compression-codec", "hive.compression-codec") - .put("hive.force-local-scheduling", "hive.force-local-scheduling") - .put("hive.respect-table-format", "hive.respect-table-format") - .put("hive.immutable-partitions", "hive.immutable-partitions") + .put("hive.config.resources", TRINO_PROPERTIES_PREFIX + "hive.config.resources") + .put( + "hive.recursive-directories", + TRINO_PROPERTIES_PREFIX + "hive.recursive-directories") + .put( + "hive.ignore-absent-partitions", + TRINO_PROPERTIES_PREFIX + "hive.ignore-absent-partitions") + .put("hive.storage-format", TRINO_PROPERTIES_PREFIX + "hive.storage-format") + .put("hive.compression-codec", TRINO_PROPERTIES_PREFIX + "hive.compression-codec") + .put( + "hive.force-local-scheduling", + TRINO_PROPERTIES_PREFIX + "hive.force-local-scheduling") + .put( + "hive.respect-table-format", + TRINO_PROPERTIES_PREFIX + "hive.respect-table-format") + .put( + "hive.immutable-partitions", + TRINO_PROPERTIES_PREFIX + "hive.immutable-partitions") .put( "hive.insert-existing-partitions-behavior", - "hive.insert-existing-partitions-behavior") - .put("hive.target-max-file-size", "hive.target-max-file-size") - .put("hive.create-empty-bucket-files", "hive.create-empty-bucket-files") - .put("hive.validate-bucketing", "hive.validate-bucketing") - .put("hive.partition-statistics-sample-size", "hive.partition-statistics-sample-size") - .put("hive.max-partitions-per-writers", "hive.max-partitions-per-writers") - .put("hive.max-partitions-for-eager-load", "hive.max-partitions-for-eager-load") - .put("hive.max-partitions-per-scan", "hive.max-partitions-per-scan") - .put("hive.dfs.replication", "hive.dfs.replication") - .put("hive.security", "hive.security") - .put("security.config-file", "security.config-file") - .put("hive.non-managed-table-writes-enabled", "hive.non-managed-table-writes-enabled") + TRINO_PROPERTIES_PREFIX + "hive.insert-existing-partitions-behavior") + .put( + "hive.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "hive.target-max-file-size") + .put( + "hive.create-empty-bucket-files", + TRINO_PROPERTIES_PREFIX + "hive.create-empty-bucket-files") + .put("hive.validate-bucketing", TRINO_PROPERTIES_PREFIX + "hive.validate-bucketing") + .put( + "hive.partition-statistics-sample-size", + TRINO_PROPERTIES_PREFIX + "hive.partition-statistics-sample-size") + .put( + "hive.max-partitions-per-writers", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-writers") + .put( + "hive.max-partitions-for-eager-load", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-for-eager-load") + .put( + "hive.max-partitions-per-scan", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-scan") + .put("hive.dfs.replication", TRINO_PROPERTIES_PREFIX + "hive.dfs.replication") + .put("hive.security", TRINO_PROPERTIES_PREFIX + "hive.security") + .put("security.config-file", TRINO_PROPERTIES_PREFIX + "security.config-file") + .put( + "hive.non-managed-table-writes-enabled", + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-writes-enabled") .put( "hive.non-managed-table-creates-enabled", - "hive.non-managed-table-creates-enabled") + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-creates-enabled") .put( "hive.collect-column-statistics-on-write", - "hive.collect-column-statistics-on-write") - .put("hive.file-status-cache-tables", "hive.file-status-cache-tables") + TRINO_PROPERTIES_PREFIX + "hive.collect-column-statistics-on-write") + .put( + "hive.file-status-cache-tables", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-tables") .put( "hive.file-status-cache.max-retained-size", - "hive.file-status-cache.max-retained-size") - .put("hive.file-status-cache-expire-time", "hive.file-status-cache-expire-time") + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache.max-retained-size") + .put( + "hive.file-status-cache-expire-time", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-expire-time") .put( "hive.per-transaction-file-status-cache.max-retained-size", - "hive.per-transaction-file-status-cache.max-retained-size") - .put("hive.timestamp-precision", "hive.timestamp-precision") + TRINO_PROPERTIES_PREFIX + + "hive.per-transaction-file-status-cache.max-retained-size") + .put("hive.timestamp-precision", TRINO_PROPERTIES_PREFIX + "hive.timestamp-precision") .put( "hive.temporary-staging-directory-enabled", - "hive.temporary-staging-directory-enabled") - .put("hive.temporary-staging-directory-path", "hive.temporary-staging-directory-path") - .put("hive.hive-views.enabled", "hive.hive-views.enabled") - .put("hive.hive-views.legacy-translation", "hive.hive-views.legacy-translation") + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-enabled") + .put( + "hive.temporary-staging-directory-path", + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-path") + .put("hive.hive-views.enabled", TRINO_PROPERTIES_PREFIX + "hive.hive-views.enabled") + .put( + "hive.hive-views.legacy-translation", + TRINO_PROPERTIES_PREFIX + "hive.hive-views.legacy-translation") .put( "hive.parallel-partitioned-bucketed-writes", - "hive.parallel-partitioned-bucketed-writes") - .put("hive.fs.new-directory-permissions", "hive.fs.new-directory-permissions") - .put("hive.fs.cache.max-size", "hive.fs.cache.max-size") - .put("hive.query-partition-filter-required", "hive.query-partition-filter-required") - .put("hive.table-statistics-enabled", "hive.table-statistics-enabled") - .put("hive.auto-purge", "hive.auto-purge") - .put("hive.partition-projection-enabled", "hive.partition-projection-enabled") - .put("hive.max-partition-drops-per-query", "hive.max-partition-drops-per-query") - .put("hive.single-statement-writes", "hive.single-statement-writes") + TRINO_PROPERTIES_PREFIX + "hive.parallel-partitioned-bucketed-writes") + .put( + "hive.fs.new-directory-permissions", + TRINO_PROPERTIES_PREFIX + "hive.fs.new-directory-permissions") + .put("hive.fs.cache.max-size", TRINO_PROPERTIES_PREFIX + "hive.fs.cache.max-size") + .put( + "hive.query-partition-filter-required", + TRINO_PROPERTIES_PREFIX + "hive.query-partition-filter-required") + .put( + "hive.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "hive.table-statistics-enabled") + .put("hive.auto-purge", TRINO_PROPERTIES_PREFIX + "hive.auto-purge") + .put( + "hive.partition-projection-enabled", + TRINO_PROPERTIES_PREFIX + "hive.partition-projection-enabled") + .put( + "hive.max-partition-drops-per-query", + TRINO_PROPERTIES_PREFIX + "hive.max-partition-drops-per-query") + .put( + "hive.single-statement-writes", + TRINO_PROPERTIES_PREFIX + "hive.single-statement-writes") // Hive performance - .put("hive.max-outstanding-splits", "hive.max-outstanding-splits") - .put("hive.max-outstanding-splits-size", "hive.max-outstanding-splits-size") - .put("hive.max-splits-per-second", "hive.max-splits-per-second") - .put("hive.max-initial-splits", "hive.max-initial-splits") - .put("hive.max-initial-split-size", "hive.max-initial-split-size") - .put("hive.max-split-size", "hive.max-split-size") + .put( + "hive.max-outstanding-splits", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits") + .put( + "hive.max-outstanding-splits-size", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits-size") + .put( + "hive.max-splits-per-second", + TRINO_PROPERTIES_PREFIX + "hive.max-splits-per-second") + .put("hive.max-initial-splits", TRINO_PROPERTIES_PREFIX + "hive.max-initial-splits") + .put( + "hive.max-initial-split-size", + TRINO_PROPERTIES_PREFIX + "hive.max-initial-split-size") + .put("hive.max-split-size", TRINO_PROPERTIES_PREFIX + "hive.max-split-size") // S3 - .put("hive.s3.aws-access-key", "hive.s3.aws-access-key") - .put("hive.s3.aws-secret-key", "hive.s3.aws-secret-key") - .put("hive.s3.iam-role", "hive.s3.iam-role") - .put("hive.s3.external-id", "hive.s3.external-id") - .put("hive.s3.endpoint", "hive.s3.endpoint") - .put("hive.s3.region", "hive.s3.region") - .put("hive.s3.storage-class", "hive.s3.storage-class") - .put("hive.s3.signer-type", "hive.s3.signer-type") - .put("hive.s3.signer-class", "hive.s3.signer-class") - .put("hive.s3.path-style-access", "hive.s3.path-style-access") - .put("hive.s3.staging-directory", "hive.s3.staging-directory") - .put("hive.s3.pin-client-to-current-region", "hive.s3.pin-client-to-current-region") - .put("hive.s3.ssl.enabled", "hive.s3.ssl.enabled") - .put("hive.s3.sse.enabled", "hive.s3.sse.enabled") - .put("hive.s3.sse.type", "hive.s3.sse.type") - .put("hive.s3.sse.kms-key-id", "hive.s3.sse.kms-key-id") - .put("hive.s3.kms-key-id", "hive.s3.kms-key-id") - .put("hive.s3.encryption-materials-provider", "hive.s3.encryption-materials-provider") - .put("hive.s3.upload-acl-type", "hive.s3.upload-acl-type") - .put("hive.s3.skip-glacier-objects", "hive.s3.skip-glacier-objects") - .put("hive.s3.streaming.enabled", "hive.s3.streaming.enabled") - .put("hive.s3.streaming.part-size", "hive.s3.streaming.part-size") - .put("hive.s3.proxy.host", "hive.s3.proxy.host") - .put("hive.s3.proxy.port", "hive.s3.proxy.port") - .put("hive.s3.proxy.protocol", "hive.s3.proxy.protocol") - .put("hive.s3.proxy.non-proxy-hosts", "hive.s3.proxy.non-proxy-hosts") - .put("hive.s3.proxy.username", "hive.s3.proxy.username") - .put("hive.s3.proxy.password", "hive.s3.proxy.password") - .put("hive.s3.proxy.preemptive-basic-auth", "hive.s3.proxy.preemptive-basic-auth") - .put("hive.s3.sts.endpoint", "hive.s3.sts.endpoint") - .put("hive.s3.sts.region", "hive.s3.sts.region") + .put("hive.s3.aws-access-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-access-key") + .put("hive.s3.aws-secret-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-secret-key") + .put("hive.s3.iam-role", TRINO_PROPERTIES_PREFIX + "hive.s3.iam-role") + .put("hive.s3.external-id", TRINO_PROPERTIES_PREFIX + "hive.s3.external-id") + .put("hive.s3.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.endpoint") + .put("hive.s3.region", TRINO_PROPERTIES_PREFIX + "hive.s3.region") + .put("hive.s3.storage-class", TRINO_PROPERTIES_PREFIX + "hive.s3.storage-class") + .put("hive.s3.signer-type", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-type") + .put("hive.s3.signer-class", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-class") + .put( + "hive.s3.path-style-access", + TRINO_PROPERTIES_PREFIX + "hive.s3.path-style-access") + .put( + "hive.s3.staging-directory", + TRINO_PROPERTIES_PREFIX + "hive.s3.staging-directory") + .put( + "hive.s3.pin-client-to-current-region", + TRINO_PROPERTIES_PREFIX + "hive.s3.pin-client-to-current-region") + .put("hive.s3.ssl.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.ssl.enabled") + .put("hive.s3.sse.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.enabled") + .put("hive.s3.sse.type", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.type") + .put("hive.s3.sse.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.kms-key-id") + .put("hive.s3.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.kms-key-id") + .put( + "hive.s3.encryption-materials-provider", + TRINO_PROPERTIES_PREFIX + "hive.s3.encryption-materials-provider") + .put("hive.s3.upload-acl-type", TRINO_PROPERTIES_PREFIX + "hive.s3.upload-acl-type") + .put( + "hive.s3.skip-glacier-objects", + TRINO_PROPERTIES_PREFIX + "hive.s3.skip-glacier-objects") + .put( + "hive.s3.streaming.enabled", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.enabled") + .put( + "hive.s3.streaming.part-size", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.part-size") + .put("hive.s3.proxy.host", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.host") + .put("hive.s3.proxy.port", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.port") + .put("hive.s3.proxy.protocol", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.protocol") + .put( + "hive.s3.proxy.non-proxy-hosts", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.non-proxy-hosts") + .put("hive.s3.proxy.username", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.username") + .put("hive.s3.proxy.password", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.password") + .put( + "hive.s3.proxy.preemptive-basic-auth", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.preemptive-basic-auth") + .put("hive.s3.sts.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.endpoint") + .put("hive.s3.sts.region", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.region") // Hive metastore Thrift service authentication - .put("hive.metastore.authentication.type", "hive.metastore.authentication.type") + .put( + "hive.metastore.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.metastore.authentication.type") .put( "hive.metastore.thrift.impersonation.enabled", - "hive.metastore.thrift.impersonation.enabled") - .put("hive.metastore.service.principal", "hive.metastore.service.principal") - .put("hive.metastore.client.principal", "hive.metastore.client.principal") - .put("hive.metastore.client.keytab", "hive.metastore.client.keytab") + TRINO_PROPERTIES_PREFIX + "hive.metastore.thrift.impersonation.enabled") + .put( + "hive.metastore.service.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.service.principal") + .put( + "hive.metastore.client.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.principal") + .put( + "hive.metastore.client.keytab", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.keytab") // HDFS authentication - .put("hive.hdfs.authentication.type", "hive.hdfs.authentication.type") - .put("hive.hdfs.impersonation.enabled", "hive.hdfs.impersonation.enabled") - .put("hive.hdfs.trino.principal", "hive.hdfs.trino.principal") - .put("hive.hdfs.trino.keytab", "hive.hdfs.trino.keytab") - .put("hive.hdfs.wire-encryption.enabled", "hive.hdfs.wire-encryption.enabled") + .put( + "hive.hdfs.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.authentication.type") + .put( + "hive.hdfs.impersonation.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.impersonation.enabled") + .put( + "hive.hdfs.trino.principal", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.principal") + .put("hive.hdfs.trino.keytab", TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.keytab") + .put( + "hive.hdfs.wire-encryption.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.wire-encryption.enabled") .build()); private static final Set JDBC_BACKEND_REQUIRED_PROPERTIES = @@ -182,15 +289,20 @@ public TreeBidiMap engineToGravitinoMapping() { @Override public Map gravitinoToEngineProperties(Map properties) { + Map stringStringMap; String backend = properties.get("catalog-backend"); switch (backend) { case "hive": - return buildHiveBackendProperties(properties); + stringStringMap = buildHiveBackendProperties(properties); + break; case "jdbc": - return buildJDBCBackendProperties(properties); + stringStringMap = buildJDBCBackendProperties(properties); + break; default: throw new UnsupportedOperationException("Unsupported backend type: " + backend); } + stringStringMap.putAll(super.gravitinoToEngineProperties(properties)); + return stringStringMap; } private Map buildHiveBackendProperties(Map properties) { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java index 5cc46c5a57a..4ce02a73e6c 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java @@ -28,42 +28,67 @@ public class JDBCCatalogPropertyConverter extends PropertyConverter { // Data source authentication .put(JDBC_CONNECTION_USER_KEY, "jdbc-user") .put(JDBC_CONNECTION_PASSWORD_KEY, "jdbc-password") - .put("credential-provider.type", "credential-provider.type") - .put("user-credential-name", "user-credential-name") - .put("password-credential-name", "password-credential-name") - .put("connection-credential-file", "connection-credential-file") - .put("keystore-file-path", "keystore-file-path") - .put("keystore-type", "keystore-type") - .put("keystore-password", "keystore-password") - .put("keystore-user-credential-name", "keystore-user-credential-name") - .put("keystore-user-credential-password", "keystore-user-credential-password") - .put("keystore-password-credential-name", "keystore-password-credential-name") - .put("keystore-password-credential-password", "keystore-password-credential-password") + .put("credential-provider.type", TRINO_PROPERTIES_PREFIX + "credential-provider.type") + .put("user-credential-name", TRINO_PROPERTIES_PREFIX + "user-credential-name") + .put("password-credential-name", TRINO_PROPERTIES_PREFIX + "password-credential-name") + .put( + "connection-credential-file", + TRINO_PROPERTIES_PREFIX + "connection-credential-file") + .put("keystore-file-path", TRINO_PROPERTIES_PREFIX + "keystore-file-path") + .put("keystore-type", TRINO_PROPERTIES_PREFIX + "keystore-type") + .put("keystore-password", TRINO_PROPERTIES_PREFIX + "keystore-password") + .put( + "keystore-user-credential-name", + TRINO_PROPERTIES_PREFIX + "keystore-user-credential-name") + .put( + "keystore-user-credential-password", + TRINO_PROPERTIES_PREFIX + "keystore-user-credential-password") + .put( + "keystore-password-credential-name", + TRINO_PROPERTIES_PREFIX + "keystore-password-credential-name") + .put( + "keystore-password-credential-password", + TRINO_PROPERTIES_PREFIX + "keystore-password-credential-password") // General configuration properties - .put("case-insensitive-name-matching", "ase-insensitive-name-matching") + .put( + "case-insensitive-name-matching", + TRINO_PROPERTIES_PREFIX + "ase-insensitive-name-matching") .put( "case-insensitive-name-matching.cache-ttl", - "case-insensitive-name-matching.cache-ttl") + TRINO_PROPERTIES_PREFIX + "case-insensitive-name-matching.cache-ttl") .put( "case-insensitive-name-matching.config-file", - "case-insensitive-name-matching.config-file") + TRINO_PROPERTIES_PREFIX + "case-insensitive-name-matching.config-file") .put( "case-insensitive-name-matching.config-file.refresh-period", - "case-insensitive-name-matching.config-file.refresh-period") - .put("metadata.cache-ttl", "metadata.cache-ttl") - .put("metadata.cache-missing", "metadata.cache-missing") - .put("metadata.schemas.cache-ttl", "metadata.schemas.cache-ttl") - .put("metadata.tables.cache-ttl", "metadata.tables.cache-ttl") - .put("metadata.statistics.cache-ttl", "metadata.statistics.cache-ttl") - .put("metadata.cache-maximum-size", "metadata.cache-maximum-size") - .put("write.batch-size", "write.batch-size") - .put("dynamic-filtering.enabled", "dynamic-filtering.enabled") - .put("dynamic-filtering.wait-timeout", "dynamic-filtering.wait-timeout") + TRINO_PROPERTIES_PREFIX + + "case-insensitive-name-matching.config-file.refresh-period") + .put("metadata.cache-ttl", TRINO_PROPERTIES_PREFIX + "metadata.cache-ttl") + .put("metadata.cache-missing", TRINO_PROPERTIES_PREFIX + "metadata.cache-missing") + .put( + "metadata.schemas.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.schemas.cache-ttl") + .put( + "metadata.tables.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.tables.cache-ttl") + .put( + "metadata.statistics.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.statistics.cache-ttl") + .put( + "metadata.cache-maximum-size", + TRINO_PROPERTIES_PREFIX + "metadata.cache-maximum-size") + .put("write.batch-size", TRINO_PROPERTIES_PREFIX + "write.batch-size") + .put( + "dynamic-filtering.enabled", + TRINO_PROPERTIES_PREFIX + "dynamic-filtering.enabled") + .put( + "dynamic-filtering.wait-timeout", + TRINO_PROPERTIES_PREFIX + "dynamic-filtering.wait-timeout") // Performance - .put("join-pushdown.enabled", "join-pushdown.enabled") - .put("join-pushdown.strategy", "join-pushdown.strategy") + .put("join-pushdown.enabled", TRINO_PROPERTIES_PREFIX + "join-pushdown.enabled") + .put("join-pushdown.strategy", TRINO_PROPERTIES_PREFIX + "join-pushdown.strategy") .build()); public static final Set REQUIRED_PROPERTIES = diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java index 4dd61b1c368..2157891d939 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -19,7 +19,7 @@ public class GravitinoCatalog { private final String metalake; private final Catalog catalog; - public GravitinoCatalog(String metalake, Catalog catalog, boolean usingSimpleName) { + public GravitinoCatalog(String metalake, Catalog catalog) { this.metalake = metalake; this.catalog = catalog; } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 66d1fa9821b..86fc466389c 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -61,12 +61,10 @@ public class GravitinoMockServer implements AutoCloseable { private final Map metalakes = new HashMap<>(); private boolean start = true; - private boolean simpleCatalogName; CatalogConnectorManager catalogConnectorManager; private GeneralDataTypeTransformer dataTypeTransformer = new HiveDataTypeTransformer(); - public GravitinoMockServer(boolean simpleCatalogName) { - this.simpleCatalogName = simpleCatalogName; + public GravitinoMockServer() { createMetalake(NameIdentifier.ofMetalake(testMetalake)); createCatalog(NameIdentifier.ofCatalog(testMetalake, testCatalog)); } @@ -214,8 +212,7 @@ private Catalog createCatalog(NameIdentifier catalogName) { when(mockAudit.createTime()).thenReturn(Instant.now()); when(catalog.auditInfo()).thenReturn(mockAudit); - GravitinoCatalog gravitinoCatalog = - new GravitinoCatalog(testMetalake, catalog, simpleCatalogName); + GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(testMetalake, catalog); when(catalog.asTableCatalog()).thenAnswer(answer -> createTableCatalog(gravitinoCatalog)); when(catalog.asSchemas()).thenAnswer(answer -> createSchemas(gravitinoCatalog)); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java index 5f6aa375830..bd71b021bac 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java @@ -20,7 +20,7 @@ public class TestCreateGravitinoConnector { @Test public void testCreateConnectorsWithEnableSimpleCatalog() throws Exception { - server = new GravitinoMockServer(true); + server = new GravitinoMockServer(); Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); @@ -56,7 +56,7 @@ public void testCreateConnectorsWithEnableSimpleCatalog() throws Exception { @Test public void testCreateConnectorsWithDisableSimpleCatalog() throws Exception { - server = new GravitinoMockServer(false); + server = new GravitinoMockServer(); Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index 93fb75da8c7..00fa2004b32 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -35,7 +35,7 @@ public class TestGravitinoConnector extends AbstractTestQueryFramework { @Override protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer(true)); + server = closeAfterClass(new GravitinoMockServer()); GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java index ed9d4457a0e..a7691c65cd6 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java @@ -32,7 +32,7 @@ public class TestGravitinoConnectorWithMetalakeCatalogName extends AbstractTestQ @Override protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer(false)); + server = closeAfterClass(new GravitinoMockServer()); GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java index d07f7661f9d..124762aefa5 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java @@ -5,7 +5,10 @@ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import com.google.common.collect.Sets; import java.util.Map; import java.util.Set; @@ -21,9 +24,9 @@ public void testConverter() { HiveCatalogPropertyConverter hiveCatalogPropertyConverter = new HiveCatalogPropertyConverter(); Map map = ImmutableMap.builder() - .put("hive.immutable-partitions", "true") - .put("hive.compression-codec", "ZSTD") - .put("hive.unknown-key", "1") + .put("trino.bypass.hive.immutable-partitions", "true") + .put("trino.bypass.hive.compression-codec", "ZSTD") + .put("trino.bypass.hive.unknown-key", "1") .build(); Map re = hiveCatalogPropertyConverter.gravitinoToEngineProperties(map); @@ -44,4 +47,42 @@ public void testPropertyMetadata() { gravitinoHiveKeys.remove("external"); Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("metastore.uris", "thrift://localhost:9083") + .put("hive.unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.hive.config.resources", "/tmp/hive-site.xml, /tmp/core-site.xml") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "hive", "test catalog", Catalog.Type.RELATIONAL, properties); + HiveConnectorAdapter adapter = new HiveConnectorAdapter(); + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "hive"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); + + // test fixed properties + Assert.assertEquals(propertiesMap.get("hive.security"), "allow-all"); + + // test trino passing properties + Assert.assertEquals( + propertiesMap.get("hive.config.resources"), "/tmp/hive-site.xml, /tmp/core-site.xml"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java index 52ba989616e..e4c072fef2b 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java @@ -5,8 +5,11 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata; import com.datastrato.gravitino.catalog.property.PropertyConverter; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import com.google.common.collect.Sets; import io.trino.spi.TrinoException; import java.util.Map; @@ -85,4 +88,89 @@ public void testPropertyMetadata() { Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorPropertiesWithHiveBackend() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("uri", "thrift://localhost:9083") + .put("catalog-backend", "hive") + .put("warehouse", "hdfs://tmp/warehouse") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.iceberg.table-statistics-enabled", "true") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); + IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "hive_metastore"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorPropertiesWithMySqlBackEnd() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("uri", "jdbc:mysql://%s:3306/metastore_db?createDatabaseIfNotExist=true") + .put("catalog-backend", "jdbc") + .put("warehouse", "://tmp/warehouse") + .put("jdbc-user", "root") + .put("jdbc-password", "ds123") + .put("jdbc-driver", "com.mysql.cj.jdbc.Driver") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.iceberg.table-statistics-enabled", "true") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); + IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.connection-url"), + "jdbc:mysql://%s:3306/metastore_db?createDatabaseIfNotExist=true"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-user"), "root"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-password"), "ds123"); + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.driver-class"), "com.mysql.cj.jdbc.Driver"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "jdbc"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java index ee9f10446d5..02320854041 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java @@ -9,7 +9,12 @@ import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_URL_KEY; import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_USER_KEY; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.property.PropertyConverter; +import com.datastrato.gravitino.trino.connector.catalog.jdbc.mysql.MySQLConnectorAdapter; +import com.datastrato.gravitino.trino.connector.catalog.jdbc.postgresql.PostgreSQLConnectorAdapter; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import java.util.Map; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; @@ -44,4 +49,83 @@ public void testTrinoPropertyKeyToGravitino() { propertyConverter.gravitinoToEngineProperties(gravitinoPropertiesWithoutPassword); }); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildPostgreSqlConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("jdbc-url", "jdbc:postgresql://localhost:5432/test") + .put("jdbc-user", "test") + .put("jdbc-password", "test") + .put("trino.bypass.join-pushdown.strategy", "EAGER") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); + PostgreSQLConnectorAdapter adapter = new PostgreSQLConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "postgresql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals( + propertiesMap.get("connection-url"), "jdbc:postgresql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } + + @Test + @SuppressWarnings("unchecked") + public void testBuildMySqlConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("jdbc-url", "jdbc:mysql://localhost:5432/test") + .put("jdbc-user", "test") + .put("jdbc-password", "test") + .put("trino.bypass.join-pushdown.strategy", "EAGER") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); + MySQLConnectorAdapter adapter = new MySQLConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "mysql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("connection-url"), "jdbc:mysql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 6641943465a..58453993a63 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -24,7 +24,7 @@ public void testGravitinoCatalog() { Catalog mockCatalog = mockCatalog( catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, Collections.emptyMap()); - GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog, false); + GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); assertEquals(catalogName, catalog.getName()); assertEquals(provider, catalog.getProvider()); } From 10591731745535ba318b57e720821dfd9dee5f87 Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 23 Apr 2024 10:33:11 +0800 Subject: [PATCH 103/106] [#2774] feat(doc): Add docs for messaging catalog (#3106) ### What changes were proposed in this pull request? This PR proposes to add docs for messaging catalog ### Why are the changes needed? Fix: #2774 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? No --- docs/index.md | 7 + docs/kafka-catalog.md | 61 ++++ ...nage-messaging-metadata-using-gravitino.md | 307 ++++++++++++++++++ 3 files changed, 375 insertions(+) create mode 100644 docs/kafka-catalog.md create mode 100644 docs/manage-messaging-metadata-using-gravitino.md diff --git a/docs/index.md b/docs/index.md index efcd2dc487b..b6661ee1170 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,6 +55,8 @@ REST API and the Java SDK. You can use either to manage metadata. See to learn how to manage relational metadata. * [Manage fileset metadata using Gravitino](./manage-fileset-metadata-using-gravitino.md) to learn how to manage fileset metadata. +* [Manage messaging metadata using Gravitino](./manage-messaging-metadata-using-gravitino.md) to learn how to manage + messaging metadata. Also, you can find the complete REST API definition in [Gravitino Open API](./api/rest/gravitino-rest-api), and the @@ -76,6 +78,10 @@ Gravitino currently supports the following catalogs: * [**Hadoop catalog**](./hadoop-catalog.md) +**Messaging catalogs:** + +* [**Kafka catalog**](./kafka-catalog.md) + Gravitino also provides an Iceberg REST catalog service for the Iceberg table format. See the [Iceberg REST catalog service](./iceberg-rest-service.md) for details. @@ -109,6 +115,7 @@ Gravitino supports different catalogs to manage the metadata in different source * [Doris catalog](./jdbc-doris-catalog.md): a complete guide to using Gravitino to manage Doris data. * [Hadoop catalog](./hadoop-catalog.md): a complete guide to using Gravitino to manage fileset using Hadoop Compatible File System (HCFS). +* [Kafka catalog](./kafka-catalog.md): a complete guide to using Gravitino to manage Kafka topics metadata. ### Trino connector diff --git a/docs/kafka-catalog.md b/docs/kafka-catalog.md new file mode 100644 index 00000000000..48da891a460 --- /dev/null +++ b/docs/kafka-catalog.md @@ -0,0 +1,61 @@ +--- +title: "Kafka catalog" +slug: /kafka-catalog +date: 2024-4-22 +keyword: kafka catalog +license: "Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2." +--- + +## Introduction + +Kafka catalog is a messaging catalog that offers the ability to manage Apache Kafka topics' metadata. +One Kafka catalog corresponds to one Kafka cluster. + +## Catalog + +### Catalog properties + +| Property Name | Description | Default Value | Required | Since Version | +|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `bootstrap.servers` | The Kafka broker(s) to connect to, allowing for multiple brokers by comma-separating them. | (none) | Yes | 0.5.0 | +| `gravitino.bypass.` | Property name with this prefix passed down to the underlying Kafka Admin client for use. (refer to [Kafka Admin Configs](https://kafka.apache.org/34/documentation.html#adminclientconfigs) for more details) | (none) | No | 0.5.0 | + +### Catalog operations + +Refer to [Catalog operations](./manage-messaging-metadata-using-gravitino.md#catalog-operations) for more details. + +## Schema + +A "default" schema, which includes all the topics in the Kafka cluster, will be automatically created when catalog is created. + +### Schema capabilities + +- Since the "default" schema is read-only, it only supports loading and listing schema. + +### Schema properties + +None. + +### Schema operations + +Refer to [Schema operation](./manage-messaging-metadata-using-gravitino.md#schema-operations) for more details. + +## Topic + +### Topic capabilities + +- The Kafka catalog supports creating, updating, deleting, and listing topics. + +### Topic properties + +| Property name | Description | Default value | Required | Since Version | +|----------------------|------------------------------------------|-------------------------------------------------------------------------------------|----------|---------------| +| `partition-count` | The number of partitions for the topic. | if not specified, will use the `num.partition` property in the broker. | No | 0.5.0 | +| `replication-factor` | The number of replications for the topic | if not specified, will use the `default.replication.factor` property in the broker. | No | 0.5.0 | + +You can pass other topic configurations to the topic properties. Refer to [Topic Configs](https://kafka.apache.org/34/documentation.html#topicconfigs) for more details. + +### Topic operations + +Refer to [Topic operation](./manage-messaging-metadata-using-gravitino.md#topic-operations) for more details. \ No newline at end of file diff --git a/docs/manage-messaging-metadata-using-gravitino.md b/docs/manage-messaging-metadata-using-gravitino.md new file mode 100644 index 00000000000..4a4c39685cd --- /dev/null +++ b/docs/manage-messaging-metadata-using-gravitino.md @@ -0,0 +1,307 @@ +--- +title: "Manage massaging metadata using Gravitino" +slug: /manage-massaging-metadata-using-gravitino +date: 2024-4-22 +keyword: Gravitino massaging metadata manage +license: Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +This page introduces how to manage messaging metadata using Gravitino. Messaging metadata refers to +the topic metadata of the messaging system such as Apache Kafka, Apache Pulsar, Apache RocketMQ, etc. +Through Gravitino, you can create, update, delete, and list topics via unified RESTful APIs or Java client. + +To use messaging catalog, please make sure that: + + - Gravitino server has started, and the host and port is [http://localhost:8090](http://localhost:8090). + - A metalake has been created. + +## Catalog operations + +### Create a catalog + +:::tip +For a messaging catalog, you must specify the `type` as `messaging` when creating a catalog. +::: + +You can create a catalog by sending a `POST` request to the `/api/metalakes/{metalake_name}/catalogs` +endpoint or just use the Gravitino Java client. The following is an example of creating a messaging catalog: + + + + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "catalog", + "type": "MESSAGING", + "comment": "comment", + "provider": "kafka", + "properties": { + "bootstrap.servers": "localhost:9092", + } +}' http://localhost:8090/api/metalakes/metalake/catalogs +``` + + + + +```java +GravitinoClient gravitinoClient = GravitinoClient + .builder("http://127.0.0.1:8090") + .withMetalake("metalake") + .build(); + +Map properties = ImmutableMap.builder() + // You should repalce the following with your own Kafka bootstrap servers that Gravitino can connect to. + .put("bootstrap.servers", "localhost:9092") + .build(); + +Catalog catalog = gravitinoClient.createCatalog( + NameIdentifier.of("metalake", "catalog"), + Type.MESSAGING, + "kafka", // provider, Gravitino only supports "kafka" for now. + "This is a Kafka catalog", + properties); +// ... +``` + + + + +Currently, Gravitino supports the following catalog providers: + +| Catalog provider | Catalog property | +|------------------|-----------------------------------------------------------------| +| `kafka` | [Kafka catalog property](./kafka-catalog.md#catalog-properties) | + +### Load a catalog + +Refer to [Load a catalog](./manage-relational-metadata-using-gravitino.md#load-a-catalog) +in relational catalog for more details. For a messaging catalog, the load operation is the same. + +### Alter a catalog + +Refer to [Alter a catalog](./manage-relational-metadata-using-gravitino.md#alter-a-catalog) +in relational catalog for more details. For a messaging catalog, the alter operation is the same. + +### Drop a catalog + +Refer to [Drop a catalog](./manage-relational-metadata-using-gravitino.md#drop-a-catalog) +in relational catalog for more details. For a messaging catalog, the drop operation is the same. + +### List all catalogs in a metalake + +Please refer to [List all catalogs in a metalake](./manage-relational-metadata-using-gravitino.md#list-all-catalogs-in-a-metalake) +in relational catalog for more details. For a messaging catalog, the list operation is the same. + +### List all catalogs' information in a metalake + +Please refer to [List all catalogs' information in a metalake](./manage-relational-metadata-using-gravitino.md#list-all-catalogs-information-in-a-metalake) +in relational catalog for more details. For a messaging catalog, the list operation is the same. + +## Schema operations + +`Schema` is a logical grouping of topics in a messaging catalog, if the messaging system does not support topics grouping, +schema operations are not supported but a "default" schema will be automatically created to include all topics + +:::caution note +Gravitino currently only supports the Kafka catalog. Since Kafka does not support topic grouping, only list and load operations are supported for schema. +::: + +### Create a schema + +Please refer to [Create a schema](./manage-relational-metadata-using-gravitino.md#create-a-schema) +in relational catalog for more details. For a messaging catalog, the create operation is the same. + +### Load a schema + +Please refer to [Load a schema](./manage-relational-metadata-using-gravitino.md#load-a-schema) +in relational catalog for more details. For a messaging catalog, the load operation is the same. + +### Alter a schema + +Please refer to [Alter a schema](./manage-relational-metadata-using-gravitino.md#alter-a-schema) +in relational catalog for more details. For a messaging catalog, the alter operation is the same. + +### Drop a schema + +Please refer to [Drop a schema](./manage-relational-metadata-using-gravitino.md#drop-a-schema) +in relational catalog for more details. For a messaging catalog, the drop operation is the same. + +### List all schemas under a catalog + +Please refer to [List all schemas under a catalog](./manage-relational-metadata-using-gravitino.md#list-all-schemas-under-a-catalog) +in relational catalog for more details. For a messaging catalog, the list operation is the same. + +## Topic operations + +:::tip +Users should create a metalake, a catalog and a schema before creating a table. +::: + +### Create a topic + +You can create a topic by sending a `POST` request to the `/api/metalakes/{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/topics` +endpoint or just use the Gravitino Java client. The following is an example of creating a topic: + + + + + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "example_topic", + "comment": "This is an example topic", + "properties": { + "partition-count": "3", + "replication-factor": 1 + } +}' http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/default/topics +``` + + + + +```java +GravitinoClient gravitinoClient = GravitinoClient + .builder("http://127.0.0.1:8090") + .withMetalake("metalake") + .build(); + +Catalog catalog = gravitinoClient.loadCatalog(NameIdentifier.of("metalake", "catalog")); +TopicCatalog topicCatalog = catalog.asTopicCatalog(); + +Map propertiesMap = ImmutableMap.builder() + .put("partition-count": "3") + .put("replication-factor": "1") + .build(); + +topicCatalog.createTopic( + NameIdentifier.of("metalake", "catalog", "default", "example_topic"), + "This is an example topic", + null, // The message schema of the topic object. Always null because it's not supported yet. + propertiesMap, +); +``` + + + + +### Alter a topic + +You can modify a topic by sending a `PUT` request to the `/api/metalakes/{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/topics/{topic_name}` +endpoint or just use the Gravitino Java client. The following is an example of altering a topic: + + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "updates": [ + { + "@type": "removeProperty", + "property": "key2" + }, { + "@type": "setProperty", + "property": "key3", + "value": "value3" + } + ] +}' http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/default/topics/topic +``` + + + + +```java +// ... +// Assuming you have just created a Kafka catalog named `catalog` +Catalog catalog = gravitinoClient.loadCatalog(NameIdentifier.of("metalake", "catalog")); + +TopicCatalog topicCatalog = catalog.asTopicCatalog(); + +Topic t = topicCatalog.alterTopic(NameIdentifier.of("metalake", "catalog", "default", "topic"), + TopicChange.removeProperty("key2"), TopicChange.setProperty("key3", "value3")); +// ... +``` + + + + +Currently, Gravitino supports the following changes to a topic: + +| Supported modification | JSON | Java | +|-------------------------|--------------------------------------------------------------|---------------------------------------------| +| Update a comment | `{"@type":"updateComment","newComment":"new_comment"}` | `TopicChange.updateComment("new_comment")` | +| Set a topic property | `{"@type":"setProperty","property":"key1","value":"value1"}` | `TopicChange.setProperty("key1", "value1")` | +| Remove a topic property | `{"@type":"removeProperty","property":"key1"}` | `TopicChange.removeProperty("key1")` | + +### Drop a topic + +You can remove a topic by sending a `DELETE` request to the `/api/metalakes/{metalake_name} +/catalogs/{catalog_name}/schemas/{schema_name}/topics/{topic_name}` endpoint or by using the +Gravitino Java client. The following is an example of dropping a topic: + + + + +```shell +curl -X DELETE -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" \ +http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/default/topics/topic +``` + + + + +```java +// ... +// Assuming you have just created a Kafka catalog named `catalog` +Catalog catalog = gravitinoClient.loadCatalog(NameIdentifier.of("metalake", "catalog")); + +TopicCatalog topicCatalog = catalog.asTopicCatalog(); + +// Drop a topic +topicCatalog.dropTopic(NameIdentifier.of("metalake", "catalog", "default", "topic")); +// ... +``` + + + + +### List all topics under a schema + +You can list all topics in a schema by sending a `GET` request to the `/api/metalakes/ +{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/topics` endpoint or by using the +Gravitino Java client. The following is an example of listing all the topics in a schema: + + + + +```shell +curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" \ +http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/topics +``` + + + + +```java +// ... +Catalog catalog = gravitinoClient.loadCatalog(NameIdentifier.of("metalake", "catalog")); + +TopicCatalog topicCatalog = catalog.asTopicCatalog(); +NameIdentifier[] identifiers = + topicCatalog.listTopics(Namespace.ofTopic("metalake", "catalog", "default")); +// ... +``` + + + \ No newline at end of file From 24f38839c4788839a20e5d3aad4b923cede9c2be Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 23 Apr 2024 11:12:20 +0800 Subject: [PATCH 104/106] [#2773] feat(doc): Add OAS for messaging catalog (#3108) ### What changes were proposed in this pull request? Add OAS for messaging catalog ### Why are the changes needed? Fix: #2773 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no --- docs/open-api/filesets.yaml | 18 +- docs/open-api/openapi.yaml | 14 ++ docs/open-api/topics.yaml | 336 ++++++++++++++++++++++++++++++++++++ 3 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 docs/open-api/topics.yaml diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 5f8174305bc..cfec475f964 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -208,7 +208,7 @@ components: FilesetUpdatesRequest: type: object required: - - name + - updates properties: updates: type: array @@ -218,16 +218,16 @@ components: FilesetUpdateRequest: oneOf: - $ref: "#/components/schemas/RenameFilesetRequest" - - $ref: "#/components/schemas/SetFilesetPropertiesRequest" + - $ref: "#/components/schemas/SetFilesetPropertyRequest" - $ref: "#/components/schemas/UpdateFilesetCommentRequest" - - $ref: "#/components/schemas/RemoveFilesetPropertiesRequest" + - $ref: "#/components/schemas/RemoveFilesetPropertyRequest" discriminator: propertyName: "@type" mapping: rename: "#/components/schemas/RenameFilesetRequest" - setProperties: "#/components/schemas/SetFilesetPropertiesRequest" + setProperty: "#/components/schemas/SetFilesetPropertyRequest" updateComment: "#/components/schemas/UpdateFilesetCommentRequest" - removeProperties: "#/components/schemas/RemoveFilesetPropertiesRequest" + removeProperty: "#/components/schemas/RemoveFilesetPropertyRequest" RenameFilesetRequest: type: object @@ -248,7 +248,7 @@ components: "newName": "newName" } - SetFilesetPropertiesRequest: + SetFilesetPropertyRequest: type: object required: - "@type" @@ -291,7 +291,7 @@ components: "newComment": "new comment" } - RemoveFilesetPropertiesRequest: + RemoveFilesetPropertyRequest: type: object required: - "@type" @@ -361,7 +361,7 @@ components: FilesetAlreadyExistsException: value: { - "code": 1001, + "code": 1004, "type": "FilesetAlreadyExistsException", "message": "Fileset already exists", "stack": [ @@ -371,7 +371,7 @@ components: NoSuchFilesetException: value: { - "code": 1004, + "code": 1003, "type": "NoSuchFilesetException", "message": "Fileset does not exist", "stack": [ diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index 02f7a191e46..bd1d0318a0c 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -74,6 +74,12 @@ paths: /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}: $ref: "./filesets.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1filesets~1%7Bfileset%7D" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics: + $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: + $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics~1%7Btopic%7D" + components: schemas: @@ -260,6 +266,14 @@ components: schema: type: string + topic: + name: topic + in: path + description: The name of the topic + required: true + schema: + type: string + securitySchemes: OAuth2WithJWT: diff --git a/docs/open-api/topics.yaml b/docs/open-api/topics.yaml new file mode 100644 index 00000000000..b24e4325740 --- /dev/null +++ b/docs/open-api/topics.yaml @@ -0,0 +1,336 @@ +# +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# + +--- + +paths: + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + + get: + tags: + - topic + summary: List topics + operationId: listTopics + responses: + "200": + $ref: "./openapi.yaml#/components/responses/EntityListResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - topic + summary: Create topic + operationId: createTopic + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TopicCreateRequest" + examples: + TopicCreateRequest: + $ref: "#/components/examples/TopicCreateRequest" + responses: + "200": + $ref: "#/components/responses/TopicResponse" + "409": + description: Conflict - The target topic already exists + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + TopicAlreadyExistsErrorResponse: + $ref: "#/components/examples/TopicAlreadyExistsException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/topic" + + get: + tags: + - topic + summary: Get topic + operationId: loadTopic + description: Return the specified topic object + responses: + "200": + $ref: "#/components/responses/TopicResponse" + "404": + description: Not Found - The target topic does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchTopicException: + $ref: "#/components/examples/NoSuchTopicException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + put: + tags: + - topic + summary: Update topic + operationId: alterTopic + description: Update the specified topic in a schema + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TopicUpdatesRequest" + responses: + "200": + $ref: "#/components/responses/TopicResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "404": + description: Not Found - The target topic does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchTopicException: + $ref: "#/components/examples/NoSuchTopicException" + "409": + description: Conflict - The target topic already exists + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + TopicAlreadyExistsErrorResponse: + $ref: "#/components/examples/TopicAlreadyExistsException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - topic + summary: Delete topic + operationId: dropTopic + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + + schemas: + Topic: + type: object + required: + - name + properties: + name: + type: string + description: The name of the topic + comment: + type: string + description: The comment of the topic + properties: + type: object + description: The properties of the topic + nullable: true + default: {} + additionalProperties: + type: string + + TopicCreateRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the topic + comment: + type: string + description: The comment of the topic + nullable: true + properties: + type: object + description: The properties of the topic + nullable: true + default: {} + additionalProperties: + type: string + + TopicUpdatesRequest: + type: object + required: + - updates + properties: + updates: + type: array + items: + $ref: "#/components/schemas/TopicUpdateRequest" + + TopicUpdateRequest: + oneOf: + - $ref: "#/components/schemas/UpdateTopicCommentRequest" + - $ref: "#/components/schemas/SetTopicPropertyRequest" + - $ref: "#/components/schemas/RemoveTopicPropertyRequest" + discriminator: + propertyName: "@type" + mapping: + updateComment: "#/components/schemas/UpdateTopicCommentRequest" + setProperty: "#/components/schemas/SetTopicPropertyRequest" + removeProperty: "#/components/schemas/RemoveTopicPropertyRequest" + + UpdateTopicCommentRequest: + type: object + required: + - "@type" + - newComment + properties: + "@type": + type: string + enum: + - "updateComment" + newComment: + type: string + description: The new comment of the topic + example: { + "@type": "updateComment", + "newComment": "This is the new comment" + } + + SetTopicPropertyRequest: + type: object + required: + - "@type" + - property + - value + properties: + "@type": + type: string + enum: + - "setProperty" + property: + type: string + description: The name of the property to set + value: + type: string + description: The value of the property to set + example: { + "@type": "setProperty", + "property": "key", + "value": "value" + } + + RemoveTopicPropertyRequest: + type: object + required: + - "@type" + - property + properties: + "@type": + type: string + enum: + - "removeProperty" + property: + type: string + description: The name of the property to remove + example: { + "@type": "removeProperty", + "property": "key" + } + + responses: + TopicResponse: + description: Returns include the topic object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + topic: + $ref: "#/components/schemas/Topic" + examples: + TopicResponse: + $ref: "#/components/examples/TopicResponse" + + examples: + TopicCreateRequest: + value: { + "name": "topic1", + "comment": "This is a topic", + "properties": { + "partition-count": "1", + "replication-factor": "1" + } + } + + TopicResponse: + value: { + "code": 0, + "topic": { + "name": "topic1", + "comment": "This is a topic", + "properties": { + "partition-count": "1", + "replication-factor": "1" + } + } + } + + TopicAlreadyExistsException: + value: { + "code": 1004, + "type": "TopicAlreadyExistsException", + "message": "Topic already exists", + "stack": [ + "com.datastrato.gravitino.exceptions.TopicAlreadyExistsException: Topic already exists: topic1" + ] + } + + NoSuchTopicException: + value: { + "code": 1003, + "type": "NoSuchTopicException", + "message": "Topic does not exist", + "stack": [ + "com.datastrato.gravitino.exceptions.NoSuchTopicException: Topic does not exist", + "..." + ] + } + + + From 825a3b69d9fa3f63c7b3bd7e9cb3b442efaf18a9 Mon Sep 17 00:00:00 2001 From: Eliza Sorber <53704365+elizasorber@users.noreply.github.com> Date: Mon, 22 Apr 2024 23:15:21 -0400 Subject: [PATCH 105/106] [#2915] improvement(catalog-lakehouse-iceberg): simplify ternary operators (#3109) ### What changes were proposed in this pull request? Simplified some nested ternary operator code in catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java Fix: #2915 ### How was this patch tested? Followed testing instructions in how-to-test.md --- .../catalog/lakehouse/iceberg/IcebergSchema.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java index 94d6ab6f2a7..37e2487e22e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java @@ -38,12 +38,15 @@ private Builder() {} protected IcebergSchema internalBuild() { IcebergSchema icebergSchema = new IcebergSchema(); icebergSchema.name = name; - icebergSchema.comment = - null == comment - ? (null == properties - ? null - : properties.get(IcebergSchemaPropertiesMetadata.COMMENT)) - : comment; + if (null == comment) { + if (null == properties) { + icebergSchema.comment = null; + } else { + icebergSchema.comment = properties.get(IcebergSchemaPropertiesMetadata.COMMENT); + } + } else { + icebergSchema.comment = comment; + } icebergSchema.properties = properties; icebergSchema.auditInfo = auditInfo; return icebergSchema; From 89faa8070eda4b66b7102b58da60293dc880a0cc Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Tue, 23 Apr 2024 12:03:25 +0800 Subject: [PATCH 106/106] [#3057] fix(web): fix breadcrumbs text display styles and appbar layout (#3107) ### What changes were proposed in this pull request? Fix breadcrumbs text display styles. ### Why are the changes needed? Fix: #3057 ### Does this PR introduce _any_ user-facing change? image image image image image image ### How was this patch tested? local --- .../app/metalakes/metalake/MetalakeTree.js | 10 ++++++- .../metalake/rightContent/MetalakePath.js | 29 ++++++++++++++++--- .../metalake/rightContent/RightContent.js | 7 +++-- web/src/app/rootLayout/AppBar.js | 28 ++++++++++++++---- web/src/components/DetailsDrawer.js | 3 +- 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index 28bd1552aa9..02d22a46c03 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -237,10 +237,18 @@ const MetalakeTree = props => { } const renderNode = nodeData => { + const len = extractPlaceholder(nodeData.key).length + const maxWidth = 260 - (26 * 2 - 26 * (5 - len)) if (nodeData.path) { return ( theme.palette.text.secondary }} + sx={{ + color: theme => theme.palette.text.secondary, + whiteSpace: 'nowrap', + maxWidth, + overflow: 'hidden', + textOverflow: 'ellipsis' + }} data-refer='tree-node' data-refer-node={nodeData.key} > diff --git a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js index 9274780ef00..8ce72021a24 100644 --- a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js +++ b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js @@ -12,12 +12,17 @@ import { Link as MUILink, Breadcrumbs, Typography, Tooltip, styled } from '@mui/ import Icon from '@/components/Icon' -const Text = styled(Typography)(({ theme }) => ({ - maxWidth: '120px', +const TextWrapper = styled(Typography)(({ theme }) => ({ + mixWidth: '120px', overflow: 'hidden', - textOverflow: 'ellipsis' + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' })) +const Text = props => { + return +} + const MetalakePath = props => { const searchParams = useSearchParams() @@ -47,10 +52,26 @@ const MetalakePath = props => { return ( li.MuiBreadcrumbs-li': { + overflow: 'hidden', + display: 'inline-flex', + '& > a': { + width: '100%', + '& > svg': { + minWidth: 20 + } + } + }, '& ol > li:last-of-type': { - color: theme => `${theme.palette.text.primary} !important` + color: theme => `${theme.palette.text.primary} !important`, + overflow: 'hidden' } }} > diff --git a/web/src/app/metalakes/metalake/rightContent/RightContent.js b/web/src/app/metalakes/metalake/rightContent/RightContent.js index e26c192bd70..7b645b372d4 100644 --- a/web/src/app/metalakes/metalake/rightContent/RightContent.js +++ b/web/src/app/metalakes/metalake/rightContent/RightContent.js @@ -37,9 +37,9 @@ const RightContent = () => { borderBottom: theme => `1px solid ${theme.palette.divider}` }} > - - - + + + @@ -54,6 +54,7 @@ const RightContent = () => { variant='contained' startIcon={} onClick={handleCreateCatalog} + sx={{ width: 200 }} data-refer='create-catalog-btn' > Create Catalog diff --git a/web/src/app/rootLayout/AppBar.js b/web/src/app/rootLayout/AppBar.js index 2e61c6b599b..94d885365b4 100644 --- a/web/src/app/rootLayout/AppBar.js +++ b/web/src/app/rootLayout/AppBar.js @@ -51,12 +51,18 @@ const AppBar = () => { elevation={6} position={'sticky'} className={ - 'layout-navbar-container twc-px-6 twc-items-center twc-justify-center twc-transition-[border-bottom] twc-ease-in-out twc-duration-200 twc-bg-customs-white' + 'layout-navbar-container twc-items-center twc-justify-center twc-transition-[border-bottom] twc-ease-in-out twc-duration-200 twc-bg-customs-white' } > - - + + { {metalake ? ( - + Metalake

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface GroupMetaMapper { + String TABLE_NAME = "group_meta"; + + @Select( + "SELECT group_id as groupId FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0") + Long selectGroupIdBySchemaIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name); + + @Select( + "SELECT group_id as groupId, group_name as groupName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0") + GroupPO selectGroupMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(group_id, group_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )") + void insertGroupMeta(@Param("groupMeta") GroupPO groupPO); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(group_id, group_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " group_name = #{groupMeta.groupName}," + + " metalake_id = #{groupMeta.metalakeId}," + + " audit_info = #{groupMeta.auditInfo}," + + " current_version = #{groupMeta.currentVersion}," + + " last_version = #{groupMeta.lastVersion}," + + " deleted_at = #{groupMeta.deletedAt}") + void insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE group_id = #{groupId} AND deleted_at = 0") + void softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + void softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET group_name = #{newGroupMeta.groupName}," + + " metalake_id = #{newGroupMeta.metalakeId}," + + " audit_info = #{newGroupMeta.auditInfo}," + + " current_version = #{newGroupMeta.currentVersion}," + + " last_version = #{newGroupMeta.lastVersion}," + + " deleted_at = #{newGroupMeta.deletedAt}" + + " WHERE group_id = #{oldGroupMeta.groupId}" + + " AND group_name = #{oldGroupMeta.groupName}" + + " AND metalake_id = #{oldGroupMeta.metalakeId}" + + " AND audit_info = #{oldGroupMeta.auditInfo}" + + " AND current_version = #{oldGroupMeta.currentVersion}" + + " AND last_version = #{oldGroupMeta.lastVersion}" + + " AND deleted_at = 0") + Integer updateGroupMeta( + @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java new file mode 100644 index 00000000000..05afc0e19bc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.GroupRoleRelPO; +import java.util.List; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *