From 0e7c2bfa078f3527e51acbf7a5492fb54526ddb2 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Sat, 16 Mar 2024 14:31:51 -0700 Subject: [PATCH] Adding datasource status and filter for hive tables (#1549) * adding datasource status and filter for hive tables Signed-off-by: Shenoy Pratik * update tests and replace dataConnection variable Signed-off-by: Shenoy Pratik --------- Signed-off-by: Shenoy Pratik --- common/constants/data_sources.ts | 1 + common/constants/shared.ts | 1 + common/types/data_connections.ts | 3 + .../data_connection.test.tsx.snap | 22 +- .../inactive_data_connection.test.tsx.snap | 58 ++ ...anage_data_connections_table.test.tsx.snap | 686 +----------------- .../__tests__/data_connection.test.tsx | 2 +- .../inactive_data_connection.test.tsx | 63 ++ .../components/manage/data_connection.tsx | 74 +- .../manage/inactive_data_connection.tsx | 53 ++ .../manage/manage_data_connections_table.tsx | 68 +- .../catalog_cache/cache_loader.test.tsx | 6 +- .../framework/catalog_cache/cache_loader.tsx | 15 +- .../data_connections_router.ts | 39 +- test/datasources.ts | 26 +- 15 files changed, 374 insertions(+), 743 deletions(-) create mode 100644 public/components/datasources/components/__tests__/__snapshots__/inactive_data_connection.test.tsx.snap create mode 100644 public/components/datasources/components/__tests__/inactive_data_connection.test.tsx create mode 100644 public/components/datasources/components/manage/inactive_data_connection.tsx diff --git a/common/constants/data_sources.ts b/common/constants/data_sources.ts index 3faa8987e..c396951a8 100644 --- a/common/constants/data_sources.ts +++ b/common/constants/data_sources.ts @@ -37,6 +37,7 @@ export const ACCELERATION_TIME_INTERVAL = [ export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z][a-z_]*$/; export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; +export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; export const TIMESTAMP_DATATYPE = 'timestamp'; export const ACCELERATION_INDEX_TYPES = [ diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 5af693399..3e8cb4906 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -16,6 +16,7 @@ export const INTEGRATIONS_BASE = '/api/integrations'; export const JOBS_BASE = '/query/jobs'; export const DATACONNECTIONS_BASE = '/api/dataconnections'; export const EDIT = '/edit'; +export const DATACONNECTIONS_UPDATE_STATUS = '/status'; export const SECURITY_ROLES = '/api/v1/configuration/roles'; export const EVENT_ANALYTICS = '/event_analytics'; export const SAVED_OBJECTS = '/saved_objects'; diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index 8ebab0d11..62b48bad9 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -56,12 +56,15 @@ export interface PrometheusProperties { 'prometheus.uri': string; } +export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; + export interface DatasourceDetails { allowedRoles: string[]; name: string; connector: DatasourceType; description: string; properties: S3GlueProperties | PrometheusProperties; + status: DatasourceStatus; } interface AsyncApiDataResponse { diff --git a/public/components/datasources/components/__tests__/__snapshots__/data_connection.test.tsx.snap b/public/components/datasources/components/__tests__/__snapshots__/data_connection.test.tsx.snap index 5d14f6038..031297a66 100644 --- a/public/components/datasources/components/__tests__/__snapshots__/data_connection.test.tsx.snap +++ b/public/components/datasources/components/__tests__/__snapshots__/data_connection.test.tsx.snap @@ -156,24 +156,16 @@ exports[`Data Connection Page test Renders S3 data connection page with data 1`] > @@ -230,7 +222,7 @@ exports[`Data Connection Page test Renders S3 data connection page with data 1`] > diff --git a/public/components/datasources/components/__tests__/__snapshots__/inactive_data_connection.test.tsx.snap b/public/components/datasources/components/__tests__/__snapshots__/inactive_data_connection.test.tsx.snap new file mode 100644 index 000000000..2587fc5cb --- /dev/null +++ b/public/components/datasources/components/__tests__/__snapshots__/inactive_data_connection.test.tsx.snap @@ -0,0 +1,58 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Data Connection Inactive Page test Renders inactive data connection callout 1`] = ` +
+
+
+ + + This data source connection is inactive + +
+
+
+

+ Associated objects and accelerations are not available while this connection is inactive. +

+ +
+
+
+
+`; diff --git a/public/components/datasources/components/__tests__/__snapshots__/manage_data_connections_table.test.tsx.snap b/public/components/datasources/components/__tests__/__snapshots__/manage_data_connections_table.test.tsx.snap index 673613875..b8cf20aa6 100644 --- a/public/components/datasources/components/__tests__/__snapshots__/manage_data_connections_table.test.tsx.snap +++ b/public/components/datasources/components/__tests__/__snapshots__/manage_data_connections_table.test.tsx.snap @@ -268,701 +268,61 @@ exports[`Manage Data Connections Table test Renders manage data connections tabl - Actions + Status - - - - - -
- Name -
-
-
-
-
- - my_spark3 - -
-
-
- - -
- - - Edit - - - - Query in Observability Logs - -
-
-
- - - -
-
-
-
- - - - -
- Name -
-
-
-
-
- - my_spark4 - -
-
-
- - -
- - - Edit - - - - Query in Observability Logs - -
-
-
- - - -
-
-
-
- - - - -
- Name -
-
-
-
-
- - my_spark - -
-
-
- - -
- - Edit - - - - Query in Observability Logs + Actions -
-
-
- - - -
-
-
-
- + + + + -
- Name -
-
-
-
-
- - my_spark2 - -
-
-
- -
- - Edit - - - - Query in Observability Logs + No items found -
-
-
- - - -
-
-
-
-
-
-
-
-
- -
-
-
-
- -
-
-
diff --git a/public/components/datasources/components/__tests__/data_connection.test.tsx b/public/components/datasources/components/__tests__/data_connection.test.tsx index 57aab1c7c..32f30a6a8 100644 --- a/public/components/datasources/components/__tests__/data_connection.test.tsx +++ b/public/components/datasources/components/__tests__/data_connection.test.tsx @@ -8,6 +8,7 @@ import { configure } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import React from 'react'; import ReactDOM from 'react-dom'; +import { CatalogCacheManager } from '../../../../../public/framework/catalog_cache/cache_manager'; import { coreRefs } from '../../../../../public/framework/core_refs'; import { describePrometheusDataConnection, @@ -16,7 +17,6 @@ import { mockDataSourceCacheData, } from '../../../../../test/datasources'; import { DataConnection } from '../manage/data_connection'; -import { CatalogCacheManager } from '../../../../../public/framework/catalog_cache/cache_manager'; jest.mock('../../../../plugin', () => ({ getRenderAccelerationDetailsFlyout: jest.fn(), diff --git a/public/components/datasources/components/__tests__/inactive_data_connection.test.tsx b/public/components/datasources/components/__tests__/inactive_data_connection.test.tsx new file mode 100644 index 000000000..aabad4ddc --- /dev/null +++ b/public/components/datasources/components/__tests__/inactive_data_connection.test.tsx @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { act } from '@testing-library/react'; +import { configure } from 'enzyme'; +import Adapter from 'enzyme-adapter-react-16'; +import React from 'react'; +import ReactDOM from 'react-dom'; +import { coreRefs } from '../../../../../public/framework/core_refs'; +import { describePrometheusDataConnection } from '../../../../../test/datasources'; +import { DatasourceDetails } from '../manage/data_connection'; +import { InactiveDataConnectionCallout } from '../manage/inactive_data_connection'; + +jest.mock('../../../../../public/framework/core_refs', () => ({ + coreRefs: { + chrome: { + setBreadcrumbs: jest.fn(), + }, + http: { + get: jest.fn(), + }, + }, +})); + +describe('Data Connection Inactive Page test', () => { + configure({ adapter: new Adapter() }); + + beforeEach(() => { + // Clear the mock implementation before each test + (coreRefs.http!.get as jest.Mock).mockClear(); + }); + + it('Renders inactive data connection callout', async () => { + const mockDatasourceDetails: DatasourceDetails = { + allowedRoles: [], + name: '', + description: '', + connector: 'PROMETHEUS', + properties: { 'prometheus.uri': 'placeholder' }, + status: 'DISABLED', + }; + + const container = document.createElement('div'); + (coreRefs.http!.get as jest.Mock).mockResolvedValue(describePrometheusDataConnection); + await act(() => { + ReactDOM.render( + , + container + ); + }); + expect(container).toMatchSnapshot(); + }); +}); diff --git a/public/components/datasources/components/manage/data_connection.tsx b/public/components/datasources/components/manage/data_connection.tsx index 248d3b05a..b3380d229 100644 --- a/public/components/datasources/components/manage/data_connection.tsx +++ b/public/components/datasources/components/manage/data_connection.tsx @@ -19,8 +19,8 @@ import { EuiText, EuiTitle, } from '@elastic/eui'; -import React, { useEffect, useState } from 'react'; import _ from 'lodash'; +import React, { useEffect, useState } from 'react'; import { DATACONNECTIONS_BASE, INTEGRATIONS_BASE, @@ -28,22 +28,23 @@ import { observabilityLogsID, observabilityMetricsID, } from '../../../../../common/constants/shared'; -import { coreRefs } from '../../../../framework/core_refs'; -import { getRenderCreateAccelerationFlyout } from '../../../../plugin'; -import { NoAccess } from '../no_access'; import { DatasourceDetails, PrometheusProperties, } from '../../../../../common/types/data_connections'; -import { AssociatedObjectsTab } from './associated_objects/associated_objects_tab'; -import { AccelerationTable } from './accelerations/acceleration_table'; -import { AccessControlTab } from './access_control_tab'; -import { InstalledIntegrationsTable } from './integrations/installed_integrations_table'; import { useLoadAccelerationsToCache, useLoadDatabasesToCache, useLoadTablesToCache, } from '../../../../../public/framework/catalog_cache/cache_loader'; +import { coreRefs } from '../../../../framework/core_refs'; +import { getRenderCreateAccelerationFlyout } from '../../../../plugin'; +import { NoAccess } from '../no_access'; +import { AccelerationTable } from './accelerations/acceleration_table'; +import { AccessControlTab } from './access_control_tab'; +import { AssociatedObjectsTab } from './associated_objects/associated_objects_tab'; +import { InactiveDataConnectionCallout } from './inactive_data_connection'; +import { InstalledIntegrationsTable } from './integrations/installed_integrations_table'; const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); @@ -55,6 +56,7 @@ export const DataConnection = (props: any) => { description: '', connector: 'PROMETHEUS', properties: { 'prometheus.uri': 'placeholder' }, + status: 'ACTIVE', }); const [hasAccess, setHasAccess] = useState(true); const { http, chrome, application } = coreRefs; @@ -203,17 +205,7 @@ export const DataConnection = (props: any) => { ); }; - useEffect(() => { - chrome!.setBreadcrumbs([ - { - text: 'Data sources', - href: '#/', - }, - { - text: `${dataSource}`, - href: `#/manage/${dataSource}`, - }, - ]); + const fetchSelectedDatasource = () => { http! .get(`${DATACONNECTIONS_BASE}/${dataSource}`) .then((data) => { @@ -223,11 +215,26 @@ export const DataConnection = (props: any) => { name: data.name, connector: data.connector, properties: data.properties, + status: data.status, }); }) .catch((_err) => { setHasAccess(false); }); + }; + + useEffect(() => { + chrome!.setBreadcrumbs([ + { + text: 'Data sources', + href: '#/', + }, + { + text: `${dataSource}`, + href: `#/manage/${dataSource}`, + }, + ]); + fetchSelectedDatasource(); }, [chrome, http]); const tabs = [ @@ -395,15 +402,26 @@ export const DataConnection = (props: any) => { - - - - + + {datasourceDetails.status !== 'ACTIVE' ? ( + + ) : ( + <> + + + + + + )} + diff --git a/public/components/datasources/components/manage/inactive_data_connection.tsx b/public/components/datasources/components/manage/inactive_data_connection.tsx new file mode 100644 index 000000000..50a668ae1 --- /dev/null +++ b/public/components/datasources/components/manage/inactive_data_connection.tsx @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiButton, EuiCallOut } from '@elastic/eui'; +import React from 'react'; +import { + DATACONNECTIONS_BASE, + DATACONNECTIONS_UPDATE_STATUS, + EDIT, +} from '../../../../../common/constants/shared'; +import { coreRefs } from '../../../../framework/core_refs'; +import { useToast } from '../../../common/toast'; +import { DatasourceDetails } from './data_connection'; + +interface InactiveDataConnectionCalloutProps { + datasourceDetails: DatasourceDetails; + fetchSelectedDatasource: () => void; +} + +export const InactiveDataConnectionCallout = ({ + datasourceDetails, + fetchSelectedDatasource, +}: InactiveDataConnectionCalloutProps) => { + const { setToast } = useToast(); + const { http } = coreRefs; + const enableDataSource = () => { + http! + .post(`${DATACONNECTIONS_BASE}${EDIT}${DATACONNECTIONS_UPDATE_STATUS}`, { + body: JSON.stringify({ name: datasourceDetails.name, status: 'active' }), + }) + .then(() => { + setToast(`Data connection ${datasourceDetails.name} enabled successfully`); + fetchSelectedDatasource(); + }) + .catch((err) => { + console.error(err); + setToast(`Data connection ${datasourceDetails.name} could not be enabled.`, 'danger'); + }); + }; + + return ( + +

+ Associated objects and accelerations are not available while this connection is inactive. +

+ + Enable connection + + + ); +}; diff --git a/public/components/datasources/components/manage/manage_data_connections_table.tsx b/public/components/datasources/components/manage/manage_data_connections_table.tsx index 4d8c65751..5f67869ee 100644 --- a/public/components/datasources/components/manage/manage_data_connections_table.tsx +++ b/public/components/datasources/components/manage/manage_data_connections_table.tsx @@ -6,6 +6,7 @@ import { EuiFlexGroup, EuiFlexItem, + EuiHealth, EuiIcon, EuiInMemoryTable, EuiLink, @@ -17,29 +18,30 @@ import { } from '@elastic/eui'; import _ from 'lodash'; import React, { useEffect, useState } from 'react'; -import { DataConnectionsHeader } from '../data_connections_header'; -import { HomeProps } from '../../home'; -import { DataConnectionsDescription } from './manage_data_connections_description'; import { DATACONNECTIONS_BASE, observabilityIntegrationsID, observabilityLogsID, observabilityMetricsID, } from '../../../../../common/constants/shared'; -import { useToast } from '../../../common/toast'; +import { DatasourceStatus, DatasourceType } from '../../../../../common/types/data_connections'; +import { coreRefs } from '../../../../../public/framework/core_refs'; import { DeleteModal } from '../../../common/helpers/delete_modal'; -import S3Logo from '../../icons/s3-logo.svg'; +import { useToast } from '../../../common/toast'; +import { HomeProps } from '../../home'; import PrometheusLogo from '../../icons/prometheus-logo.svg'; -import { DatasourceType } from '../../../../../common/types/data_connections'; -import { coreRefs } from '../../../../../public/framework/core_refs'; +import S3Logo from '../../icons/s3-logo.svg'; +import { DataConnectionsHeader } from '../data_connections_header'; +import { DataConnectionsDescription } from './manage_data_connections_description'; interface DataConnection { connectionType: DatasourceType; name: string; + dsStatus: DatasourceStatus; } export const ManageDataConnectionsTable = (props: HomeProps) => { - const { http, chrome, pplService } = props; + const { http, chrome } = props; const { application } = coreRefs; const { setToast } = useToast(); @@ -60,10 +62,30 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { ); }) .catch((err) => { + console.error(err); setToast(`Data connection $${connectionName} not deleted. See output for more details.`); }); }; + const fetchDataSources = () => { + http! + .get(`${DATACONNECTIONS_BASE}`) + .then((res: any) => { + const dataConnections = res.map((dataConnection: any) => { + return { + name: dataConnection.name, + connectionType: dataConnection.connector, + dsStatus: dataConnection.status, + }; + }); + setData(dataConnections); + }) + .catch((err) => { + console.error(err); + setToast(`Could not fetch datasources`, 'danger'); + }); + }; + useEffect(() => { chrome.setBreadcrumbs([ { @@ -71,19 +93,9 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { href: '#/', }, ]); - handleDataRequest(); + fetchDataSources(); }, [chrome]); - async function handleDataRequest() { - pplService!.fetch({ query: 'show datasources', format: 'jdbc' }).then((dataconnections) => - setData( - dataconnections.jsonData.map((x: any) => { - return { name: x.DATASOURCE_NAME, connectionType: x.CONNECTOR_TYPE }; - }) - ) - ); - } - const displayDeleteModal = (connectionName: string) => { setModalLayout( { ), }, + { + field: 'status', + name: 'Status', + sortable: true, + truncateText: true, + render: (value, record: DataConnection) => + record.dsStatus === 'ACTIVE' ? ( + Active + ) : ( + Inactive + ), + }, { field: 'actions', name: 'Actions', @@ -208,7 +232,8 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { const entries = data.map((dataconnection: DataConnection) => { const name = dataconnection.name; const connectionType = dataconnection.connectionType; - return { connectionType, name, data: { name, connectionType } }; + const dsStatus = dataconnection.dsStatus; + return { connectionType, name, dsStatus, data: { name, connectionType } }; }); return ( @@ -216,8 +241,7 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { - - + { dataSourceName, expect.objectContaining({ name: databaseName, - tables: [{ name: 'Table1' }, { name: 'Table2' }], + tables: [{ name: 'http_logs1' }, { name: 'http_logs2' }], lastUpdated: expect.any(String), status: CachedDataSourceStatus.Updated, }) @@ -305,7 +305,7 @@ describe('loadCacheTests', () => { const loadCacheType = 'tables'; const dataSourceName = 'example'; const databaseName = 'test'; - const expectedQuery = 'SHOW TABLES IN `example`.`test`'; + const expectedQuery = "SHOW TABLE EXTENDED IN `example`.`test` LIKE '*'"; expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); }); @@ -326,7 +326,7 @@ describe('loadCacheTests', () => { const loadCacheType = 'tables'; const dataSourceName = 'example'; const databaseName = '`sample`'; - const expectedQuery = 'SHOW TABLES IN `example`.`sample`'; + const expectedQuery = "SHOW TABLE EXTENDED IN `example`.`sample` LIKE '*'"; expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); }); }); diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index 66e59f2d5..63129a4c3 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -6,6 +6,7 @@ import { useEffect, useState } from 'react'; import { ASYNC_POLLING_INTERVAL, + SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO, } from '../../../common/constants/data_sources'; import { @@ -81,9 +82,11 @@ export const updateTablesToCache = ( } const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newTables = combinedData.map((row: any) => ({ - name: row.tableName, - })); + const newTables = combinedData + .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) + .map((row: any) => ({ + name: row.tableName, + })); CatalogCacheManager.updateDatabase(dataSourceName, { ...cachedDatabase, @@ -205,9 +208,9 @@ export const createLoadQuery = ( query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; break; case 'tables': - query = `SHOW TABLES IN ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( - databaseName! - )}`; + query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( + dataSourceName + )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; break; case 'accelerations': query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; diff --git a/server/routes/data_connections/data_connections_router.ts b/server/routes/data_connections/data_connections_router.ts index 66c0d3b6e..2788653de 100644 --- a/server/routes/data_connections/data_connections_router.ts +++ b/server/routes/data_connections/data_connections_router.ts @@ -9,7 +9,11 @@ import { IRouter, ResponseError, } from '../../../../../src/core/server'; -import { DATACONNECTIONS_BASE, EDIT } from '../../../common/constants/shared'; +import { + DATACONNECTIONS_BASE, + DATACONNECTIONS_UPDATE_STATUS, + EDIT, +} from '../../../common/constants/shared'; export function registerDataConnectionsRoute(router: IRouter) { router.get( @@ -103,6 +107,39 @@ export function registerDataConnectionsRoute(router: IRouter) { } ); + router.post( + { + path: `${DATACONNECTIONS_BASE}${EDIT}${DATACONNECTIONS_UPDATE_STATUS}`, + validate: { + body: schema.object({ + name: schema.string(), + status: schema.string(), + }), + }, + }, + async (context, request, response): Promise => { + try { + const dataConnectionsresponse = await context.observability_plugin.observabilityClient + .asScoped(request) + .callAsCurrentUser('ppl.modifyDataConnection', { + body: { + name: request.body.name, + status: request.body.status, + }, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in modifying data connection:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + router.post( { path: `${DATACONNECTIONS_BASE}`, diff --git a/test/datasources.ts b/test/datasources.ts index 41ce908f1..c5319b4ae 100644 --- a/test/datasources.ts +++ b/test/datasources.ts @@ -5,12 +5,12 @@ import { AccelerationsCacheData, + AssociatedObject, AsyncPollingResult, CachedDataSourceStatus, DataSourceCacheData, - DatasourceType, DatasourceDetails, - AssociatedObject, + DatasourceType, } from '../common/types/data_connections'; export const showDataConnectionsData = { @@ -89,6 +89,7 @@ export const describeS3Dataconnection = { 'glue.indexstore.opensearch.uri': 'y', 'glue.indexstore.opensearch.region': 'us-west-2', }, + status: 'ACTIVE', }; export const mockRoleData = { @@ -825,10 +826,27 @@ export const mockShowTablesPollingResult: AsyncPollingResult = { { name: 'namespace', type: 'string' }, { name: 'tableName', type: 'string' }, { name: 'isTemporary', type: 'boolean' }, + { name: 'information', type: 'string' }, ], datarows: [ - ['TestDatabase', 'Table1', false], - ['TestDatabase', 'Table2', false], + [ + 'default', + 'dummy_table', + false, + 'Catalog: spark_catalog\nDatabase: default\nTable: dummy_table\nOwner: hadoop\nCreated Time: Sun Mar 10 01:28:06 UTC 2024\nLast Access: UNKNOWN\nCreated By: Spark 3.4.1-amzn-0\nType: MANAGED\nProvider: hive\nTable Properties: [transient_lastDdlTime=1710034086]\nLocation: file:/home/hadoop/spark-warehouse/dummy_table\nSerde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\nInputFormat: org.apache.hadoop.mapred.TextInputFormat\nOutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat\nStorage Properties: [serialization.format=1]\nPartition Provider: Catalog\nSchema: root\n |-- dummy: string (nullable = true)\n\n', + ], + [ + 'default', + 'http_logs1', + false, + 'Catalog: spark_catalog\nDatabase: default\nTable: http_logs\nOwner: hadoop\nCreated Time: Thu Aug 17 01:01:32 UTC 2023\nLast Access: UNKNOWN\nCreated By: Spark 3.3.2-amzn-0\nType: EXTERNAL\nProvider: json\nLocation: s3://flint-dataset/mini_http_logs_partitioned_json_bz2\nSerde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\nInputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat\nOutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat\nStorage Properties: [compression=bzip2]\nPartition Provider: Catalog\nPartition Columns: [`year`, `month`, `day`]\nSchema: root\n |-- @timestamp: timestamp (nullable = true)\n |-- clientip: string (nullable = true)\n |-- request: string (nullable = true)\n |-- status: integer (nullable = true)\n |-- size: integer (nullable = true)\n |-- year: integer (nullable = true)\n |-- month: integer (nullable = true)\n |-- day: integer (nullable = true)\n\n', + ], + [ + 'default', + 'http_logs2', + false, + 'Catalog: spark_catalog\nDatabase: default\nTable: http_logs\nOwner: hadoop\nCreated Time: Thu Aug 17 01:01:32 UTC 2023\nLast Access: UNKNOWN\nCreated By: Spark 3.3.2-amzn-0\nType: EXTERNAL\nProvider: json\nLocation: s3://flint-dataset/mini_http_logs_partitioned_json_bz2\nSerde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\nInputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat\nOutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat\nStorage Properties: [compression=bzip2]\nPartition Provider: Catalog\nPartition Columns: [`year`, `month`, `day`]\nSchema: root\n |-- @timestamp: timestamp (nullable = true)\n |-- clientip: string (nullable = true)\n |-- request: string (nullable = true)\n |-- status: integer (nullable = true)\n |-- size: integer (nullable = true)\n |-- year: integer (nullable = true)\n |-- month: integer (nullable = true)\n |-- day: integer (nullable = true)\n\n', + ], ], };