diff --git a/common/constants/data_connections.ts b/common/constants/data_connections.ts index b265b2290d..573a862475 100644 --- a/common/constants/data_connections.ts +++ b/common/constants/data_connections.ts @@ -19,6 +19,7 @@ export const QUERY_ALL = 'query-all'; export const DatasourceTypeToDisplayName: { [key in DatasourceType]: string } = { PROMETHEUS: 'Prometheus', S3GLUE: 'Amazon S3', + SECURITYLAKE: 'Amazon Security Lake', }; export const PrometheusURL = 'Prometheus'; diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index 4507262750..ddca7919ee 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -47,12 +47,11 @@ export interface AssociatedObject { export type Role = EuiComboBoxOptionOption; -export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; +export type DatasourceType = 'S3GLUE' | 'PROMETHEUS' | 'SECURITYLAKE'; export interface S3GlueProperties { 'glue.indexstore.opensearch.uri': string; 'glue.indexstore.opensearch.region': string; - 'glue.lakeformation.enabled'?: boolean; } export interface PrometheusProperties { @@ -252,10 +251,12 @@ export interface StartLoadingParams { dataSourceMDSId?: string; databaseName?: string; tableName?: string; + dataSourceType?: DatasourceType; } export interface RenderAccelerationFlyoutParams { dataSource: string; + dataSourceType: DatasourceType; dataSourceMDSId?: string; databaseName?: string; tableName?: string; @@ -265,9 +266,9 @@ export interface RenderAccelerationFlyoutParams { export interface RenderAssociatedObjectsDetailsFlyoutParams { tableDetail: AssociatedObject; dataSourceName: string; + dataSourceType: DatasourceType; handleRefresh?: () => void; dataSourceMDSId?: string; - isS3ConnectionWithLakeFormation?: boolean; } export interface RenderAccelerationDetailsFlyoutParams { diff --git a/common/types/integrations.ts b/common/types/integrations.ts new file mode 100644 index 0000000000..74a3a436c5 --- /dev/null +++ b/common/types/integrations.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export type IntegrationConnectionType = 's3' | 'index' | 'securityLake'; diff --git a/public/components/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap b/public/components/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap index f79d97ee56..ae01f99ea1 100644 --- a/public/components/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap +++ b/public/components/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap @@ -319,6 +319,7 @@ exports[`AssociatedObjectsTab Component renders correctly with associated object > @@ -738,9 +739,9 @@ exports[`AssociatedObjectsTab Component renders correctly with associated object }, ] } + dataSourceType="S3GLUE" datasourceName="mock_data_source" handleRefresh={[Function]} - isS3ConnectionWithLakeFormation={false} > diff --git a/public/components/datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap b/public/components/datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap index 04b01d305d..68748fbe6f 100644 --- a/public/components/datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap +++ b/public/components/datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap @@ -409,6 +409,7 @@ exports[`Installed Integrations Table test Renders the installed integrations ta Array [ Object { "assets": 3, + "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea", "locator": Object { "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea", "name": "aws_waf-sample", @@ -444,6 +445,7 @@ exports[`Installed Integrations Table test Renders the installed integrations ta Array [ Object { "assets": 3, + "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea", "locator": Object { "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea", "name": "aws_waf-sample", diff --git a/public/components/datasources/components/__tests__/acceleration_table.test.tsx b/public/components/datasources/components/__tests__/acceleration_table.test.tsx index 2806c095cf..9400215d5a 100644 --- a/public/components/datasources/components/__tests__/acceleration_table.test.tsx +++ b/public/components/datasources/components/__tests__/acceleration_table.test.tsx @@ -95,7 +95,11 @@ describe('AccelerationTable Component', () => { it('renders without crashing', () => { const wrapper = mount( - + ); expect(wrapper).toBeDefined(); }); @@ -111,7 +115,11 @@ describe('AccelerationTable Component', () => { let wrapper: ReactWrapper; await act(async () => { wrapper = mount( - + ); }); @@ -132,7 +140,11 @@ describe('AccelerationTable Component', () => { let wrapper: ReactWrapper; await act(async () => { wrapper = mount( - + ); }); wrapper!.update(); @@ -152,7 +164,11 @@ describe('AccelerationTable Component', () => { let wrapper: ReactWrapper; await act(async () => { wrapper = mount( - + ); await new Promise((resolve) => setTimeout(resolve, 0)); wrapper!.update(); @@ -172,7 +188,11 @@ describe('AccelerationTable Component', () => { let wrapper: ReactWrapper; await act(async () => { wrapper = mount( - + ); }); wrapper!.update(); diff --git a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx index b6e79f359e..3e74c73b6b 100644 --- a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx +++ b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx @@ -21,6 +21,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import { CachedAcceleration, CachedDataSourceStatus, + DatasourceType, } from '../../../../../../common/types/data_connections'; import { DirectQueryLoadingStatus } from '../../../../../../common/types/explorer'; import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager'; @@ -45,7 +46,7 @@ import { interface AccelerationTableProps { dataSourceName: string; cacheLoadingHooks: any; - isS3ConnectionWithLakeFormation: boolean; + dataSourceType: DatasourceType; } interface ModalState { @@ -56,7 +57,7 @@ interface ModalState { export const AccelerationTable = ({ dataSourceName, cacheLoadingHooks, - isS3ConnectionWithLakeFormation, + dataSourceType, }: AccelerationTableProps) => { const [accelerations, setAccelerations] = useState([]); const [updatedTime, setUpdatedTime] = useState(); @@ -171,6 +172,7 @@ export const AccelerationTable = ({ @@ -327,11 +329,12 @@ export const AccelerationTable = ({ }, }; - const accelerationTableColumns = !isS3ConnectionWithLakeFormation - ? Object.values(accelerationTableColumnsCollection) - : Object.entries(accelerationTableColumnsCollection) - .filter(([key]) => key !== 'database' && key !== 'table') - .map(([_key, val]) => val); + const accelerationTableColumns = + dataSourceType.toUpperCase() === 'SECURITYLAKE' + ? Object.entries(accelerationTableColumnsCollection) + .filter(([key]) => key !== 'database' && key !== 'table') + .map(([_key, val]) => val) + : Object.values(accelerationTableColumnsCollection); const pagination = { initialPageSize: 10, diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx index 100005ae50..1172c11c7b 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx @@ -36,7 +36,11 @@ describe('Create acceleration flyout components', () => { coreStartMock.http.get = jest.fn().mockResolvedValue(mockDatasourcesQuery); const wrapper = mount( - + ); wrapper.update(); await waitFor(() => { diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx index 027c7ba615..21d1613684 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx @@ -24,6 +24,7 @@ import { import { CachedTable, CreateAccelerationForm, + DatasourceType, } from '../../../../../../../../common/types/data_connections'; import { DirectQueryLoadingStatus } from '../../../../../../../../common/types/explorer'; import { useLoadTableColumnsToCache } from '../../../../../../../framework/catalog_cache/cache_loader'; @@ -39,11 +40,10 @@ import { QueryVisualEditor } from '../visual_editors/query_visual_editor'; import { CreateAccelerationButton } from './create_acceleration_button'; import { CreateAccelerationHeader } from './create_acceleration_header'; import { hasError } from './utils'; -import { DATACONNECTIONS_BASE } from '../../../../../../../../common/constants/shared'; -import { checkIsConnectionWithLakeFormation } from '../../../../../utils/helpers'; export interface CreateAccelerationProps { selectedDatasource: string; + selectedDatasourceType: DatasourceType; resetFlyout: () => void; databaseName?: string; tableName?: string; @@ -53,6 +53,7 @@ export interface CreateAccelerationProps { export const CreateAcceleration = ({ selectedDatasource, + selectedDatasourceType, resetFlyout, databaseName, tableName, @@ -61,13 +62,12 @@ export const CreateAcceleration = ({ }: CreateAccelerationProps) => { const { setToast } = useToast(); const http = coreRefs!.http; - const [isS3ConnectionWithLakeFormation, setIsS3ConnectionWithLakeFormation] = useState(false); const [accelerationFormData, setAccelerationFormData] = useState({ dataSource: selectedDatasource, database: databaseName ?? '', dataTable: tableName ?? '', dataTableFields: [], - accelerationIndexType: 'skipping', + accelerationIndexType: selectedDatasourceType === 'SECURITYLAKE' ? 'materialized' : 'skipping', skippingIndexQueryData: [], coveringIndexQueryData: [], materializedViewQueryData: { @@ -170,16 +170,6 @@ export const CreateAcceleration = ({ } }; - const updateDataSourceConnectionInfo = () => { - coreRefs.http!.get(`${DATACONNECTIONS_BASE}/${selectedDatasource}`).then((data: any) => { - setIsS3ConnectionWithLakeFormation(checkIsConnectionWithLakeFormation(data)); - }); - }; - - useEffect(() => { - updateDataSourceConnectionInfo(); - }, [selectedDatasource]); - useEffect(() => { if (databaseName !== undefined && tableName !== undefined) { initiateColumnLoad( @@ -245,6 +235,7 @@ export const CreateAcceleration = ({ setDataSourceFormData: setAccelerationFormData, }} selectedDatasource={selectedDatasource} + selectedDataSourceType={selectedDatasourceType} dataSourcesPreselected={dataSourcesPreselected} tableFieldsLoading={tableFieldsLoading} dataSourceMDSId={dataSourceMDSId} @@ -252,7 +243,7 @@ export const CreateAcceleration = ({ diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx index d22380e43c..0cc4dc5aa4 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx @@ -21,9 +21,9 @@ describe('Index type selector components', () => { const wrapper = mount( ); wrapper.update(); @@ -46,9 +46,9 @@ describe('Index type selector components', () => { const wrapper = mount( ); wrapper.update(); diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/index_type_selector.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/index_type_selector.tsx index 6f36840ab1..9e3855e717 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/index_type_selector.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/index_type_selector.tsx @@ -3,8 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiFormRow, EuiLink, EuiSpacer, EuiSuperSelect, EuiText } from '@elastic/eui'; -import React, { Fragment, useEffect, useState, useRef } from 'react'; +import { + EuiFormRow, + EuiLink, + EuiSpacer, + EuiSuperSelect, + EuiSuperSelectOption, + EuiText, +} from '@elastic/eui'; +import React, { Fragment, useEffect, useState } from 'react'; import { ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME, ACC_INDEX_TYPE_DOCUMENTATION_URL, @@ -12,32 +19,25 @@ import { import { AccelerationIndexType, CreateAccelerationForm, + DatasourceType, } from '../../../../../../../../common/types/data_connections'; interface IndexTypeSelectorProps { accelerationFormData: CreateAccelerationForm; - isS3ConnectionWithLakeFormation: boolean; + dataSourceType: DatasourceType; setAccelerationFormData: React.Dispatch>; initiateColumnLoad: (dataSource: string, database: string, dataTable: string) => void; } export const IndexTypeSelector = ({ accelerationFormData, - isS3ConnectionWithLakeFormation, + dataSourceType, setAccelerationFormData, initiateColumnLoad, }: IndexTypeSelectorProps) => { - const [value, setValue] = useState('skipping'); - // This is used to track if user changed the value - // If so, we skip changing it based on 'isS3ConnectionWithLakeFormation' in the effect below - const valueSetAlready = useRef(false); - - useEffect(() => { - if (!valueSetAlready.current) { - const defaultSelectedOption = isS3ConnectionWithLakeFormation ? 'materialized' : 'skipping'; - updateState(defaultSelectedOption); - } - }, [isS3ConnectionWithLakeFormation]); + const [value, setValue] = useState( + dataSourceType === 'SECURITYLAKE' ? 'materialized' : 'skipping' + ); useEffect(() => { initiateColumnLoad( @@ -47,41 +47,41 @@ export const IndexTypeSelector = ({ ); }, [accelerationFormData.dataTable]); - const updateState = (indexType: string) => { + const updateState = (indexType: AccelerationIndexType) => { setAccelerationFormData({ ...accelerationFormData, - accelerationIndexType: indexType as AccelerationIndexType, + accelerationIndexType: indexType, accelerationIndexName: indexType === 'skipping' ? ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME : '', }); setValue(indexType); }; - const onChangeSupeSelect = (indexType: string) => { + const onChangeSupeSelect = (indexType: AccelerationIndexType) => { updateState(indexType); - valueSetAlready.current = true; }; - const baseOptions = !isS3ConnectionWithLakeFormation - ? [ - { - value: 'skipping', - inputDisplay: 'Skipping index', - dropdownDisplay: ( - - Skipping index - -

- Accelerate direct queries by storing table meta-data in OpenSearch. -

- - - ), - }, - ] - : []; + const baseOptions: Array> = + dataSourceType.toUpperCase() !== 'SECURITYLAKE' + ? [ + { + value: 'skipping', + inputDisplay: 'Skipping index', + dropdownDisplay: ( + + Skipping index + +

+ Accelerate direct queries by storing table meta-data in OpenSearch. +

+ + + ), + }, + ] + : []; - const superSelectOptions = [ + const superSelectOptions: Array> = [ ...baseOptions, { value: 'covering', diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx index 18fdbf32f4..9cdc4c0c4a 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx @@ -11,9 +11,11 @@ import { useLoadTablesToCache, } from '../../../../../../../../framework/catalog_cache/cache_loader'; import { useToast } from '../../../../../../../common/toast'; +import { DatasourceType } from '../../../../../../../../../common/types/data_connections'; interface SelectorLoadDatabasesProps { dataSourceName: string; + dataSourceType: DatasourceType; databaseName: string; loadTables: () => void; loadingComboBoxes: { @@ -34,6 +36,7 @@ interface SelectorLoadDatabasesProps { export const SelectorLoadObjects = ({ dataSourceName, + dataSourceType, databaseName, loadTables, loadingComboBoxes, @@ -67,7 +70,7 @@ export const SelectorLoadObjects = ({ tableStatus: true, accelerationsStatus: true, }); - startLoadingTables({ dataSourceName, databaseName, dataSourceMDSId }); + startLoadingTables({ dataSourceName, databaseName, dataSourceMDSId, dataSourceType }); startLoadingAccelerations({ dataSourceName, dataSourceMDSId }); }; diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx index 9c32a12909..4810fd56f4 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx @@ -23,6 +23,7 @@ import { CachedDataSourceStatus, CachedDatabase, CreateAccelerationForm, + DatasourceType, } from '../../../../../../../../common/types/data_connections'; import { CatalogCacheManager } from '../../../../../../../framework/catalog_cache/cache_manager'; import { useToast } from '../../../../../../common/toast'; @@ -51,20 +52,24 @@ interface DataSourceSelectorProps { http: CoreStart['http']; dataSourceFormProps: DataSourceFormProps; selectedDatasource: string; + selectedDataSourceType: DatasourceType; dataSourcesPreselected: boolean; tableFieldsLoading: boolean; dataSourceMDSId?: string; hideHeader?: boolean; + hideDataSourceDescription?: boolean; } export const DataSourceSelector: React.FC = ({ http, - dataSourceFormProps: { dataSourceFormData, setDataSourceFormData }, + dataSourceFormProps: { dataSourceFormData, formType, setDataSourceFormData }, selectedDatasource, + selectedDataSourceType, dataSourcesPreselected, tableFieldsLoading, dataSourceMDSId, hideHeader, + hideDataSourceDescription, }) => { const { setToast } = useToast(); const [databases, setDatabases] = useState>>([]); @@ -79,7 +84,7 @@ export const DataSourceSelector: React.FC = ({ dataTable: false, }); - const dataSourceDescription = ( + const dataSourceDescription = hideDataSourceDescription ? null : ( Data source {dataSourceFormData.dataSource} @@ -87,13 +92,16 @@ export const DataSourceSelector: React.FC = ({ ); const loadDataSource = () => { + if (formType === 'SetupIntegration') { + return; + } setLoadingComboBoxes({ ...loadingComboBoxes, dataSource: true }); http .get(`${DATACONNECTIONS_BASE}/dataSourceMDSId=${dataSourceMDSId ?? ''}`) .then((res) => { const isValidDataSource = res.some( (connection: any) => - connection.connector.toUpperCase() === 'S3GLUE' && + ['S3GLUE', 'SECURITYLAKE'].includes(connection.connector.toUpperCase()) && connection.name === selectedDatasource ); @@ -296,6 +304,7 @@ export const DataSourceSelector: React.FC = ({ renderCreateAccelerationFlyout({ dataSource: dataSourceName, + dataSourceType, handleRefresh, }) } diff --git a/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx b/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx index c57496234d..04424ecd4b 100644 --- a/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx +++ b/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx @@ -25,11 +25,13 @@ import { } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import React, { useEffect, useState } from 'react'; +import { isArray } from 'lodash'; import { DATA_SOURCE_TYPES } from '../../../../../../common/constants/data_sources'; import { AssociatedObject, CachedAcceleration, CachedColumn, + DatasourceType, } from '../../../../../../common/types/data_connections'; import { DirectQueryLoadingStatus } from '../../../../../../common/types/explorer'; import { useToast } from '../../../../../../public/components/common/toast'; @@ -53,10 +55,10 @@ import { export interface AssociatedObjectsFlyoutProps { tableDetail: AssociatedObject; datasourceName: string; + dataSourceType: DatasourceType; resetFlyout: () => void; handleRefresh?: () => void; dataSourceMDSId?: string; - isS3ConnectionWithLakeFormation?: boolean; } export const AssociatedObjectsDetailsFlyout = ({ @@ -65,7 +67,7 @@ export const AssociatedObjectsDetailsFlyout = ({ resetFlyout, handleRefresh, dataSourceMDSId, - isS3ConnectionWithLakeFormation, + dataSourceType, }: AssociatedObjectsFlyoutProps) => { const { loadStatus, startLoading } = useLoadTableColumnsToCache(); const [tableColumns, setTableColumns] = useState([]); @@ -96,6 +98,7 @@ export const AssociatedObjectsDetailsFlyout = ({ const onCreateAcceleration = () => renderCreateAccelerationFlyout({ dataSource: datasourceName, + dataSourceType, databaseName: tableDetail.database, tableName: tableDetail.name, handleRefresh, @@ -148,10 +151,17 @@ export const AssociatedObjectsDetailsFlyout = ({ ); }; - const accelerationData = tableDetail.accelerations.map((acc, index) => ({ - ...acc, - id: index, - })); + const accelerationData = isArray(tableDetail.accelerations) + ? tableDetail.accelerations.map((acc, index) => ({ + ...acc, + id: index, + })) + : [ + { + ...tableDetail.accelerations, + id: 0, + }, + ]; const accelerationColumns = [ { @@ -159,7 +169,7 @@ export const AssociatedObjectsDetailsFlyout = ({ name: 'Name', 'data-test-subj': 'accelerationName', render: (_: string, item: CachedAcceleration) => { - const name = getAccelerationName(item, datasourceName); + const name = getAccelerationName(item); return ( @@ -210,6 +220,7 @@ export const AssociatedObjectsDetailsFlyout = ({ onClick={() => renderCreateAccelerationFlyout({ dataSource: datasourceName, + dataSourceType, databaseName: tableDetail.database, tableName: tableDetail.name, handleRefresh, @@ -321,14 +332,14 @@ export const AssociatedObjectsDetailsFlyout = ({ - {isS3ConnectionWithLakeFormation && ( + {dataSourceType === 'SECURITYLAKE' && ( Create acceleration diff --git a/public/components/datasources/components/manage/associated_objects/associated_objects_tab.tsx b/public/components/datasources/components/manage/associated_objects/associated_objects_tab.tsx index bd1e962b77..14bc598a07 100644 --- a/public/components/datasources/components/manage/associated_objects/associated_objects_tab.tsx +++ b/public/components/datasources/components/manage/associated_objects/associated_objects_tab.tsx @@ -14,7 +14,7 @@ import { EuiSelectableOption, } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import React, { useEffect, useState, useMemo } from 'react'; +import React, { useEffect, useState } from 'react'; import { ACCELERATION_INDEX_TYPES } from '../../../../../../common/constants/data_sources'; import { AssociatedObject, @@ -48,7 +48,6 @@ import { ASSC_OBJ_PANEL_TITLE_FOR_S3_WITH_LAKE_FORMATION, isCatalogCacheFetching, } from './utils/associated_objects_tab_utils'; -import { checkIsConnectionWithLakeFormation } from '../../../utils/helpers'; export interface AssociatedObjectsTabProps { datasource: DatasourceDetails; @@ -59,10 +58,6 @@ export interface AssociatedObjectsTabProps { export const AssociatedObjectsTab: React.FC = (props) => { const { datasource, cacheLoadingHooks, selectedDatabase, setSelectedDatabase } = props; - const isS3ConnectionWithLakeFormation = useMemo( - () => checkIsConnectionWithLakeFormation(datasource), - [datasource] - ); const [isRefreshing, setIsRefreshing] = useState(false); const [lastUpdated, setLastUpdated] = useState(new Date().toLocaleString()); const [isObjectsLoading, setIsObjectsLoading] = useState(false); @@ -112,15 +107,17 @@ export const AssociatedObjectsTab: React.FC = (props) const AssociatedObjectsHeader = () => { const panelTitle = i18n.translate('datasources.associatedObjectsTab.panelTitle', { - defaultMessage: isS3ConnectionWithLakeFormation - ? ASSC_OBJ_PANEL_TITLE_FOR_S3_WITH_LAKE_FORMATION - : ASSC_OBJ_PANEL_TITLE, + defaultMessage: + datasource.connector === 'SECURITYLAKE' + ? ASSC_OBJ_PANEL_TITLE_FOR_S3_WITH_LAKE_FORMATION + : ASSC_OBJ_PANEL_TITLE, }); const panelDescription = i18n.translate('datasources.associatedObjectsTab.panelDescription', { - defaultMessage: isS3ConnectionWithLakeFormation - ? ASSC_OBJ_PANEL_DESCRIPTION_FOR_S3_WITH_LAKE_FORMATION - : ASSC_OBJ_PANEL_DESCRIPTION, + defaultMessage: + datasource.connector === 'SECURITYLAKE' + ? ASSC_OBJ_PANEL_DESCRIPTION_FOR_S3_WITH_LAKE_FORMATION + : ASSC_OBJ_PANEL_DESCRIPTION, }); const LastUpdatedText = () => { @@ -156,10 +153,11 @@ export const AssociatedObjectsTab: React.FC = (props) onClick={onRefreshButtonClick} /> - {!isS3ConnectionWithLakeFormation && ( + {datasource.connector !== 'SECURITYLAKE' && ( @@ -336,28 +334,31 @@ export const AssociatedObjectsTab: React.FC = (props) columns: table.columns, }; }); - // For data connections using lake formation we don't want to show accelerations, so we simply assign empty array - const accelerationObjects: AssociatedObject[] = isS3ConnectionWithLakeFormation - ? [] - : cachedAccelerations - .filter((acceleration: CachedAcceleration) => acceleration.database === selectedDatabase) - .map((acceleration: CachedAcceleration) => ({ - tableName: acceleration.table, - datasource: datasource.name, - id: acceleration.indexName, - name: getAccelerationName(acceleration), - database: acceleration.database, - type: ACCELERATION_INDEX_TYPES.find( - (accelType) => accelType.value === acceleration.type - )!.value as AssociatedObjectIndexType, - accelerations: - acceleration.type === 'covering' || acceleration.type === 'skipping' - ? tableObjects.find( - (tableObject: AssociatedObject) => tableObject.name === acceleration.table - ) || [] - : [], - columns: undefined, - })); + // For security lake connections we don't want to show accelerations, so we simply assign empty array + const accelerationObjects: AssociatedObject[] = + datasource.connector === 'SECURITYLAKE' + ? [] + : cachedAccelerations + .filter( + (acceleration: CachedAcceleration) => acceleration.database === selectedDatabase + ) + .map((acceleration: CachedAcceleration) => ({ + tableName: acceleration.table, + datasource: datasource.name, + id: acceleration.indexName, + name: getAccelerationName(acceleration), + database: acceleration.database, + type: ACCELERATION_INDEX_TYPES.find( + (accelType) => accelType.value === acceleration.type + )!.value as AssociatedObjectIndexType, + accelerations: + acceleration.type === 'covering' || acceleration.type === 'skipping' + ? tableObjects.find( + (tableObject: AssociatedObject) => tableObject.name === acceleration.table + ) || [] + : [], + columns: undefined, + })); setAssociatedObjects([...tableObjects, ...accelerationObjects]); }, [selectedDatabase, cachedTables, cachedAccelerations]); @@ -366,7 +367,7 @@ export const AssociatedObjectsTab: React.FC = (props) return ( <> - {!isS3ConnectionWithLakeFormation && ( + {datasource.connector !== 'SECURITYLAKE' && ( <> @@ -417,9 +418,9 @@ export const AssociatedObjectsTab: React.FC = (props) ).length > 0 ? ( ) : ( diff --git a/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx b/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx index d68173c46a..354e88f8ab 100644 --- a/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx +++ b/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx @@ -18,6 +18,7 @@ import { import { AssociatedObject, CachedAcceleration, + DatasourceType, } from '../../../../../../../common/types/data_connections'; import { getRenderAccelerationDetailsFlyout, @@ -36,9 +37,9 @@ import { interface AssociatedObjectsTableProps { datasourceName: string; + dataSourceType: DatasourceType; associatedObjects: AssociatedObject[]; cachedAccelerations: CachedAcceleration[]; - isS3ConnectionWithLakeFormation: boolean; handleRefresh: () => void; } @@ -56,9 +57,9 @@ interface AssociatedTableFilter { export const AssociatedObjectsTable = ({ datasourceName, + dataSourceType, associatedObjects, cachedAccelerations, - isS3ConnectionWithLakeFormation, handleRefresh, }: AssociatedObjectsTableProps) => { const [accelerationFilterOptions, setAccelerationFilterOptions] = useState([]); @@ -68,7 +69,7 @@ export const AssociatedObjectsTable = ({ { field: 'name', name: i18n.translate('datasources.associatedObjectsTab.column.name', { - defaultMessage: isS3ConnectionWithLakeFormation ? 'Table' : 'Name', + defaultMessage: dataSourceType === 'SECURITYLAKE' ? 'Table' : 'Name', }), sortable: true, 'data-test-subj': 'nameCell', @@ -79,7 +80,7 @@ export const AssociatedObjectsTable = ({ renderAssociatedObjectsDetailsFlyout({ tableDetail: item, dataSourceName: datasourceName, - isS3ConnectionWithLakeFormation, + dataSourceType, handleRefresh, }); } else { @@ -100,7 +101,7 @@ export const AssociatedObjectsTable = ({ { field: 'accelerations', name: i18n.translate('datasources.associatedObjectsTab.column.accelerations', { - defaultMessage: isS3ConnectionWithLakeFormation ? 'Accelerations' : 'Associations', + defaultMessage: dataSourceType === 'SECURITYLAKE' ? 'Accelerations' : 'Associations', }), sortable: true, render: (accelerations: CachedAcceleration[] | AssociatedObject, obj: AssociatedObject) => { @@ -129,7 +130,7 @@ export const AssociatedObjectsTable = ({ renderAssociatedObjectsDetailsFlyout({ tableDetail: obj, dataSourceName: datasourceName, - isS3ConnectionWithLakeFormation, + dataSourceType, handleRefresh, }) } @@ -144,7 +145,7 @@ export const AssociatedObjectsTable = ({ renderAssociatedObjectsDetailsFlyout({ tableDetail: accelerations, dataSourceName: datasourceName, - isS3ConnectionWithLakeFormation, + dataSourceType, handleRefresh, }) } @@ -176,6 +177,7 @@ export const AssociatedObjectsTable = ({ onClick: (item: AssociatedObject) => renderCreateAccelerationFlyout({ dataSource: datasourceName, + dataSourceType, databaseName: item.database, tableName: item.tableName, handleRefresh, @@ -214,7 +216,7 @@ export const AssociatedObjectsTable = ({ }, ] as Array>; - if (!isS3ConnectionWithLakeFormation) { + if (dataSourceType !== 'SECURITYLAKE') { columns.splice(1, 0, { field: 'type', name: i18n.translate('datasources.associatedObjectsTab.column.type', { @@ -274,9 +276,10 @@ export const AssociatedObjectsTable = ({ filters: searchFilters, box: { incremental: true, - placeholder: isS3ConnectionWithLakeFormation - ? ASSC_OBJ_TABLE_FOR_S3_WITH_LAKE_FORMATION_SEARCH_HINT - : ASSC_OBJ_TABLE_SEARCH_HINT, + placeholder: + dataSourceType === 'SECURITYLAKE' + ? ASSC_OBJ_TABLE_FOR_S3_WITH_LAKE_FORMATION_SEARCH_HINT + : ASSC_OBJ_TABLE_SEARCH_HINT, schema: { fields: { name: { type: 'string' }, database: { type: 'string' } }, }, diff --git a/public/components/datasources/components/manage/connection_details.tsx b/public/components/datasources/components/manage/connection_details.tsx index f78272b73e..ec85bdd6cc 100644 --- a/public/components/datasources/components/manage/connection_details.tsx +++ b/public/components/datasources/components/manage/connection_details.tsx @@ -4,7 +4,7 @@ */ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiHorizontalRule } from '@elastic/eui'; -import React, { useState } from 'react'; +import React from 'react'; import { EuiPanel } from '@elastic/eui'; import { ConnectionManagementCallout } from './connection_management_callout'; import { PrometheusProperties, S3GlueProperties } from './data_connection'; @@ -113,7 +113,7 @@ export const ConnectionDetails = (props: ConnectionDetailProps) => { - {connector === 'S3GLUE' ? ( + {connector === 'S3GLUE' || connector === 'SECURITYLAKE' ? ( ) : ( diff --git a/public/components/datasources/components/manage/data_connection.tsx b/public/components/datasources/components/manage/data_connection.tsx index 8819615c35..e5fcb3c709 100644 --- a/public/components/datasources/components/manage/data_connection.tsx +++ b/public/components/datasources/components/manage/data_connection.tsx @@ -30,6 +30,7 @@ import { import { DatasourceDetails, PrometheusProperties, + StartLoadingParams, } from '../../../../../common/types/data_connections'; import { useLoadAccelerationsToCache, @@ -48,7 +49,6 @@ import { InstallIntegrationFlyout, InstalledIntegrationsTable, } from './integrations/installed_integrations_table'; -import { checkIsConnectionWithLakeFormation } from '../../utils/helpers'; const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); @@ -62,9 +62,6 @@ export const DataConnection = (props: { dataSource: string }) => { properties: { 'prometheus.uri': 'placeholder' }, status: 'ACTIVE', }); - const [isS3ConnectionWithLakeFormation, setIsS3ConnectionWithLakeFormation] = useState( - false - ); const [hasAccess, setHasAccess] = useState(true); const { http, chrome, application } = coreRefs; const [selectedDatabase, setSelectedDatabase] = useState(''); @@ -85,7 +82,12 @@ export const DataConnection = (props: { dataSource: string }) => { databasesLoadStatus, startLoadingDatabases, tablesLoadStatus, - startLoadingTables, + startLoadingTables: (loadingParams: StartLoadingParams) => { + startLoadingTables({ + ...loadingParams, + dataSourceType: datasourceDetails.connector, + }); + }, accelerationsLoadStatus, startLoadingAccelerations, }; @@ -96,10 +98,6 @@ export const DataConnection = (props: { dataSource: string }) => { const [refreshIntegrationsFlag, setRefreshIntegrationsFlag] = useState(false); const refreshInstances = () => setRefreshIntegrationsFlag((prev) => !prev); - useEffect(() => { - setIsS3ConnectionWithLakeFormation(checkIsConnectionWithLakeFormation(datasourceDetails)); - }, [datasourceDetails]); - useEffect(() => { const searchDataSourcePattern = new RegExp(`flint_${escapeRegExp(datasourceDetails.name)}_.*`); const findIntegrations = async () => { @@ -128,12 +126,11 @@ export const DataConnection = (props: { dataSource: string }) => { closeFlyout={() => setShowIntegrationsFlyout(false)} datasourceType={datasourceDetails.connector} datasourceName={datasourceDetails.name} - isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} /> ) : null; const onclickAccelerationsCard = () => { - renderCreateAccelerationFlyout({ dataSource }); + renderCreateAccelerationFlyout({ dataSource, dataSourceType: datasourceDetails.connector }); }; const onclickDiscoverCard = () => { @@ -236,55 +233,54 @@ export const DataConnection = (props: { dataSource: string }) => { }, ]; - const conditionalTabs = - datasourceDetails.connector === 'S3GLUE' - ? [ - { - id: 'associated_objects', - name: isS3ConnectionWithLakeFormation ? 'Tables' : 'Associated Objects', - disabled: false, - content: ( - - ), - }, - { - id: 'acceleration_table', - name: 'Accelerations', - disabled: false, - content: ( - - ), - }, - { - id: 'installed_integrations', - name: 'Installed Integrations', - disabled: false, - content: ( - - ), - }, - ] - : []; + const conditionalTabs = ['S3GLUE', 'SECURITYLAKE'].includes(datasourceDetails.connector) + ? [ + { + id: 'associated_objects', + name: datasourceDetails.connector === 'SECURITYLAKE' ? 'Tables' : 'Associated Objects', + disabled: false, + content: ( + + ), + }, + { + id: 'acceleration_table', + name: 'Accelerations', + disabled: false, + content: ( + + ), + }, + { + id: 'installed_integrations', + name: 'Installed Integrations', + disabled: false, + content: ( + + ), + }, + ] + : []; const tabs = [...conditionalTabs, ...genericTabs]; const QueryOrAccelerateData = () => { switch (datasourceDetails.connector) { + case 'SECURITYLAKE': case 'S3GLUE': return ; case 'PROMETHEUS': @@ -384,6 +380,7 @@ export const DataConnection = (props: { dataSource: string }) => { const DatasourceOverview = () => { switch (datasourceDetails.connector) { + case 'SECURITYLAKE': case 'S3GLUE': return ; case 'PROMETHEUS': diff --git a/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx b/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx index de2d7820ee..7583e5c88e 100644 --- a/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx +++ b/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx @@ -27,8 +27,10 @@ import { AvailableIntegrationsTable } from '../../../../integrations/components/ import { INTEGRATIONS_BASE } from '../../../../../../common/constants/shared'; import { AvailableIntegrationsList } from '../../../../integrations/components/available_integration_overview_page'; import { DatasourceType } from '../../../../../../common/types/data_connections'; +import { isS3Connection } from '../../../utils/helpers'; interface IntegrationInstanceTableEntry { + id: string; name: string; locator: { name: string; @@ -40,6 +42,8 @@ interface IntegrationInstanceTableEntry { const labelFromDataSourceType = (dsType: DatasourceType): string | null => { switch (dsType) { + case 'SECURITYLAKE': + return 'Amazon Security Lake'; case 'S3GLUE': return 'Flint S3'; case 'PROMETHEUS': @@ -79,6 +83,7 @@ const instanceToTableEntry = ( instance: IntegrationInstanceResult ): IntegrationInstanceTableEntry => { return { + id: instance.id, name: instance.name, locator: { name: instance.name, id: instance.id }, status: instance.status, @@ -126,15 +131,13 @@ const NoInstalledIntegrations = ({ toggleFlyout }: { toggleFlyout: () => void }) export interface InstallIntegrationFlyoutProps { datasourceType: DatasourceType; datasourceName: string; - isS3ConnectionWithLakeFormation?: boolean; closeFlyout: () => void; - refreshInstances: () => void; + refreshInstances?: () => void; } export const InstallIntegrationFlyout = ({ datasourceType, datasourceName, - isS3ConnectionWithLakeFormation, closeFlyout, refreshInstances, }: InstallIntegrationFlyoutProps) => { @@ -153,9 +156,12 @@ export const InstallIntegrationFlyout = ({ }); }, []); - const s3FilteredIntegrations = { + const integrationLabelToCheck = + datasourceType === 'SECURITYLAKE' ? 'Security Lake' : labelFromDataSourceType(datasourceType); + + const integrationsFilteredByLabel = { hits: availableIntegrations.hits.filter((config) => - config.labels?.includes(labelFromDataSourceType(datasourceType) ?? '') + config.labels?.includes(integrationLabelToCheck ?? '') ), }; @@ -171,7 +177,7 @@ export const InstallIntegrationFlyout = ({ {installingIntegration === null ? ( @@ -181,13 +187,10 @@ export const InstallIntegrationFlyout = ({ unsetIntegration={() => setInstallingIntegration(null)} renderType="flyout" forceConnection={ - datasourceType === 'S3GLUE' + isS3Connection(datasourceType) ? { name: datasourceName, - type: 's3', - properties: { - lakeFormationEnabled: isS3ConnectionWithLakeFormation, - }, + type: datasourceType.toLowerCase() === 'securitylake' ? 'securityLake' : 's3', } : undefined } @@ -195,7 +198,7 @@ export const InstallIntegrationFlyout = ({ setIsInstalling(installing); if (success) { closeFlyout(); - refreshInstances(); + refreshInstances?.(); } }} /> @@ -208,13 +211,11 @@ export const InstalledIntegrationsTable = ({ integrations, datasourceType, datasourceName, - isS3ConnectionWithLakeFormation, refreshInstances, }: { integrations: IntegrationInstanceResult[]; datasourceType: DatasourceType; datasourceName: string; - isS3ConnectionWithLakeFormation?: boolean; refreshInstances: () => void; }) => { const [query, setQuery] = useState(''); @@ -265,7 +266,6 @@ export const InstalledIntegrationsTable = ({ closeFlyout={() => setShowAvailableFlyout(false)} datasourceType={datasourceType} datasourceName={datasourceName} - isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} refreshInstances={refreshInstances} /> ) : null} diff --git a/public/components/datasources/components/manage/manage_data_connections_table.tsx b/public/components/datasources/components/manage/manage_data_connections_table.tsx index bc32ea6645..f68fa138b2 100644 --- a/public/components/datasources/components/manage/manage_data_connections_table.tsx +++ b/public/components/datasources/components/manage/manage_data_connections_table.tsx @@ -37,7 +37,6 @@ import S3Logo from '../../icons/s3-logo.svg'; import SecurityLakeLogo from '../../icons/security-lake-logo.svg'; import { DataConnectionsHeader } from '../data_connections_header'; import { redirectToExplorerS3 } from './associated_objects/utils/associated_objects_tab_utils'; -import { checkIsConnectionWithLakeFormation } from '../../utils/helpers'; import { InstallIntegrationFlyout } from './integrations/installed_integrations_table'; import { DataConnectionsDescription } from './manage_data_connections_description'; @@ -45,7 +44,6 @@ interface DataConnection { connectionType: DatasourceType; name: string; dsStatus: DatasourceStatus; - isConnectionWithLakeFormation: boolean; } export const ManageDataConnectionsTable = (props: HomeProps) => { @@ -87,7 +85,6 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { name, connectionType: connector, dsStatus: status, - isConnectionWithLakeFormation: checkIsConnectionWithLakeFormation(dataSourceDetails), }; } ); @@ -142,7 +139,7 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { onClick: (datasource: DataConnection) => { if (datasource.connectionType === 'PROMETHEUS') { application!.navigateToApp(observabilityMetricsID); - } else if (datasource.connectionType === 'S3GLUE') { + } else if (['S3GLUE', 'SECURITYLAKE'].includes(datasource.connectionType)) { redirectToExplorerS3(datasource.name); } }, @@ -155,7 +152,10 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { type: 'icon', available: (datasource: DataConnection) => datasource.connectionType !== 'PROMETHEUS', onClick: (datasource: DataConnection) => { - renderCreateAccelerationFlyout({ dataSource: datasource.name }); + renderCreateAccelerationFlyout({ + dataSource: datasource.name, + dataSourceType: datasource.connectionType, + }); }, 'data-test-subj': 'action-accelerate', }, @@ -171,7 +171,6 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { closeFlyout={() => setShowIntegrationsFlyout(false)} datasourceType={datasource.connectionType} datasourceName={datasource.name} - isS3ConnectionWithLakeFormation={datasource.isConnectionWithLakeFormation} /> ); setShowIntegrationsFlyout(true); @@ -192,8 +191,10 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { const icon = (record: DataConnection) => { switch (record.connectionType) { + case 'SECURITYLAKE': + return ; case 'S3GLUE': - return ; + return ; case 'PROMETHEUS': return ; default: @@ -228,9 +229,9 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { case 'PROMETHEUS': return 'Prometheus'; case 'S3GLUE': - return connection.isConnectionWithLakeFormation - ? 'Amazon Security Lake' - : 'Amazon S3 with AWS Glue Data Catalog'; + return 'Amazon S3 with AWS Glue Data Catalog'; + case 'SECURITYLAKE': + return 'Amazon Security Lake'; default: return '-'; } @@ -261,17 +262,14 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { }, }; - const entries = data.map( - ({ name, connectionType, dsStatus, isConnectionWithLakeFormation }: DataConnection) => { - return { - connectionType, - name, - dsStatus, - data: { name, connectionType }, - isConnectionWithLakeFormation, - }; - } - ); + const entries = data.map(({ name, connectionType, dsStatus }: DataConnection) => { + return { + connectionType, + name, + dsStatus, + data: { name, connectionType }, + }; + }); const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); diff --git a/public/components/datasources/utils/helpers.ts b/public/components/datasources/utils/helpers.ts index dce500a108..76fac37aa5 100644 --- a/public/components/datasources/utils/helpers.ts +++ b/public/components/datasources/utils/helpers.ts @@ -3,12 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { S3GlueProperties } from 'common/types/data_connections'; -import { DatasourceDetails } from '../components/manage/data_connection'; +import { DatasourceType } from '../../../../common/types/data_connections'; -export function checkIsConnectionWithLakeFormation({ - connector, - properties, -}: DatasourceDetails): boolean { - return connector === 'S3GLUE' && !!(properties as S3GlueProperties)['glue.lakeformation.enabled']; +export function isS3Connection(dataSourceType: DatasourceType): boolean { + return ['s3glue', 'securitylake'].includes(dataSourceType?.toLowerCase()); } diff --git a/public/components/event_analytics/explorer/datasources/tables_flyout.tsx b/public/components/event_analytics/explorer/datasources/tables_flyout.tsx index f11861d3eb..87b66e6967 100644 --- a/public/components/event_analytics/explorer/datasources/tables_flyout.tsx +++ b/public/components/event_analytics/explorer/datasources/tables_flyout.tsx @@ -30,6 +30,7 @@ import { CachedDataSourceStatus, CachedDatabase, CachedTable, + DatasourceType, } from '../../../../../common/types/data_connections'; import { isCatalogCacheFetching } from '../../../datasources/components/manage/associated_objects/utils/associated_objects_tab_utils'; import { DirectQueryLoadingStatus } from '../../../../../common/types/explorer'; @@ -40,6 +41,7 @@ import { AssociatedObjectsTabEmpty } from '../../../datasources/components/manag export interface TablesFlyoutProps { dataSourceName: string; + dataSourceType: DatasourceType; resetFlyout: () => void; } @@ -48,7 +50,11 @@ interface TableItemType { type: string; } -export const TablesFlyout = ({ dataSourceName, resetFlyout }: TablesFlyoutProps) => { +export const TablesFlyout = ({ + dataSourceName, + dataSourceType, + resetFlyout, +}: TablesFlyoutProps) => { const { setToast } = useToast(); const { loadStatus: databasesLoadStatus, @@ -140,7 +146,7 @@ export const TablesFlyout = ({ dataSourceName, resetFlyout }: TablesFlyoutProps) databaseCache.status === CachedDataSourceStatus.Failed) && !isCatalogCacheFetching(tablesLoadStatus) ) { - startLoadingTables({ dataSourceName, databaseName: selectedDatabase }); + startLoadingTables({ dataSourceName, databaseName: selectedDatabase, dataSourceType }); setIsObjectsLoading(true); } else if (databaseCache.status === CachedDataSourceStatus.Updated) { setCachedTables(databaseCache.tables); diff --git a/public/components/event_analytics/explorer/explorer.tsx b/public/components/event_analytics/explorer/explorer.tsx index 0f7ed30d22..1dd3fc22c0 100644 --- a/public/components/event_analytics/explorer/explorer.tsx +++ b/public/components/event_analytics/explorer/explorer.tsx @@ -65,6 +65,7 @@ import { } from '../../../../common/constants/explorer'; import { QUERY_ASSIST_API } from '../../../../common/constants/query_assist'; import { + DATACONNECTIONS_BASE, LIVE_END_TIME, LIVE_OPTIONS, PPL_DESCRIBE_INDEX_REGEX, @@ -141,6 +142,7 @@ import { getRenderLogExplorerTablesFlyout, } from '../../../plugin'; import { AccelerateCallout } from './accelerate_callout'; +import { DatasourceType } from '../../../../common/types/data_connections'; export const Explorer = ({ pplService, @@ -212,11 +214,18 @@ export const Explorer = ({ const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT); const [triggerAvailability, setTriggerAvailability] = useState(false); const [isQueryRunning, setIsQueryRunning] = useState(false); + const [dataSourceConnectionType, setDataSourceConnectionType] = useState( + 'PROMETHEUS' + ); const dataSourceName = explorerSearchMeta?.datasources[0]?.label; const renderTablesFlyout = getRenderLogExplorerTablesFlyout(); const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); const isS3Connection = explorerSearchMeta.datasources?.[0]?.type === 's3glue'; - const onCreateAcceleration = () => renderCreateAccelerationFlyout(dataSourceName); + const onCreateAcceleration = () => + renderCreateAccelerationFlyout({ + dataSource: dataSourceName, + dataSourceType: dataSourceConnectionType, + }); const currentPluggable = useMemo(() => { return explorerSearchMeta.datasources?.[0]?.type ? dataSourcePluggables[explorerSearchMeta?.datasources[0]?.type] @@ -259,6 +268,16 @@ export const Explorer = ({ liveTailNameRef.current = liveTailName; tempQueryRef.current = tempQuery; + const updateDataSourceConnectionInfo = () => { + coreRefs.http!.get(`${DATACONNECTIONS_BASE}/${dataSourceName}`).then((data: any) => { + setDataSourceConnectionType(data.connector); + }); + }; + + useEffect(() => { + updateDataSourceConnectionInfo(); + }, [dataSourceName]); + const findAutoInterval = (start: string = '', end: string = '') => { const minInterval = findMinInterval(start, end); @@ -694,7 +713,7 @@ export const Explorer = ({ ) : ( - + )} ); @@ -1082,7 +1101,7 @@ export const Explorer = ({ { - renderTablesFlyout(dataSourceName); + renderTablesFlyout(dataSourceName, dataSourceConnectionType); }} > View databases and tables diff --git a/public/components/event_analytics/explorer/no_results.tsx b/public/components/event_analytics/explorer/no_results.tsx index e1c79517f1..e6aa050b0a 100644 --- a/public/components/event_analytics/explorer/no_results.tsx +++ b/public/components/event_analytics/explorer/no_results.tsx @@ -26,10 +26,15 @@ import { selectQueries } from '../redux/slices/query_slice'; import { selectSearchMetaData } from '../redux/slices/search_meta_data_slice'; import { DATA_SOURCE_TYPES, QUERY_LANGUAGE } from '../../../../common/constants/data_sources'; import { CatalogCacheManager } from '../../../framework/catalog_cache/cache_manager'; -import { CachedDataSourceStatus } from '../../../../common/types/data_connections'; +import { CachedDataSourceStatus, DatasourceType } from '../../../../common/types/data_connections'; import { getRenderLogExplorerTablesFlyout } from '../../../plugin'; -export const NoResults = ({ tabId }: any) => { +export interface NoResultsProps { + tabId: string; + dataSourceConnectionType: DatasourceType; +} + +export const NoResults = ({ tabId, dataSourceConnectionType }: NoResultsProps) => { // get the queries isLoaded, if it exists AND is true = show no res const queryInfo = useSelector(selectQueries)[tabId]; const summaryData = useSelector(selectQueryAssistantSummarization)[tabId]; @@ -99,7 +104,7 @@ export const NoResults = ({ tabId }: any) => { To start exploring this datasource, enter a query or{' '} { - renderTablesFlyout(datasourceName); + renderTablesFlyout(datasourceName, dataSourceConnectionType); }} > view databases and tables. @@ -127,7 +132,9 @@ export const NoResults = ({ tabId }: any) => {

Show a list of tables within a database

LIKE '*'`} + code={`SHOW ${ + dataSourceConnectionType === 'SECURITYLAKE' ? 'TABLES' : 'TABLE EXTENDED' + } IN ${datasourceName}. LIKE '*'`} /> diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration_inputs.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration_inputs.test.tsx.snap index aa6ac4ef59..2b74ba8818 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration_inputs.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration_inputs.test.tsx.snap @@ -203,37 +203,17 @@ exports[`Integration Setup Inputs Renders the S3 connector form as expected 1`] @@ -443,37 +423,17 @@ exports[`Integration Setup Inputs Renders the S3 connector form without workflow @@ -2208,7 +2168,7 @@ exports[`Integration Setup Inputs Renders the workflows inputs 1`] = ` fullWidth={false} hasChildLabel={true} hasEmptyLabelSpace={false} - isInvalid={false} + isInvalid={true} labelType="label" >
+ - - <_EuiSplitPanelOuter - className="euiCheckableCard euiCheckableCard-isChecked" - direction="row" - hasBorder={true} - responsive={false} - > - -
- <_EuiSplitPanelInner - color="primary" - grow={false} - onClick={[Function]} - > - -
- -
- -
-
- -
- - - <_EuiSplitPanelInner> - -
- -
- This is a test workflow. -
-
- - -
- - - - -
- - + Must select at least one workflow. +
+
diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 209e9656ed..490759146f 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -25,10 +25,11 @@ import { addIntegrationRequest } from './create_integration_helpers'; import { SetupIntegrationFormInputs } from './setup_integration_inputs'; import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { SetupIntegrationInputsForSecurityLake } from './setup_integration_inputs_security_lake'; +import { IntegrationConnectionType } from '../../../../common/types/integrations'; export interface IntegrationSetupInputs { displayName: string; - connectionType: string; + connectionType: IntegrationConnectionType; connectionDataSource: string; connectionLocation: string; checkpointLocation: string; @@ -335,10 +336,7 @@ export function SetupIntegrationForm({ unsetIntegration?: () => void; forceConnection?: { name: string; - type: string; - properties?: { - lakeFormationEnabled?: boolean; - }; + type: IntegrationConnectionType; }; setIsInstalling?: (isInstalling: boolean, success?: boolean) => void; }) { @@ -376,9 +374,10 @@ export function SetupIntegrationForm({ const updateConfig = (updates: Partial) => setConfig(Object.assign({}, integConfig, updates)); - const IntegrationInputFormComponent = forceConnection?.properties?.lakeFormationEnabled - ? SetupIntegrationInputsForSecurityLake - : SetupIntegrationFormInputs; + const IntegrationInputFormComponent = + forceConnection?.type === 'securityLake' || integConfig.connectionType === 'securityLake' + ? SetupIntegrationInputsForSecurityLake + : SetupIntegrationFormInputs; const content = ( <> diff --git a/public/components/integrations/components/setup_integration_inputs.tsx b/public/components/integrations/components/setup_integration_inputs.tsx index 1a43843398..8978b825c8 100644 --- a/public/components/integrations/components/setup_integration_inputs.tsx +++ b/public/components/integrations/components/setup_integration_inputs.tsx @@ -19,10 +19,11 @@ import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; import { CONSOLE_PROXY, DATACONNECTIONS_BASE } from '../../../../common/constants/shared'; import { IntegrationConfigProps, IntegrationSetupInputs } from './setup_integration'; +import { IntegrationConnectionType } from '../../../../common/types/integrations'; // TODO support localization const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< - string, + IntegrationConnectionType, { title: string; lower: string; @@ -37,6 +38,14 @@ const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< help: 'Select a data source to pull the data from.', }, ], + [ + 'securityLake', + { + title: 'Data Source', + lower: 'data_source', + help: 'Select a data source to pull the data from.', + }, + ], [ 'index', { @@ -47,7 +56,10 @@ const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< ], ]); -const integrationConnectionSelectorItems = [ +const integrationConnectionSelectorItems: Array<{ + value: 's3' | 'index' | 'securityLake'; + text: string; +}> = [ { value: 's3', text: 'S3 Connection', @@ -56,9 +68,15 @@ const integrationConnectionSelectorItems = [ value: 'index', text: 'OpenSearch Index', }, + { + value: 'securityLake', + text: 'Security Lake Connection', + }, ]; -const suggestDataSources = async (type: string): Promise> => { +const suggestDataSources = async ( + type: IntegrationConnectionType +): Promise> => { const http = coreRefs.http!; try { if (type === 'index') { @@ -74,17 +92,20 @@ const suggestDataSources = async (type: string): Promise; + const filterCondition = + type === 's3' + ? (item: any) => item.connector === 'S3GLUE' + : (item: any) => item.connector === 'SECURITYLAKE'; + return ( - result - ?.filter((item) => item.connector === 'S3GLUE') - .map((item) => { - return { label: item.name }; - }) ?? [] + result?.filter(filterCondition).map((item) => { + return { label: item.name }; + }) ?? [] ); } else { console.error(`Unknown connection type: ${type}`); @@ -97,19 +118,19 @@ const suggestDataSources = async (type: string): Promise; toggleWorkflow: (name: string) => void; }) { - if (!integration.workflows) { + if (!integrationWorkflows) { return null; } - const cards = integration.workflows.map((workflow) => { + const cards = integrationWorkflows.map((workflow) => { return ( <> { - if (item.value === 's3') { + if (item.value === 'securityLake') { + return ( + integration.assets.some((asset) => asset.type === 'query') && + integration.workflows?.some((workflow) => workflow.name.includes('security-lake')) + ); + } else if (item.value === 's3') { return integration.assets.some((asset) => asset.type === 'query'); } else if (item.value === 'index') { return integration.assets.some((asset) => asset.type === 'savedObjectBundle'); @@ -204,7 +230,10 @@ export function IntegrationConnectionInputs({ })} value={config.connectionType} onChange={(event) => - updateConfig({ connectionType: event.target.value, connectionDataSource: '' }) + updateConfig({ + connectionType: event.target.value as IntegrationConnectionType, + connectionDataSource: '', + }) } disabled={lockConnectionType} /> @@ -244,19 +273,17 @@ export function IntegrationQueryInputs({ config, updateConfig, integration, - isS3ConnectionWithLakeFormation, }: { config: IntegrationSetupInputs; updateConfig: (updates: Partial) => void; integration: IntegrationConfig; - isS3ConnectionWithLakeFormation?: boolean; }) { const [isBucketBlurred, setIsBucketBlurred] = useState(false); const [isCheckpointBlurred, setIsCheckpointBlurred] = useState(false); return ( <> - {!isS3ConnectionWithLakeFormation && ( + {config.connectionType !== 'securityLake' && ( <> ) => void; - integration: IntegrationConfig; + workflows?: IntegrationWorkflow[]; }) { const [useWorkflows, setUseWorkflows] = useState(new Map()); const toggleWorkflow = (name: string) => { @@ -332,10 +359,10 @@ export function IntegrationWorkflowsInputs({ }; useEffect(() => { - if (integration.workflows) { - setUseWorkflows(new Map(integration.workflows.map((w) => [w.name, w.enabled_by_default]))); + if (workflows) { + setUseWorkflows(new Map(workflows.map((w) => [w.name, w.enabled_by_default]))); } - }, [integration.workflows]); + }, [workflows]); useEffect(() => { updateConfig({ @@ -351,7 +378,7 @@ export function IntegrationWorkflowsInputs({ error={['Must select at least one workflow.']} > @@ -430,7 +457,10 @@ export function SetupIntegrationFormInputs(props: IntegrationConfigProps) { - + ) : null} {/* Bottom bar will overlap content if there isn't some space at the end */} diff --git a/public/components/integrations/components/setup_integration_inputs_security_lake.tsx b/public/components/integrations/components/setup_integration_inputs_security_lake.tsx index 122bdf41aa..b1eb3a8380 100644 --- a/public/components/integrations/components/setup_integration_inputs_security_lake.tsx +++ b/public/components/integrations/components/setup_integration_inputs_security_lake.tsx @@ -19,6 +19,7 @@ import { } from '../../datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector'; import { IntegrationConfigProps } from './setup_integration'; import { + IntegrationConnectionInputs, IntegrationDetailsInputs, IntegrationQueryInputs, IntegrationWorkflowsInputs, @@ -29,6 +30,7 @@ export const SetupIntegrationInputsForSecurityLake = ({ updateConfig, integration, setupCallout, + lockConnectionType, }: IntegrationConfigProps) => { const http = coreRefs.http!; const [dataSourceFormData, setDataSourceFormData] = useState({ @@ -38,6 +40,10 @@ export const SetupIntegrationInputsForSecurityLake = ({ formErrors: {}, }); + const [securityLakeWorkflows, setSecurityLakeWorkflows] = useState< + IntegrationWorkflow[] | undefined + >(undefined); + useEffect(() => { updateConfig({ connectionDatabaseName: dataSourceFormData.database, @@ -45,6 +51,20 @@ export const SetupIntegrationInputsForSecurityLake = ({ }); }, [dataSourceFormData]); + useEffect(() => { + setDataSourceFormData({ + ...dataSourceFormData, + dataSource: config.connectionDataSource, + }); + }, [config.connectionDataSource]); + + useEffect(() => { + // TODO: Refactor the filter condition to use `applicable_data_sources` #1855 + setSecurityLakeWorkflows( + integration.workflows?.filter((workflow) => workflow.name.includes('security-lake')) + ); + }, integration.workflows); + return ( <> @@ -65,58 +85,70 @@ export const SetupIntegrationInputsForSecurityLake = ({ integration={integration} /> - {config.connectionType === 's3' ? ( - <> - -

Integration data location

- - - - + +

Integration data location

+ - - - + + - {integration.workflows ? ( - <> - - -

Included resources

- - - -

- This integration offers resources compatible with your data source. These can - include dashboards, visualizations, indexes, and queries. Select at least one of - the following options. -

- - - - - - ) : null} - {/* Bottom bar will overlap content if there isn't some space at the end */} + + )} + + + + + + + + {securityLakeWorkflows && ( + <> + +

Included resources

+ + + +

+ This integration offers resources compatible with your data source. These can + include dashboards, visualizations, indexes, and queries. Select at least one of the + following options. +

+ + + - ) : null} + )} + {/* Bottom bar will overlap content if there isn't some space at the end */} + + ); }; diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index 294346c441..341b6ae376 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -11,10 +11,11 @@ import { } from '../../../common/constants/data_sources'; import { AsyncPollingResult, - CachedAccelerations, + CachedAcceleration, CachedColumn, CachedDataSourceStatus, CachedTable, + DatasourceType, LoadCacheType, StartLoadingParams, } from '../../../common/types/data_connections'; @@ -146,7 +147,7 @@ export const updateAccelerationsToCache = ( const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newAccelerations: CachedAccelerations[] = combinedData.map((row: any) => ({ + const newAccelerations: CachedAcceleration[] = combinedData.map((row: any) => ({ flintIndexName: row.flint_index_name, type: row.kind === 'mv' ? 'materialized' : row.kind, database: row.database, @@ -255,7 +256,8 @@ export const createLoadQuery = ( loadCacheType: LoadCacheType, dataSourceName: string, databaseName?: string, - tableName?: string + tableName?: string, + dataSourceType?: DatasourceType ) => { let query; switch (loadCacheType) { @@ -263,7 +265,9 @@ export const createLoadQuery = ( query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; break; case 'tables': - query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( + const showTableQueryBase = + dataSourceType?.toLowerCase() === 'securitylake' ? 'SHOW TABLES' : 'SHOW TABLE EXTENDED'; + query = `${showTableQueryBase} IN ${addBackticksIfNeeded( dataSourceName )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; break; @@ -316,6 +320,7 @@ export const useLoadToCache = (loadCacheType: LoadCacheType) => { const startLoading = ({ dataSourceName, + dataSourceType, dataSourceMDSId, databaseName, tableName, @@ -328,7 +333,13 @@ export const useLoadToCache = (loadCacheType: LoadCacheType) => { let requestPayload: DirectQueryRequest = { lang: 'sql', - query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), + query: createLoadQuery( + loadCacheType, + dataSourceName, + databaseName, + tableName, + dataSourceType + ), datasource: dataSourceName, }; diff --git a/public/plugin.tsx b/public/plugin.tsx index 7941d387a7..92ecd76871 100644 --- a/public/plugin.tsx +++ b/public/plugin.tsx @@ -57,6 +57,7 @@ import { } from '../common/constants/shared'; import { QueryManager } from '../common/query_manager'; import { + DatasourceType, RenderAccelerationDetailsFlyoutParams, RenderAccelerationFlyoutParams, RenderAssociatedObjectsDetailsFlyoutParams, @@ -147,6 +148,7 @@ export const [ ] = createGetterSetter< ({ dataSource, + dataSourceType, dataSourceMDSId, databaseName, tableName, @@ -157,7 +159,9 @@ export const [ export const [ getRenderLogExplorerTablesFlyout, setRenderLogExplorerTablesFlyout, -] = createGetterSetter<(dataSourceName: string) => void>('renderLogExplorerTablesFlyout'); +] = createGetterSetter<(dataSourceName: string, dataSourceType: DatasourceType) => void>( + 'renderLogExplorerTablesFlyout' +); export class ObservabilityPlugin implements @@ -508,7 +512,7 @@ export class ObservabilityPlugin dataSourceName, handleRefresh, dataSourceMDSId, - isS3ConnectionWithLakeFormation, + dataSourceType, }: RenderAssociatedObjectsDetailsFlyoutParams) => { const associatedObjectsDetailsFlyout = core.overlays.openFlyout( toMountPoint( @@ -518,7 +522,7 @@ export class ObservabilityPlugin resetFlyout={() => associatedObjectsDetailsFlyout.close()} handleRefresh={handleRefresh} dataSourceMDSId={dataSourceMDSId} - isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} + dataSourceType={dataSourceType} /> ) ); @@ -527,6 +531,7 @@ export class ObservabilityPlugin const renderCreateAccelerationFlyout = ({ dataSource, + dataSourceType, databaseName, tableName, handleRefresh, @@ -536,6 +541,7 @@ export class ObservabilityPlugin toMountPoint( createAccelerationFlyout.close()} databaseName={databaseName} tableName={tableName} @@ -547,11 +553,15 @@ export class ObservabilityPlugin }; setRenderCreateAccelerationFlyout(renderCreateAccelerationFlyout); - const renderLogExplorerTablesFlyout = (dataSourceName: string) => { + const renderLogExplorerTablesFlyout = ( + dataSourceName: string, + dataSourceType: DatasourceType + ) => { const createLogExplorerTablesFlyout = core.overlays.openFlyout( toMountPoint( createLogExplorerTablesFlyout.close()} /> )