Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[discover] S3 data connection #7917

Merged
merged 5 commits into from
Aug 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changelogs/fragments/7917.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
feat:
- Add S3 data exploration for connections, databases, and tables ([#7917](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7917))
7 changes: 6 additions & 1 deletion src/plugins/data/common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,15 @@ export const DEFAULT_DATA = {
type: 'ROOT',
meta: {
type: DATA_STRUCTURE_META_TYPES.FEATURE,
icon: 'folderOpen',
icon: { type: 'folderOpen' },
tooltip: 'Root Data Structure',
},
} as DataStructure,
LOCAL_DATASOURCE: {
id: '',
title: 'Local Cluster',
type: 'DATA_SOURCE',
},
},

SET_TYPES: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

import { CoreStart, SavedObjectsClientContract } from 'opensearch-dashboards/public';
import { CoreStart } from 'opensearch-dashboards/public';
import {
Dataset,
DataStructure,
Expand All @@ -15,6 +15,7 @@
import { DatasetTypeConfig } from './types';
import { indexPatternTypeConfig, indexTypeConfig } from './lib';
import { IndexPatternsContract } from '../../../index_patterns';
import { IDataPluginServices } from '../../../types';

export class DatasetService {
private indexPatterns?: IndexPatternsContract;
Expand Down Expand Up @@ -83,15 +84,15 @@
}

public fetchOptions(
savedObjects: SavedObjectsClientContract,
services: IDataPluginServices,
path: DataStructure[],
dataType: string
): Promise<DataStructure> {
const type = this.typesRegistry.get(dataType);
if (!type) {
throw new Error(`No handler found for type: ${path[0]}`);
}
return type.fetch(savedObjects, path);
return type.fetch(services, path);

Check warning on line 95 in src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts

View check run for this annotation

Codecov / codecov/patch

src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts#L95

Added line #L95 was not covered by tests
}

private async fetchDefaultDataset(): Promise<Dataset | undefined> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@
} as Dataset;
},

fetch: async (savedObjects, path) => {
fetch: async (services, path) => {
const dataStructure = path[path.length - 1];
const indexPatterns = await fetchIndexPatterns(savedObjects);
const indexPatterns = await fetchIndexPatterns(services.savedObjects.client);

Check warning on line 49 in src/plugins/data/public/query/query_string/dataset_service/lib/index_pattern_type.ts

View check run for this annotation

Codecov / codecov/patch

src/plugins/data/public/query/query_string/dataset_service/lib/index_pattern_type.ts#L49

Added line #L49 was not covered by tests
return {
...dataStructure,
columnHeader: 'Index patterns',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,6 @@
import { getSearchService, getIndexPatterns } from '../../../../services';
import { injectMetaToDataStructures } from './utils';

const INDEX_INFO = {
LOCAL_DATASOURCE: {
id: '',
title: 'Local Cluster',
type: 'DATA_SOURCE',
},
};

export const indexTypeConfig: DatasetTypeConfig = {
id: DEFAULT_DATA.SET_TYPES.INDEX,
title: 'Indexes',
Expand All @@ -47,11 +39,11 @@
title: dataSource.title,
type: dataSource.type,
}
: INDEX_INFO.LOCAL_DATASOURCE,
: DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE,
};
},

fetch: async (savedObjects, path) => {
fetch: async (services, path) => {
const dataStructure = path[path.length - 1];
switch (dataStructure.type) {
case 'DATA_SOURCE': {
Expand All @@ -69,7 +61,7 @@
}

default: {
const dataSources = await fetchDataSources(savedObjects);
const dataSources = await fetchDataSources(services.savedObjects.client);

Check warning on line 64 in src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts

View check run for this annotation

Codecov / codecov/patch

src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts#L64

Added line #L64 was not covered by tests
return {
...dataStructure,
columnHeader: 'Cluster',
Expand Down Expand Up @@ -97,12 +89,12 @@
};

const fetchDataSources = async (client: SavedObjectsClientContract) => {
const resp = await client.find<any>({
const response = await client.find<any>({

Check warning on line 92 in src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts

View check run for this annotation

Codecov / codecov/patch

src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts#L92

Added line #L92 was not covered by tests
type: 'data-source',
perPage: 10000,
});
const dataSources: DataStructure[] = [INDEX_INFO.LOCAL_DATASOURCE].concat(
resp.savedObjects.map((savedObject) => ({
const dataSources: DataStructure[] = [DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE].concat(
response.savedObjects.map((savedObject) => ({

Check warning on line 97 in src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts

View check run for this annotation

Codecov / codecov/patch

src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts#L96-L97

Added lines #L96 - L97 were not covered by tests
id: savedObject.id,
title: savedObject.attributes.title,
type: 'DATA_SOURCE',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
import { EuiIconProps } from '@elastic/eui';
import { Dataset, DatasetField, DataStructure } from '../../../../common';
import { IDataPluginServices } from '../../../types';

/**
* Configuration for handling dataset operations.
Expand All @@ -29,11 +29,11 @@ export interface DatasetTypeConfig {
toDataset: (path: DataStructure[]) => Dataset;
/**
* Fetches child options for a given DataStructure.
* @param {SavedObjectsClientContract} client - The saved objects client.
* @param {IDataPluginServices} services - The data plugin services.
* @param {DataStructure} dataStructure - The parent DataStructure.
* @returns {Promise<DatasetHandlerFetchResponse>} A promise that resolves to a DatasetHandlerFetchResponse.
*/
fetch: (client: SavedObjectsClientContract, path: DataStructure[]) => Promise<DataStructure>;
fetch: (services: IDataPluginServices, path: DataStructure[]) => Promise<DataStructure>;
/**
* Fetches fields for the dataset.
* @returns {Promise<DatasetField[]>} A promise that resolves to an array of DatasetFields.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
*/

import React, { useState } from 'react';
import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
import {
BaseDataset,
DATA_STRUCTURE_META_TYPES,
Expand All @@ -15,13 +14,14 @@ import {
import { DatasetExplorer } from './dataset_explorer';
import { Configurator } from './configurator';
import { getQueryService } from '../../services';
import { IDataPluginServices } from '../../types';

export const AdvancedSelector = ({
savedObjects,
services,
onSelect,
onCancel,
}: {
savedObjects: SavedObjectsClientContract;
services: IDataPluginServices;
onSelect: (dataset: Dataset) => void;
onCancel: () => void;
}) => {
Expand Down Expand Up @@ -59,7 +59,7 @@ export const AdvancedSelector = ({
/>
) : (
<DatasetExplorer
savedObjects={savedObjects}
services={services}
queryString={queryString}
path={path}
setPath={setPath}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ import {
EuiToolTip,
} from '@elastic/eui';
import { FormattedMessage } from '@osd/i18n/react';
import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
import { BaseDataset, DATA_STRUCTURE_META_TYPES, DataStructure } from '../../../common';
import { QueryStringContract } from '../../query';
import { IDataPluginServices } from '../../types';

export const DatasetExplorer = ({
savedObjects,
services,
queryString,
path,
setPath,
onNext,
onCancel,
}: {
savedObjects: SavedObjectsClientContract;
services: IDataPluginServices;
queryString: QueryStringContract;
path: DataStructure[];
setPath: (path: DataStructure[]) => void;
Expand All @@ -55,7 +55,7 @@ export const DatasetExplorer = ({
}

setLoading(true);
const nextDataStructure = await typeConfig.fetch(savedObjects, nextPath);
const nextDataStructure = await typeConfig.fetch(services, nextPath);
setLoading(false);

setPath([...newPath, nextDataStructure]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export const DatasetSelector = ({
}: DatasetSelectorProps) => {
const [isOpen, setIsOpen] = useState(false);
const [datasets, setDatasets] = useState<Dataset[]>([]);
const { overlays, savedObjects } = services;
const { overlays } = services;

const isMounted = useRef(true);

Expand All @@ -53,7 +53,7 @@ export const DatasetSelector = ({
const typeConfig = datasetService.getType(DEFAULT_DATA.SET_TYPES.INDEX_PATTERN);
if (!typeConfig) return;

const fetchedIndexPatternDataStructures = await typeConfig.fetch(savedObjects.client, []);
const fetchedIndexPatternDataStructures = await typeConfig.fetch(services, []);

if (!isMounted.current) return;

Expand All @@ -67,7 +67,7 @@ export const DatasetSelector = ({
if (!selectedDataset && fetchedDatasets.length > 0) {
setSelectedDataset(fetchedDatasets[0]);
}
}, [datasetService, savedObjects.client, selectedDataset, setSelectedDataset]);
}, [datasetService, selectedDataset, services, setSelectedDataset]);

useEffect(() => {
fetchDatasets();
Expand Down Expand Up @@ -183,7 +183,7 @@ export const DatasetSelector = ({
const overlay = overlays?.openModal(
toMountPoint(
<AdvancedSelector
savedObjects={savedObjects.client}
services={services}
onSelect={(dataset?: Dataset) => {
overlay?.close();
if (dataset) {
Expand Down
Loading
Loading