Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MD]Allow create and distinguish index pattern with same name but from different datasources #3604

Merged
Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -67,6 +67,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- [Multiple DataSource] Refactor test connection to support SigV4 auth type ([#3456](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3456))
- [Darwin] Add support for Darwin for running OpenSearch snapshots with `yarn opensearch snapshot` ([#3537](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3537))
- [Vis Builder] Add metric to metric, bucket to bucket aggregation persistence ([#3495](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3495))
- [Multiple DataSource] Prepend data source name to index pattern title in UI to distinguish across data sources ([#3571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3571))
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved

### 🐛 Bug Fixes

1 change: 1 addition & 0 deletions src/plugins/data/common/index_patterns/index.ts
Original file line number Diff line number Diff line change
@@ -33,3 +33,4 @@ export * from './types';
export { IndexPatternsService } from './index_patterns';
export type { IndexPattern } from './index_patterns';
export * from './errors';
export { validateDataSourceReference, getIndexPatternTitle } from './utils';
Original file line number Diff line number Diff line change
@@ -366,7 +366,7 @@ export class IndexPattern implements IIndexPattern {
};
}

getSaveObjectReference() {
getSaveObjectReference = () => {
return this.dataSourceRef
? [
{
@@ -376,7 +376,7 @@ export class IndexPattern implements IIndexPattern {
},
]
: [];
}
};

/**
* Provide a field, get its formatter
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@
*/

import { i18n } from '@osd/i18n';
import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
import { SavedObjectsClientCommon } from '../..';
import { createIndexPatternCache } from '.';
import { IndexPattern } from './index_pattern';
@@ -53,7 +54,7 @@ import { FieldFormatsStartCommon } from '../../field_formats';
import { UI_SETTINGS, SavedObject } from '../../../common';
import { SavedObjectNotFound } from '../../../../opensearch_dashboards_utils/common';
import { IndexPatternMissingIndices } from '../lib';
import { findByTitle } from '../utils';
import { findByTitle, getIndexPatternTitle } from '../utils';
import { DuplicateIndexPatternError } from '../errors';

const indexPatternCache = createIndexPatternCache();
@@ -118,8 +119,28 @@ export class IndexPatternsService {
fields: ['title'],
perPage: 10000,
});

this.savedObjectsCache = await Promise.all(
this.savedObjectsCache.map(async (obj) => {
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
if (obj.type === 'index-pattern') {
const result = { ...obj };
result.attributes.title = await getIndexPatternTitle(
obj.attributes.title,
obj.references,
this.getDataSource
);
return result;
} else {
return obj;
}
})
);
}

getDataSource = async (id: string) => {
return await this.savedObjectsClient.get<DataSourceAttributes>('data-source', id);
};

/**
* Get list of index pattern ids
* @param refresh Force refresh of index pattern list
@@ -557,7 +578,11 @@ export class IndexPatternsService {
*/

async createSavedObject(indexPattern: IndexPattern, override = false) {
const dupe = await findByTitle(this.savedObjectsClient, indexPattern.title);
const dupe = await findByTitle(
this.savedObjectsClient,
indexPattern.title,
indexPattern.dataSourceRef?.id
);
if (dupe) {
if (override) {
await this.delete(dupe.id);
61 changes: 53 additions & 8 deletions src/plugins/data/common/index_patterns/utils.ts
Original file line number Diff line number Diff line change
@@ -28,26 +28,71 @@
* under the License.
*/

import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
import type { IndexPatternSavedObjectAttrs } from './index_patterns';
import type { SavedObjectsClientCommon } from '../types';
import type { SavedObject, SavedObjectReference, SavedObjectsClientCommon } from '../types';

/**
* Returns an object matching a given title
*
* @param client {SavedObjectsClientCommon}
* @param title {string}
* @param dataSourceId {string}{optional}
* @returns {Promise<SavedObject|undefined>}
*/
export async function findByTitle(client: SavedObjectsClientCommon, title: string) {
export async function findByTitle(
client: SavedObjectsClientCommon,
title: string,
dataSourceId?: string
) {
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
if (title) {
const savedObjects = await client.find<IndexPatternSavedObjectAttrs>({
type: 'index-pattern',
perPage: 10,
search: `"${title}"`,
searchFields: ['title'],
fields: ['title'],
const savedObjects = (
await client.find<IndexPatternSavedObjectAttrs>({
type: 'index-pattern',
perPage: 10,
search: `"${title}"`,
searchFields: ['title'],
fields: ['title'],
})
).filter((obj) => {
return obj && obj.attributes && validateDataSourceReference(obj, dataSourceId);
});

return savedObjects.find((obj) => obj.attributes.title.toLowerCase() === title.toLowerCase());
}
}

// This is used to validate datasource reference of index pattern
export const validateDataSourceReference = (
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
indexPattern: SavedObject<any>,
dataSourceId?: string
) => {
const references = indexPattern.references;
if (dataSourceId) {
return (
references[0] && references[0].id === dataSourceId && references[0].type === 'data-source'
);
} else {
// No datasource id passed as input meaning we are getting index pattern from default cluster,
// and it's supposed to be an empty array
return references.length === 0;
}
};

export const getIndexPatternTitle = async (
indexPatternTitle: string,
references: SavedObjectReference[],
getDataSource: (id: string) => Promise<SavedObject<DataSourceAttributes>>
): Promise<string> => {
const DELIMITER = '.';
// If an index-pattern references datasource, prepend data source name with index pattern name for display purpose
if (Array.isArray(references) && references[0] && references[0].type === 'data-source') {
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
const {
attributes: { title: dataSourceTitle },
} = await getDataSource(references[0].id);
return dataSourceTitle.concat(DELIMITER).concat(indexPatternTitle);
} else {
// if index pattern doesn't reference datasource, return as it is.
return indexPatternTitle;
}
};
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
2 changes: 1 addition & 1 deletion src/plugins/data/opensearch_dashboards.json
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
"ui": true,
"requiredPlugins": ["expressions", "uiActions"],
"optionalPlugins": ["usageCollection", "dataSource"],
"extraPublicDirs": ["common", "common/utils/abort_utils"],
"extraPublicDirs": ["common", "common/utils/abort_utils", "common/index_patterns/utils.ts"],
"requiredBundles": [
"usageCollection",
"opensearchDashboardsUtils",
25 changes: 13 additions & 12 deletions src/plugins/data_source/server/routes/test_connection.ts
Original file line number Diff line number Diff line change
@@ -30,18 +30,19 @@ export const registerTestConnectionRoute = (
schema.literal(AuthType.NoAuth),
schema.literal(AuthType.SigV4),
]),
credentials: schema.oneOf([
schema.object({
username: schema.string(),
password: schema.string(),
}),
schema.object({
region: schema.string(),
accessKey: schema.string(),
secretKey: schema.string(),
}),
schema.literal(null),
]),
credentials: schema.maybe(
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
schema.oneOf([
schema.object({
username: schema.string(),
password: schema.string(),
}),
schema.object({
region: schema.string(),
accessKey: schema.string(),
secretKey: schema.string(),
}),
])
),
})
),
}),
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
*/

import { HttpStart, SavedObjectsClientContract } from 'src/core/public';
import { AuthType, DataSourceAttributes, DataSourceTableItem } from '../types';
import { DataSourceAttributes, DataSourceTableItem } from '../types';

export async function getDataSources(savedObjectsClient: SavedObjectsClientContract) {
return savedObjectsClient
Original file line number Diff line number Diff line change
@@ -70,8 +70,10 @@ export function DiscoverIndexPattern({
});
useEffect(() => {
const { id, title } = selectedIndexPattern;
setSelected({ id, title });
}, [selectedIndexPattern]);
const indexPattern = indexPatternList.find((pattern) => pattern.id === id);
zhongnansu marked this conversation as resolved.
Show resolved Hide resolved
const titleToDisplay = indexPattern ? indexPattern.attributes!.title : title;
setSelected({ id, title: titleToDisplay });
}, [indexPatternList, selectedIndexPattern]);
if (!selectedId) {
return null;
}
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ import { context as contextType } from '../../../../../../opensearch_dashboards_
import { IndexPatternCreationConfig } from '../../../../../../../plugins/index_pattern_management/public';
import { MatchedItem, StepInfo } from '../../types';
import { DataSourceRef, IndexPatternManagmentContextValue } from '../../../../types';
import { validateDataSourceReference } from '../../../../../../../plugins/data/common';

interface StepIndexPatternProps {
allIndices: MatchedItem[];
@@ -131,7 +132,7 @@ export class StepIndexPattern extends Component<StepIndexPatternProps, StepIndex

ILLEGAL_CHARACTERS = [...indexPatterns.ILLEGAL_CHARACTERS];

dataSrouceEnabled: boolean;
dataSourceEnabled: boolean;

constructor(props: StepIndexPatternProps, context: IndexPatternManagmentContextValue) {
super(props, context);
@@ -140,7 +141,7 @@ export class StepIndexPattern extends Component<StepIndexPatternProps, StepIndex
this.state.query =
initialQuery || context.services.uiSettings.get(UI_SETTINGS.INDEXPATTERN_PLACEHOLDER);
this.state.indexPatternName = indexPatternCreationType.getIndexPatternName();
this.dataSrouceEnabled = context.services.dataSourceEnabled;
this.dataSourceEnabled = context.services.dataSourceEnabled;
}

lastQuery = '';
@@ -163,7 +164,9 @@ export class StepIndexPattern extends Component<StepIndexPatternProps, StepIndex
});

const existingIndexPatterns = savedObjects.map((obj) =>
obj && obj.attributes ? obj.attributes.title : ''
obj && obj.attributes && validateDataSourceReference(obj, this.props.dataSourceRef?.id)
? obj.attributes.title
: ''
) as string[];

this.setState({ existingIndexPatterns });
@@ -461,7 +464,7 @@ export class StepIndexPattern extends Component<StepIndexPatternProps, StepIndex
{this.renderStatusMessage(matchedIndices)}
<EuiSpacer />
{this.renderList(matchedIndices)}
{this.dataSrouceEnabled && this.renderGoToPrevious()}
{this.dataSourceEnabled && this.renderGoToPrevious()}
</>
);
}
Original file line number Diff line number Diff line change
@@ -60,6 +60,8 @@ import {
SavedObjectsStart,
} from 'src/core/public';

import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
import { getIndexPatternTitle } from '../../../data/common/index_patterns/utils';
import { LISTING_LIMIT_SETTING } from '../../common';

export interface SavedObjectMetaData<T = unknown> {
@@ -155,7 +157,28 @@ class SavedObjectFinderUi extends React.Component<
defaultSearchOperator: 'AND',
});

resp.savedObjects = resp.savedObjects.filter((savedObject) => {
const getDataSource = async (id: string) => {
const client = this.props.savedObjects.client;
return await client.get<DataSourceAttributes>('data-source', id);
};

const savedObjects = await Promise.all(
resp.savedObjects.map(async (obj) => {
if (obj.type === 'index-pattern') {
const result = { ...obj };
result.attributes.title = await getIndexPatternTitle(
obj.attributes.title!,
obj.references,
getDataSource
);
return result;
Comment on lines +167 to +174
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doing an explicit type check within the generalized saved_object plugin seems like an antipattern. Ideally, any logic unique to a particular saved object should exist in that saved object definition (index-pattern, in this case).

Luckily, there are better patterns available to do what we're doing here. One option is to provide a custom title getter method in the index-pattern saved object definition, if we always want the title to have the appended data source. Alternatively, if we sometimes need just the index pattern title, and other times need the appended version, we can just define a new custom property, such as "displayTitle" and have this logic in the getter for that.

As an example, see how the visualize plugin provides its own custom finder method as part of the saved object loader: https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/src/plugins/visualizations/public/saved_visualizations/saved_visualizations.ts

Copy link
Member Author

@zhongnansu zhongnansu Apr 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prepending datasource to title requires fetching datasource name first, which replies on a saved object client to make a call. This should happen where there's a client to use.

  • saved object management definition doesn't provide client , passing it to magement -> getTitlte() will change the interface that shard by other saved object. And to pass it for index-pattern type only, again need explicit check.
  • index-pattern saved object can't provide client, because unfortunately unlike savedVis that you referred to, there's no SavedIndexPattern extends SavedObjectClass exists. Don't think it's worth to create one only for prepending datasource to the title for display purpose. Comparing to visualization, index patterns are comparatively simpler objects

Besides, there's an issue under discussion about having an OUI component that can be used to select datasource then index pattern when creating Visualzaiton/Dashboards/Discover. #2841. The logic we have here may need be refactored after that discussion is settled.

Thanks for bringing this up, this might not be a good pattern to use, but it might not be a blocker for this PR. Can we create an issue to track and address in the future improvements? Does it sound good to you? @joshuarrrr

} else {
return obj;
}
})
);

resp.savedObjects = savedObjects.filter((savedObject) => {
const metaData = metaDataMap[savedObject.type];
if (metaData.showSavedObject) {
return metaData.showSavedObject(savedObject);
24 changes: 23 additions & 1 deletion src/plugins/saved_objects_management/server/routes/find.ts
Original file line number Diff line number Diff line change
@@ -30,6 +30,8 @@

import { schema } from '@osd/config-schema';
import { IRouter } from 'src/core/server';
import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
import { getIndexPatternTitle } from '../../../data/common/index_patterns/utils';
import { injectMetaAttributes } from '../lib';
import { ISavedObjectsManagement } from '../services';

@@ -84,13 +86,33 @@ export const registerFindRoute = (
}
});

const getDataSource = async (id: string) => {
return await client.get<DataSourceAttributes>('data-source', id);
};

const findResponse = await client.find<any>({
...req.query,
fields: undefined,
searchFields: [...searchFields],
});

const enhancedSavedObjects = findResponse.saved_objects
const savedObjects = await Promise.all(
findResponse.saved_objects.map(async (obj) => {
if (obj.type === 'index-pattern') {
const result = { ...obj };
result.attributes.title = await getIndexPatternTitle(
obj.attributes.title,
obj.references,
getDataSource
);
return result;
} else {
return obj;
}
})
);
Comment on lines +99 to +113
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar to my other comment above, this logic shouldn't be here, it should be accounted for in the saved object management definition: https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/src/plugins/data/server/saved_objects/index_patterns.ts#L42-L44


const enhancedSavedObjects = savedObjects
.map((so) => injectMetaAttributes(so, managementService))
.map((obj) => {
const result = { ...obj, attributes: {} as Record<string, any> };