From 1cb25111da711e1a40bb8bcf2e20bd38a9c31236 Mon Sep 17 00:00:00 2001 From: Anan Zhuang Date: Thu, 21 Nov 2024 12:00:01 -0800 Subject: [PATCH] Remove filter out serverless cluster and add support to extract index name (#8872) * Remove filter out serverless cluster and add support to extract index name Allow extract index name for both serverless and non-serverless clusters Allow different key formats: - datasource-id::TIMESERIES:::0 - datasource-id:::0 - (non-serverless case) Signed-off-by: Anan Zhuang * fix PR comment Signed-off-by: Anan Zhuang * Changeset file for PR #8872 created/updated --------- Signed-off-by: Anan Zhuang Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> --- changelogs/fragments/8872.yml | 2 + .../dataset_service/lib/index_type.test.ts | 81 ++++++++++++++++--- .../dataset_service/lib/index_type.ts | 34 ++++---- .../public/datasets/s3_type.test.ts | 58 ++++++------- .../public/datasets/s3_type.ts | 28 +++---- 5 files changed, 133 insertions(+), 70 deletions(-) create mode 100644 changelogs/fragments/8872.yml diff --git a/changelogs/fragments/8872.yml b/changelogs/fragments/8872.yml new file mode 100644 index 000000000000..1e43b2ae770d --- /dev/null +++ b/changelogs/fragments/8872.yml @@ -0,0 +1,2 @@ +fix: +- Remove filter out serverless cluster and add support to extract index name ([#8872](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8872)) \ No newline at end of file diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts index 7a3d9810fb7f..a645fd4acc7c 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts @@ -10,11 +10,18 @@ import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { DATA_STRUCTURE_META_TYPES, DataStructure, Dataset } from '../../../../../common'; import * as services from '../../../../services'; import { IDataPluginServices } from 'src/plugins/data/public'; +import { of } from 'rxjs'; jest.mock('../../../../services', () => { + const mockSearchFunction = jest.fn(); + return { - getSearchService: jest.fn(), getIndexPatterns: jest.fn(), + getSearchService: jest.fn(() => ({ + getDefaultSearchInterceptor: () => ({ + search: mockSearchFunction, + }), + })), getQueryService: () => ({ queryString: { getLanguageService: () => ({ @@ -90,9 +97,7 @@ describe('indexTypeConfig', () => { test('should fetch data sources for unknown type', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ - savedObjects: [ - { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } }, - ], + savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }], }); const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [ @@ -104,18 +109,18 @@ describe('indexTypeConfig', () => { expect(result.hasNext).toBe(true); }); - test('should filter out data sources with versions lower than 1.0.0', async () => { + test('should NOT filter out data sources regardless of version', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ savedObjects: [ { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } }, { id: 'ds2', - attributes: { title: 'DataSource 2', dataSourceVersion: '' }, + attributes: { title: 'DataSource 2', dataSourceVersion: '' }, // empty version }, { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } }, { id: 'ds4', - attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, + attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, // invalid version }, ], }); @@ -124,10 +129,64 @@ describe('indexTypeConfig', () => { { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, ]); - expect(result.children).toHaveLength(2); - expect(result.children?.[0].title).toBe('DataSource 1'); - expect(result.children?.[1].title).toBe('DataSource 3'); - expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false); + // Verify all data sources are included regardless of version + expect(result.children).toHaveLength(4); + expect(result.children?.map((child) => child.title)).toEqual([ + 'DataSource 1', + 'DataSource 2', + 'DataSource 3', + 'DataSource 4', + ]); expect(result.hasNext).toBe(true); }); + + describe('fetchIndices', () => { + test('should extract index names correctly from different formats', async () => { + const mockResponse = { + rawResponse: { + aggregations: { + indices: { + buckets: [ + { key: '123::TIMESERIES::sample-index-1:0' }, + // Serverless format without TIMESERIES + { key: '123::sample-index-2:0' }, + // Non-serverless format + { key: 'simple-index' }, + ], + }, + }, + }, + }; + + const searchService = services.getSearchService(); + const interceptor = searchService.getDefaultSearchInterceptor(); + (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse)); + + const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' }, + ]); + + expect(result.children).toEqual([ + { id: 'datasource1::sample-index-1', title: 'sample-index-1', type: 'INDEX' }, + { id: 'datasource1::sample-index-2', title: 'sample-index-2', type: 'INDEX' }, + { id: 'datasource1::simple-index', title: 'simple-index', type: 'INDEX' }, + ]); + }); + + test('should handle response without aggregations', async () => { + const mockResponse = { + rawResponse: {}, + }; + + const searchService = services.getSearchService(); + const interceptor = searchService.getDefaultSearchInterceptor(); + (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse)); + + const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' }, + ]); + + expect(result.children).toEqual([]); + }); + }); }); diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index 13fc4ce14f72..31b331e1433d 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -6,7 +6,6 @@ import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { map } from 'rxjs/operators'; import { i18n } from '@osd/i18n'; -import semver from 'semver'; import { DEFAULT_DATA, DataStructure, @@ -17,6 +16,8 @@ import { DatasetTypeConfig } from '../types'; import { getSearchService, getIndexPatterns } from '../../../../services'; import { injectMetaToDataStructures } from './utils'; +export const DELIMITER = '::'; + export const indexTypeConfig: DatasetTypeConfig = { id: DEFAULT_DATA.SET_TYPES.INDEX, title: 'Indexes', @@ -120,16 +121,11 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { type: 'data-source', perPage: 10000, }); - const dataSources: DataStructure[] = response.savedObjects - .filter((savedObject) => { - const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion); - return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false; - }) - .map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - type: 'DATA_SOURCE', - })); + const dataSources: DataStructure[] = response.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + type: 'DATA_SOURCE', + })); return injectMetaToDataStructures(dataSources); }; @@ -158,9 +154,19 @@ const fetchIndices = async (dataStructure: DataStructure): Promise => const searchResponseToArray = (response: any) => { const { rawResponse } = response; - return rawResponse.aggregations - ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) - : []; + if (!rawResponse.aggregations) { + return []; + } + + return rawResponse.aggregations.indices.buckets.map((bucket: { key: string }) => { + const key = bucket.key; + // Note: Index names cannot contain ':' or '::' in OpenSearch, so these delimiters + // are guaranteed not to be part of the regular format of index name + const parts = key.split(DELIMITER); + const lastPart = parts[parts.length - 1] || key; + // extract index name or return original key if pattern doesn't match + return lastPart.split(':')[0] || key; + }); }; return search diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index 6a2d5cc6182c..1a6066f72f14 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -141,9 +141,7 @@ describe('s3TypeConfig', () => { it('should fetch data sources for unknown type', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ - savedObjects: [ - { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } }, - ], + savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }], }); const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [ @@ -154,33 +152,37 @@ describe('s3TypeConfig', () => { expect(result.children?.[0].title).toBe('DataSource 1'); expect(result.hasNext).toBe(true); }); + }); - it('should filter out data sources with versions lower than 1.0.0', async () => { - mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ - savedObjects: [ - { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } }, - { - id: 'ds2', - attributes: { title: 'DataSource 2', dataSourceVersion: '' }, - }, - { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } }, - { - id: 'ds4', - attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, - }, - ], - }); - - const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [ - { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, - ]); - - expect(result.children).toHaveLength(2); - expect(result.children?.[0].title).toBe('DataSource 1'); - expect(result.children?.[1].title).toBe('DataSource 3'); - expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false); - expect(result.hasNext).toBe(true); + it('should NOT filter out data sources regardless of version', async () => { + mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ + savedObjects: [ + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } }, + { + id: 'ds2', + attributes: { title: 'DataSource 2', dataSourceVersion: '' }, // empty version + }, + { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } }, + { + id: 'ds4', + attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, // invalid version + }, + ], }); + + const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, + ]); + + // Verify all data sources are included + expect(result.children).toHaveLength(4); + expect(result.children?.map((child) => child.title)).toEqual([ + 'DataSource 1', + 'DataSource 2', + 'DataSource 3', + 'DataSource 4', + ]); + expect(result.hasNext).toBe(true); }); test('fetchFields returns table fields', async () => { diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index c13b5e898670..4e8c41959f2d 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -6,7 +6,6 @@ import { i18n } from '@osd/i18n'; import { trimEnd } from 'lodash'; import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import semver from 'semver'; import { DATA_STRUCTURE_META_TYPES, DEFAULT_DATA, @@ -198,22 +197,17 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise { - const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion); - return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false; - }) - .map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - type: 'DATA_SOURCE', - meta: { - query: { - id: savedObject.id, - }, - type: DATA_STRUCTURE_META_TYPES.CUSTOM, - } as DataStructureCustomMeta, - })); + const dataSources: DataStructure[] = resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + type: 'DATA_SOURCE', + meta: { + query: { + id: savedObject.id, + }, + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + } as DataStructureCustomMeta, + })); return dataSources; };