Remove filter out serverless cluster and add support to extract index…

… name (opensearch-project#8872) * Remove filter out serverless cluster and add support to extract index name Allow extract index name for both serverless and non-serverless clusters Allow different key formats: - datasource-id::TIMESERIES::<index-name>:0 - datasource-id::<index-name>:0 - <index-name> (non-serverless case) Signed-off-by: Anan Zhuang <[email protected]> * fix PR comment Signed-off-by: Anan Zhuang <[email protected]> * Changeset file for PR opensearch-project#8872 created/updated --------- Signed-off-by: Anan Zhuang <[email protected]> Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
ohltyler · Nov 21, 2024 · 1cb2511 · 1cb2511
1 parent c928aec
commit 1cb2511
Show file tree

Hide file tree

Showing 5 changed files with 133 additions and 70 deletions.
diff --git a/changelogs/fragments/8872.yml b/changelogs/fragments/8872.yml
@@ -0,0 +1,2 @@
+fix:
+- Remove filter out serverless cluster and add support to extract index name ([#8872](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8872))
diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts
@@ -10,11 +10,18 @@ import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { DATA_STRUCTURE_META_TYPES, DataStructure, Dataset } from '../../../../../common';
 import * as services from '../../../../services';
 import { IDataPluginServices } from 'src/plugins/data/public';
+import { of } from 'rxjs';
 
 jest.mock('../../../../services', () => {
+  const mockSearchFunction = jest.fn();
+
   return {
-    getSearchService: jest.fn(),
     getIndexPatterns: jest.fn(),
+    getSearchService: jest.fn(() => ({
+      getDefaultSearchInterceptor: () => ({
+        search: mockSearchFunction,
+      }),
+    })),
     getQueryService: () => ({
       queryString: {
         getLanguageService: () => ({
@@ -90,9 +97,7 @@ describe('indexTypeConfig', () => {
 
   test('should fetch data sources for unknown type', async () => {
     mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-      savedObjects: [
-        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
-      ],
+      savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }],
     });
 
     const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
@@ -104,18 +109,18 @@ describe('indexTypeConfig', () => {
     expect(result.hasNext).toBe(true);
   });
 
-  test('should filter out data sources with versions lower than 1.0.0', async () => {
+  test('should NOT filter out data sources regardless of version', async () => {
     mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
       savedObjects: [
         { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
         {
           id: 'ds2',
-          attributes: { title: 'DataSource 2', dataSourceVersion: '' },
+          attributes: { title: 'DataSource 2', dataSourceVersion: '' }, // empty version
         },
         { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
         {
           id: 'ds4',
-          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
+          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, // invalid version
         },
       ],
     });
@@ -124,10 +129,64 @@ describe('indexTypeConfig', () => {
       { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
     ]);
 
-    expect(result.children).toHaveLength(2);
-    expect(result.children?.[0].title).toBe('DataSource 1');
-    expect(result.children?.[1].title).toBe('DataSource 3');
-    expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
+    // Verify all data sources are included regardless of version
+    expect(result.children).toHaveLength(4);
+    expect(result.children?.map((child) => child.title)).toEqual([
+      'DataSource 1',
+      'DataSource 2',
+      'DataSource 3',
+      'DataSource 4',
+    ]);
     expect(result.hasNext).toBe(true);
   });
+
+  describe('fetchIndices', () => {
+    test('should extract index names correctly from different formats', async () => {
+      const mockResponse = {
+        rawResponse: {
+          aggregations: {
+            indices: {
+              buckets: [
+                { key: '123::TIMESERIES::sample-index-1:0' },
+                // Serverless format without TIMESERIES
+                { key: '123::sample-index-2:0' },
+                // Non-serverless format
+                { key: 'simple-index' },
+              ],
+            },
+          },
+        },
+      };
+
+      const searchService = services.getSearchService();
+      const interceptor = searchService.getDefaultSearchInterceptor();
+      (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse));
+
+      const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+        { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' },
+      ]);
+
+      expect(result.children).toEqual([
+        { id: 'datasource1::sample-index-1', title: 'sample-index-1', type: 'INDEX' },
+        { id: 'datasource1::sample-index-2', title: 'sample-index-2', type: 'INDEX' },
+        { id: 'datasource1::simple-index', title: 'simple-index', type: 'INDEX' },
+      ]);
+    });
+
+    test('should handle response without aggregations', async () => {
+      const mockResponse = {
+        rawResponse: {},
+      };
+
+      const searchService = services.getSearchService();
+      const interceptor = searchService.getDefaultSearchInterceptor();
+      (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse));
+
+      const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+        { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' },
+      ]);
+
+      expect(result.children).toEqual([]);
+    });
+  });
 });
diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts
@@ -6,7 +6,6 @@
 import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { map } from 'rxjs/operators';
 import { i18n } from '@osd/i18n';
-import semver from 'semver';
 import {
   DEFAULT_DATA,
   DataStructure,
@@ -17,6 +16,8 @@ import { DatasetTypeConfig } from '../types';
 import { getSearchService, getIndexPatterns } from '../../../../services';
 import { injectMetaToDataStructures } from './utils';
 
+export const DELIMITER = '::';
+
 export const indexTypeConfig: DatasetTypeConfig = {
   id: DEFAULT_DATA.SET_TYPES.INDEX,
   title: 'Indexes',
@@ -120,16 +121,11 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => {
     type: 'data-source',
     perPage: 10000,
   });
-  const dataSources: DataStructure[] = response.savedObjects
-    .filter((savedObject) => {
-      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
-      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
-    })
-    .map((savedObject) => ({
-      id: savedObject.id,
-      title: savedObject.attributes.title,
-      type: 'DATA_SOURCE',
-    }));
+  const dataSources: DataStructure[] = response.savedObjects.map((savedObject) => ({
+    id: savedObject.id,
+    title: savedObject.attributes.title,
+    type: 'DATA_SOURCE',
+  }));
 
   return injectMetaToDataStructures(dataSources);
 };
@@ -158,9 +154,19 @@ const fetchIndices = async (dataStructure: DataStructure): Promise<string[]> =>
 
   const searchResponseToArray = (response: any) => {
     const { rawResponse } = response;
-    return rawResponse.aggregations
-      ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key)
-      : [];
+    if (!rawResponse.aggregations) {
+      return [];
+    }
+
+    return rawResponse.aggregations.indices.buckets.map((bucket: { key: string }) => {
+      const key = bucket.key;
+      // Note: Index names cannot contain ':' or '::' in OpenSearch, so these delimiters
+      // are guaranteed not to be part of the regular format of index name
+      const parts = key.split(DELIMITER);
+      const lastPart = parts[parts.length - 1] || key;
+      // extract index name or return original key if pattern doesn't match
+      return lastPart.split(':')[0] || key;
+    });
   };
 
   return search

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts
@@ -141,9 +141,7 @@ describe('s3TypeConfig', () => {
 
     it('should fetch data sources for unknown type', async () => {
       mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-        savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
-        ],
+        savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }],
       });
 
       const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
@@ -154,33 +152,37 @@ describe('s3TypeConfig', () => {
       expect(result.children?.[0].title).toBe('DataSource 1');
       expect(result.hasNext).toBe(true);
     });
+  });
 
-    it('should filter out data sources with versions lower than 1.0.0', async () => {
-      mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-        savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
-          {
-            id: 'ds2',
-            attributes: { title: 'DataSource 2', dataSourceVersion: '' },
-          },
-          { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
-          {
-            id: 'ds4',
-            attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
-          },
-        ],
-      });
-
-      const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
-        { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
-      ]);
-
-      expect(result.children).toHaveLength(2);
-      expect(result.children?.[0].title).toBe('DataSource 1');
-      expect(result.children?.[1].title).toBe('DataSource 3');
-      expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
-      expect(result.hasNext).toBe(true);
+  it('should NOT filter out data sources regardless of version', async () => {
+    mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
+      savedObjects: [
+        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
+        {
+          id: 'ds2',
+          attributes: { title: 'DataSource 2', dataSourceVersion: '' }, // empty version
+        },
+        { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
+        {
+          id: 'ds4',
+          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, // invalid version
+        },
+      ],
     });
+
+    const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
+      { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
+    ]);
+
+    // Verify all data sources are included
+    expect(result.children).toHaveLength(4);
+    expect(result.children?.map((child) => child.title)).toEqual([
+      'DataSource 1',
+      'DataSource 2',
+      'DataSource 3',
+      'DataSource 4',
+    ]);
+    expect(result.hasNext).toBe(true);
   });
 
   test('fetchFields returns table fields', async () => {

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts
@@ -6,7 +6,6 @@
 import { i18n } from '@osd/i18n';
 import { trimEnd } from 'lodash';
 import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public';
-import semver from 'semver';
 import {
   DATA_STRUCTURE_META_TYPES,
   DEFAULT_DATA,
@@ -198,22 +197,17 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise<Dat
     type: 'data-source',
     perPage: 10000,
   });
-  const dataSources: DataStructure[] = resp.savedObjects
-    .filter((savedObject) => {
-      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
-      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
-    })
-    .map((savedObject) => ({
-      id: savedObject.id,
-      title: savedObject.attributes.title,
-      type: 'DATA_SOURCE',
-      meta: {
-        query: {
-          id: savedObject.id,
-        },
-        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
-      } as DataStructureCustomMeta,
-    }));
+  const dataSources: DataStructure[] = resp.savedObjects.map((savedObject) => ({
+    id: savedObject.id,
+    title: savedObject.attributes.title,
+    type: 'DATA_SOURCE',
+    meta: {
+      query: {
+        id: savedObject.id,
+      },
+      type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+    } as DataStructureCustomMeta,
+  }));
   return dataSources;
 };
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		fix:
		- Remove filter out serverless cluster and add support to extract index name ([#8872](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8872))