[index pattern management] Restore cross cluster search functionality (…

…#108756)

* restore cross cluster search functionality

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md

@@ -7,5 +7,5 @@
 <b>Signature:</b>
 
 ```typescript
-isErrorResponse: (response?: IKibanaSearchResponse<any> | undefined) => boolean | undefined
+isErrorResponse: (response?: IKibanaSearchResponse<any> | undefined) => boolean
 ```

src/plugins/data/common/search/utils.ts

@@ -12,7 +12,7 @@ import type { IKibanaSearchResponse } from './types';
  * @returns true if response had an error while executing in ES
  */
 export const isErrorResponse = (response?: IKibanaSearchResponse) => {
-  return !response || !response.rawResponse || (!response.isRunning && response.isPartial);
+  return !response || !response.rawResponse || (!response.isRunning && !!response.isPartial);
 };
 
 /**

src/plugins/data/public/public.api.md

@@ -1568,7 +1568,7 @@ export interface ISearchStartSearchSource {
 // Warning: (ae-missing-release-tag) "isErrorResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
-export const isErrorResponse: (response?: IKibanaSearchResponse<any> | undefined) => boolean | undefined;
+export const isErrorResponse: (response?: IKibanaSearchResponse<any> | undefined) => boolean;
 
 // Warning: (ae-missing-release-tag) "isEsError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //

src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx

@@ -36,7 +36,7 @@ export const EmptyPrompts: FC<Props> = ({
   loadSources,
 }) => {
   const {
-    services: { docLinks, application, http },
+    services: { docLinks, application, http, searchClient },
   } = useKibana<IndexPatternEditorContext>();
 
   const [remoteClustersExist, setRemoteClustersExist] = useState<boolean>(false);
@@ -47,15 +47,21 @@ export const EmptyPrompts: FC<Props> = ({
   useCallback(() => {
     let isMounted = true;
     if (!hasDataIndices)
-      getIndices(http, () => false, '*:*', false).then((dataSources) => {
+      getIndices({
+        http,
+        isRollupIndex: () => false,
+        pattern: '*:*',
+        showAllIndices: false,
+        searchClient,
+      }).then((dataSources) => {
         if (isMounted) {
           setRemoteClustersExist(!!dataSources.filter(removeAliases).length);
         }
       });
     return () => {
       isMounted = false;
     };
-  }, [http, hasDataIndices]);
+  }, [http, hasDataIndices, searchClient]);
 
   if (!hasExistingIndexPatterns && !goToForm) {
     if (!hasDataIndices && !remoteClustersExist) {

src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx

@@ -70,7 +70,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
 }: Props) => {
   const isMounted = useRef<boolean>(false);
   const {
-    services: { http, indexPatternService, uiSettings },
+    services: { http, indexPatternService, uiSettings, searchClient },
   } = useKibana<IndexPatternEditorContext>();
 
   const { form } = useForm<IndexPatternConfig, FormInternal>({
@@ -128,13 +128,19 @@ const IndexPatternEditorFlyoutContentComponent = ({
 
   // load all data sources and set initial matchedIndices
   const loadSources = useCallback(() => {
-    getIndices(http, () => false, '*', allowHidden).then((dataSources) => {
+    getIndices({
+      http,
+      isRollupIndex: () => false,
+      pattern: '*',
+      showAllIndices: allowHidden,
+      searchClient,
+    }).then((dataSources) => {
       setAllSources(dataSources);
       const matchedSet = getMatchedIndices(dataSources, [], [], allowHidden);
       setMatchedIndices(matchedSet);
       setIsLoadingSources(false);
     });
-  }, [http, allowHidden]);
+  }, [http, allowHidden, searchClient]);
 
   // loading list of index patterns
   useEffect(() => {
@@ -223,13 +229,31 @@ const IndexPatternEditorFlyoutContentComponent = ({
         const indexRequests = [];
 
         if (query?.endsWith('*')) {
-          const exactMatchedQuery = getIndices(http, isRollupIndex, query, allowHidden);
+          const exactMatchedQuery = getIndices({
+            http,
+            isRollupIndex,
+            pattern: query,
+            showAllIndices: allowHidden,
+            searchClient,
+          });
           indexRequests.push(exactMatchedQuery);
           // provide default value when not making a request for the partialMatchQuery
           indexRequests.push(Promise.resolve([]));
         } else {
-          const exactMatchQuery = getIndices(http, isRollupIndex, query, allowHidden);
-          const partialMatchQuery = getIndices(http, isRollupIndex, `${query}*`, allowHidden);
+          const exactMatchQuery = getIndices({
+            http,
+            isRollupIndex,
+            pattern: query,
+            showAllIndices: allowHidden,
+            searchClient,
+          });
+          const partialMatchQuery = getIndices({
+            http,
+            isRollupIndex,
+            pattern: `${query}*`,
+            showAllIndices: allowHidden,
+            searchClient,
+          });
 
           indexRequests.push(exactMatchQuery);
           indexRequests.push(partialMatchQuery);
@@ -264,7 +288,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
 
       return fetchIndices(newTitle);
     },
-    [http, allowHidden, allSources, type, rollupIndicesCapabilities]
+    [http, allowHidden, allSources, type, rollupIndicesCapabilities, searchClient]
   );
 
   useEffect(() => {

src/plugins/index_pattern_editor/public/constants.ts

@@ -9,3 +9,10 @@
 export const pluginName = 'index_pattern_editor';
 export const MAX_NUMBER_OF_MATCHING_INDICES = 100;
 export const CONFIG_ROLLUPS = 'rollups:enableIndexPatterns';
+
+// This isn't ideal. We want to avoid searching for 20 indices
+// then filtering out the majority of them because they are system indices.
+// We'd like to filter system indices out in the query
+// so if we can accomplish that in the future, this logic can go away
+export const ESTIMATED_NUMBER_OF_SYSTEM_INDICES = 100;
+export const MAX_SEARCH_SIZE = MAX_NUMBER_OF_MATCHING_INDICES + ESTIMATED_NUMBER_OF_SYSTEM_INDICES;

src/plugins/index_pattern_editor/public/lib/get_indices.test.ts

@@ -6,11 +6,17 @@
  * Side Public License, v 1.
  */
 
-import { getIndices, responseToItemArray } from './get_indices';
+import {
+  getIndices,
+  getIndicesViaSearch,
+  responseToItemArray,
+  dedupeMatchedItems,
+} from './get_indices';
 import { httpServiceMock } from '../../../../core/public/mocks';
-import { ResolveIndexResponseItemIndexAttrs } from '../types';
+import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types';
+import { Observable } from 'rxjs';
 
-export const successfulResponse = {
+export const successfulResolveResponse = {
   indices: [
     {
       name: 'remoteCluster1:bar-01',
@@ -32,28 +38,99 @@ export const successfulResponse = {
   ],
 };
 
-const mockGetTags = () => [];
-const mockIsRollupIndex = () => false;
+const successfulSearchResponse = {
+  isPartial: false,
+  isRunning: false,
+  rawResponse: {
+    aggregations: {
+      indices: {
+        buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }],
+      },
+    },
+  },
+};
+
+const partialSearchResponse = {
+  isPartial: true,
+  isRunning: true,
+  rawResponse: {
+    hits: {
+      total: 2,
+      hits: [],
+    },
+  },
+};
+
+const errorSearchResponse = {
+  isPartial: true,
+  isRunning: false,
+};
+
+const isRollupIndex = () => false;
+const getTags = () => [];
+const searchClient = () =>
+  new Observable((observer) => {
+    observer.next(successfulSearchResponse);
+    observer.complete();
+  }) as any;
 
 const http = httpServiceMock.createStartContract();
-http.get.mockResolvedValue(successfulResponse);
+http.get.mockResolvedValue(successfulResolveResponse);
 
 describe('getIndices', () => {
   it('should work in a basic case', async () => {
-    const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+    const uncalledSearchClient = jest.fn();
+    const result = await getIndices({
+      http,
+      pattern: 'kibana',
+      searchClient: uncalledSearchClient,
+      isRollupIndex,
+    });
+    expect(http.get).toHaveBeenCalled();
+    expect(uncalledSearchClient).not.toHaveBeenCalled();
     expect(result.length).toBe(3);
     expect(result[0].name).toBe('f-alias');
     expect(result[1].name).toBe('foo');
   });
 
+  it('should make two calls in cross cluser case', async () => {
+    http.get.mockResolvedValue(successfulResolveResponse);
+    const result = await getIndices({ http, pattern: '*:kibana', searchClient, isRollupIndex });
+
+    expect(http.get).toHaveBeenCalled();
+    expect(result.length).toBe(4);
+    expect(result[0].name).toBe('f-alias');
+    expect(result[1].name).toBe('foo');
+    expect(result[2].name).toBe('kibana_sample_data_ecommerce');
+    expect(result[3].name).toBe('remoteCluster1:bar-01');
+  });
+
   it('should ignore ccs query-all', async () => {
-    expect((await getIndices(http, mockIsRollupIndex, '*:', false)).length).toBe(0);
+    expect((await getIndices({ http, pattern: '*:', searchClient, isRollupIndex })).length).toBe(0);
   });
 
   it('should ignore a single comma', async () => {
-    expect((await getIndices(http, mockIsRollupIndex, ',', false)).length).toBe(0);
-    expect((await getIndices(http, mockIsRollupIndex, ',*', false)).length).toBe(0);
-    expect((await getIndices(http, mockIsRollupIndex, ',foobar', false)).length).toBe(0);
+    expect((await getIndices({ http, pattern: ',', searchClient, isRollupIndex })).length).toBe(0);
+    expect((await getIndices({ http, pattern: ',*', searchClient, isRollupIndex })).length).toBe(0);
+    expect(
+      (await getIndices({ http, pattern: ',foobar', searchClient, isRollupIndex })).length
+    ).toBe(0);
+  });
+
+  it('should work with partial responses', async () => {
+    const searchClientPartialResponse = () =>
+      new Observable((observer) => {
+        observer.next(partialSearchResponse);
+        observer.next(successfulSearchResponse);
+        observer.complete();
+      }) as any;
+    const result = await getIndices({
+      http,
+      pattern: '*:kibana',
+      searchClient: searchClientPartialResponse,
+      isRollupIndex,
+    });
+    expect(result.length).toBe(4);
   });
 
   it('response object to item array', () => {
@@ -81,16 +158,37 @@ describe('getIndices', () => {
         },
       ],
     };
-    expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot();
-    expect(responseToItemArray({}, mockGetTags)).toEqual([]);
+    expect(responseToItemArray(result, getTags)).toMatchSnapshot();
+    expect(responseToItemArray({}, getTags)).toEqual([]);
+  });
+
+  it('matched items are deduped', () => {
+    const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[];
+    const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[];
+    expect(dedupeMatchedItems(setA, setB)).toHaveLength(3);
   });
 
   describe('errors', () => {
-    it('should handle errors gracefully', async () => {
+    it('should handle thrown errors gracefully', async () => {
       http.get.mockImplementationOnce(() => {
         throw new Error('Test error');
       });
-      const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+      const result = await getIndices({ http, pattern: 'kibana', searchClient, isRollupIndex });
+      expect(result.length).toBe(0);
+    });
+
+    it('getIndicesViaSearch should handle error responses gracefully', async () => {
+      const searchClientErrorResponse = () =>
+        new Observable((observer) => {
+          observer.next(errorSearchResponse);
+          observer.complete();
+        }) as any;
+      const result = await getIndicesViaSearch({
+        pattern: '*:kibana',
+        searchClient: searchClientErrorResponse,
+        showAllIndices: false,
+        isRollupIndex,
+      });
       expect(result.length).toBe(0);
     });
   });