restore cross cluster search functionality

src/plugins/data/common/search/utils.ts

@@ -12,7 +12,7 @@ import type { IKibanaSearchResponse } from './types';
  * @returns true if response had an error while executing in ES
  */
 export const isErrorResponse = (response?: IKibanaSearchResponse) => {
-  return !response || !response.rawResponse || (!response.isRunning && response.isPartial);
+  return !response || !response.rawResponse || (!response.isRunning && !!response.isPartial);
 };
 
 /**

src/plugins/index_pattern_editor/public/constants.ts

@@ -9,3 +9,10 @@
 export const pluginName = 'index_pattern_editor';
 export const MAX_NUMBER_OF_MATCHING_INDICES = 100;
 export const CONFIG_ROLLUPS = 'rollups:enableIndexPatterns';
+
+// This isn't ideal. We want to avoid searching for 20 indices
+// then filtering out the majority of them because they are system indices.
+// We'd like to filter system indices out in the query
+// so if we can accomplish that in the future, this logic can go away
+export const ESTIMATED_NUMBER_OF_SYSTEM_INDICES = 100;
+export const MAX_SEARCH_SIZE = MAX_NUMBER_OF_MATCHING_INDICES + ESTIMATED_NUMBER_OF_SYSTEM_INDICES;

src/plugins/index_pattern_editor/public/lib/get_indices.test.ts

@@ -6,11 +6,17 @@
  * Side Public License, v 1.
  */
 
-import { getIndices, responseToItemArray } from './get_indices';
+import {
+  getIndices,
+  getIndicesViaSearch,
+  responseToItemArray,
+  dedupeMatchedItems,
+} from './get_indices';
 import { httpServiceMock } from '../../../../core/public/mocks';
-import { ResolveIndexResponseItemIndexAttrs } from '../types';
+import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types';
+import { Observable } from 'rxjs';
 
-export const successfulResponse = {
+export const successfulResolveResponse = {
   indices: [
     {
       name: 'remoteCluster1:bar-01',
@@ -32,28 +38,99 @@ export const successfulResponse = {
   ],
 };
 
-const mockGetTags = () => [];
-const mockIsRollupIndex = () => false;
+const successfulSearchResponse = {
+  isPartial: false,
+  isRunning: false,
+  rawResponse: {
+    aggregations: {
+      indices: {
+        buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }],
+      },
+    },
+  },
+};
+
+const partialSearchResponse = {
+  isPartial: true,
+  isRunning: true,
+  rawResponse: {
+    hits: {
+      total: 2,
+      hits: [],
+    },
+  },
+};
+
+const errorSearchResponse = {
+  isPartial: true,
+  isRunning: false,
+};
+
+const isRollupIndex = () => false;
+const getTags = () => [];
+const searchClient = () =>
+  new Observable((observer) => {
+    observer.next(successfulSearchResponse);
+    observer.complete();
+  }) as any;
 
 const http = httpServiceMock.createStartContract();
-http.get.mockResolvedValue(successfulResponse);
+http.get.mockResolvedValue(successfulResolveResponse);
 
 describe('getIndices', () => {
   it('should work in a basic case', async () => {
-    const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+    const uncalledSearchClient = jest.fn();
+    const result = await getIndices({
+      http,
+      pattern: 'kibana',
+      searchClient: uncalledSearchClient,
+      isRollupIndex,
+    });
+    expect(http.get).toHaveBeenCalled();
+    expect(uncalledSearchClient).not.toHaveBeenCalled();
     expect(result.length).toBe(3);
     expect(result[0].name).toBe('f-alias');
     expect(result[1].name).toBe('foo');
   });
 
+  it('should make two calls in cross cluser case', async () => {
+    http.get.mockResolvedValue(successfulResolveResponse);
+    const result = await getIndices({ http, pattern: '*:kibana', searchClient, isRollupIndex });
+
+    expect(http.get).toHaveBeenCalled();
+    expect(result.length).toBe(4);
+    expect(result[0].name).toBe('f-alias');
+    expect(result[1].name).toBe('foo');
+    expect(result[2].name).toBe('kibana_sample_data_ecommerce');
+    expect(result[3].name).toBe('remoteCluster1:bar-01');
+  });
+
   it('should ignore ccs query-all', async () => {
-    expect((await getIndices(http, mockIsRollupIndex, '*:', false)).length).toBe(0);
+    expect((await getIndices({ http, pattern: '*:', searchClient, isRollupIndex })).length).toBe(0);
   });
 
   it('should ignore a single comma', async () => {
-    expect((await getIndices(http, mockIsRollupIndex, ',', false)).length).toBe(0);
-    expect((await getIndices(http, mockIsRollupIndex, ',*', false)).length).toBe(0);
-    expect((await getIndices(http, mockIsRollupIndex, ',foobar', false)).length).toBe(0);
+    expect((await getIndices({ http, pattern: ',', searchClient, isRollupIndex })).length).toBe(0);
+    expect((await getIndices({ http, pattern: ',*', searchClient, isRollupIndex })).length).toBe(0);
+    expect(
+      (await getIndices({ http, pattern: ',foobar', searchClient, isRollupIndex })).length
+    ).toBe(0);
+  });
+
+  it('should work with partial responses', async () => {
+    const searchClientPartialResponse = () =>
+      new Observable((observer) => {
+        observer.next(partialSearchResponse);
+        observer.next(successfulSearchResponse);
+        observer.complete();
+      }) as any;
+    const result = await getIndices({
+      http,
+      pattern: '*:kibana',
+      searchClient: searchClientPartialResponse,
+      isRollupIndex,
+    });
+    expect(result.length).toBe(4);
   });
 
   it('response object to item array', () => {
@@ -81,16 +158,37 @@ describe('getIndices', () => {
         },
       ],
     };
-    expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot();
-    expect(responseToItemArray({}, mockGetTags)).toEqual([]);
+    expect(responseToItemArray(result, getTags)).toMatchSnapshot();
+    expect(responseToItemArray({}, getTags)).toEqual([]);
+  });
+
+  it('matched items are deduped', () => {
+    const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[];
+    const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[];
+    expect(dedupeMatchedItems(setA, setB)).toHaveLength(3);
   });
 
   describe('errors', () => {
-    it('should handle errors gracefully', async () => {
+    it('should handle thrown errors gracefully', async () => {
       http.get.mockImplementationOnce(() => {
         throw new Error('Test error');
       });
-      const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+      const result = await getIndices({ http, pattern: 'kibana', searchClient, isRollupIndex });
+      expect(result.length).toBe(0);
+    });
+
+    it('getIndicesViaSearch should handle error responses gracefully', async () => {
+      const searchClientErrorResponse = () =>
+        new Observable((observer) => {
+          observer.next(errorSearchResponse);
+          observer.complete();
+        }) as any;
+      const result = await getIndicesViaSearch({
+        pattern: '*:kibana',
+        searchClient: searchClientErrorResponse,
+        showAllIndices: false,
+        isRollupIndex,
+      });
       expect(result.length).toBe(0);
     });
   });

src/plugins/index_pattern_editor/public/lib/get_indices.ts

@@ -8,10 +8,18 @@
 
 import { sortBy } from 'lodash';
 import { HttpStart } from 'kibana/public';
+import { map, filter } from 'rxjs/operators';
 import { i18n } from '@kbn/i18n';
 import { Tag, INDEX_PATTERN_TYPE } from '../types';
-// todo move into this plugin, consider removing all ipm references
 import { MatchedItem, ResolveIndexResponse, ResolveIndexResponseItemIndexAttrs } from '../types';
+import { MAX_SEARCH_SIZE } from '../constants';
+
+import {
+  DataPublicPluginStart,
+  IEsSearchResponse,
+  isErrorResponse,
+  isCompleteResponse,
+} from '../../../data/public';
 
 const aliasLabel = i18n.translate('indexPatternEditor.aliasLabel', { defaultMessage: 'Alias' });
 const dataStreamLabel = i18n.translate('indexPatternEditor.dataStreamLabel', {
@@ -41,13 +49,137 @@ const getIndexTags = (isRollupIndex: (indexName: string) => boolean) => (indexNa
       ]
     : [];
 
-export async function getIndices(
-  http: HttpStart,
-  isRollupIndex: (indexName: string) => boolean,
-  rawPattern: string,
+export const searchResponseToArray = (
+  getTags: (indexName: string) => Tag[],
   showAllIndices: boolean
-): Promise<MatchedItem[]> {
+) => (response: IEsSearchResponse<any>) => {
+  const { rawResponse } = response;
+  if (!rawResponse.aggregations) {
+    return [];
+  } else {
+    // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response
+    return rawResponse.aggregations.indices.buckets
+      .map((bucket: { key: string }) => {
+        return bucket.key;
+      })
+      .filter((indexName: string) => {
+        if (showAllIndices) {
+          return true;
+        } else {
+          return !indexName.startsWith('.');
+        }
+      })
+      .map((indexName: string) => {
+        return {
+          name: indexName,
+          tags: getTags(indexName),
+          item: {},
+        };
+      });
+  }
+};
+
+export const getIndicesViaSearch = async ({
+  pattern,
+  searchClient,
+  showAllIndices,
+  isRollupIndex,
+}: {
+  pattern: string;
+  searchClient: DataPublicPluginStart['search']['search'];
+  showAllIndices: boolean;
+  isRollupIndex: (indexName: string) => boolean;
+}): Promise<MatchedItem[]> =>
+  searchClient({
+    params: {
+      ignoreUnavailable: true,
+      expand_wildcards: showAllIndices ? 'all' : 'open',
+      index: pattern,
+      body: {
+        size: 0, // no hits
+        aggs: {
+          indices: {
+            terms: {
+              field: '_index',
+              size: MAX_SEARCH_SIZE,
+            },
+          },
+        },
+      },
+    },
+  })
+    .pipe(
+      filter((resp) => isCompleteResponse(resp) || isErrorResponse(resp)),
+      map(searchResponseToArray(getIndexTags(isRollupIndex), showAllIndices))
+    )
+    .toPromise()
+    .catch(() => []);
+
+export const getIndicesViaResolve = async ({
+  http,
+  pattern,
+  showAllIndices,
+  isRollupIndex,
+}: {
+  http: HttpStart;
+  pattern: string;
+  showAllIndices: boolean;
+  isRollupIndex: (indexName: string) => boolean;
+}) =>
+  http
+    .get<ResolveIndexResponse>(`/internal/index-pattern-management/resolve_index/${pattern}`, {
+      query: showAllIndices ? { expand_wildcards: 'all' } : undefined,
+    })
+    .then((response) => {
+      if (!response) {
+        return [];
+      } else {
+        return responseToItemArray(response, getIndexTags(isRollupIndex));
+      }
+    });
+
+/**
+ * Takes two MatchedItem[]s and returns a merged set, with the second set prrioritized over the first based on name
+ *
+ * @param matchedA
+ * @param matchedB
+ */
+
+export const dedupeMatchedItems = (matchedA: MatchedItem[], matchedB: MatchedItem[]) => {
+  const mergedMatchedItems = matchedA.reduce((col, item) => {
+    col[item.name] = item;
+    return col;
+  }, {} as Record<string, MatchedItem>);
+
+  matchedB.reduce((col, item) => {
+    col[item.name] = item;
+    return col;
+  }, mergedMatchedItems);
+
+  return Object.values(mergedMatchedItems).sort((a, b) => {
+    if (a.name > b.name) return 1;
+    if (b.name > a.name) return -1;
+
+    return 0;
+  });
+};
+
+export async function getIndices({
+  http,
+  pattern: rawPattern,
+  showAllIndices = false,
+  searchClient,
+  isRollupIndex,
+}: {
+  http: HttpStart;
+  pattern: string;
+  showAllIndices?: boolean;
+  searchClient: DataPublicPluginStart['search']['search'];
+  isRollupIndex: (indexName: string) => boolean;
+}): Promise<MatchedItem[]> {
   const pattern = rawPattern.trim();
+  const isCCS = pattern.indexOf(':') !== -1;
+  const requests: Array<Promise<MatchedItem[]>> = [];
 
   // Searching for `*:` fails for CCS environments. The search request
   // is worthless anyways as the we should only send a request
@@ -67,20 +199,32 @@ export async function getIndices(
     return [];
   }
 
-  const query = showAllIndices ? { expand_wildcards: 'all' } : undefined;
+  const promiseResolve = getIndicesViaResolve({
+    http,
+    pattern,
+    showAllIndices,
+    isRollupIndex,
+  }).catch(() => []);
+  requests.push(promiseResolve);
 
-  try {
-    const response = await http.get<ResolveIndexResponse>(
-      `/internal/index-pattern-management/resolve_index/${pattern}`,
-      { query }
-    );
-    if (!response) {
-      return [];
-    }
+  if (isCCS) {
+    // CCS supports ±1 major version. We won't be able to expect resolve endpoint to exist until v9
+    const promiseSearch = getIndicesViaSearch({
+      pattern,
+      searchClient,
+      showAllIndices,
+      isRollupIndex,
+    }).catch(() => []);
+    requests.push(promiseSearch);
+  }
 
-    return responseToItemArray(response, getIndexTags(isRollupIndex));
-  } catch {
-    return [];
+  const responses = await Promise.all(requests);
+
+  if (responses.length === 2) {
+    const [resolveResponse, searchResponse] = responses;
+    return dedupeMatchedItems(searchResponse, resolveResponse);
+  } else {
+    return responses[0];
   }
 }