From b084f026d183b286415a81e02ead1bdd2b20327b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 16 Aug 2021 13:03:54 -0500 Subject: [PATCH] restore cross cluster search functionality --- src/plugins/data/common/search/utils.ts | 2 +- .../index_pattern_editor/public/constants.ts | 7 + .../public/lib/get_indices.test.ts | 128 +++++++++++-- .../public/lib/get_indices.ts | 180 ++++++++++++++++-- 4 files changed, 283 insertions(+), 34 deletions(-) diff --git a/src/plugins/data/common/search/utils.ts b/src/plugins/data/common/search/utils.ts index e11957c6fa9fc..ea5ac28852d6a 100644 --- a/src/plugins/data/common/search/utils.ts +++ b/src/plugins/data/common/search/utils.ts @@ -12,7 +12,7 @@ import type { IKibanaSearchResponse } from './types'; * @returns true if response had an error while executing in ES */ export const isErrorResponse = (response?: IKibanaSearchResponse) => { - return !response || !response.rawResponse || (!response.isRunning && response.isPartial); + return !response || !response.rawResponse || (!response.isRunning && !!response.isPartial); }; /** diff --git a/src/plugins/index_pattern_editor/public/constants.ts b/src/plugins/index_pattern_editor/public/constants.ts index ff74e0827fa50..8d325184353df 100644 --- a/src/plugins/index_pattern_editor/public/constants.ts +++ b/src/plugins/index_pattern_editor/public/constants.ts @@ -9,3 +9,10 @@ export const pluginName = 'index_pattern_editor'; export const MAX_NUMBER_OF_MATCHING_INDICES = 100; export const CONFIG_ROLLUPS = 'rollups:enableIndexPatterns'; + +// This isn't ideal. We want to avoid searching for 20 indices +// then filtering out the majority of them because they are system indices. +// We'd like to filter system indices out in the query +// so if we can accomplish that in the future, this logic can go away +export const ESTIMATED_NUMBER_OF_SYSTEM_INDICES = 100; +export const MAX_SEARCH_SIZE = MAX_NUMBER_OF_MATCHING_INDICES + ESTIMATED_NUMBER_OF_SYSTEM_INDICES; diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts index fc96482f0379f..d65cd27e090bb 100644 --- a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts +++ b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts @@ -6,11 +6,17 @@ * Side Public License, v 1. */ -import { getIndices, responseToItemArray } from './get_indices'; +import { + getIndices, + getIndicesViaSearch, + responseToItemArray, + dedupeMatchedItems, +} from './get_indices'; import { httpServiceMock } from '../../../../core/public/mocks'; -import { ResolveIndexResponseItemIndexAttrs } from '../types'; +import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types'; +import { Observable } from 'rxjs'; -export const successfulResponse = { +export const successfulResolveResponse = { indices: [ { name: 'remoteCluster1:bar-01', @@ -32,28 +38,99 @@ export const successfulResponse = { ], }; -const mockGetTags = () => []; -const mockIsRollupIndex = () => false; +const successfulSearchResponse = { + isPartial: false, + isRunning: false, + rawResponse: { + aggregations: { + indices: { + buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }], + }, + }, + }, +}; + +const partialSearchResponse = { + isPartial: true, + isRunning: true, + rawResponse: { + hits: { + total: 2, + hits: [], + }, + }, +}; + +const errorSearchResponse = { + isPartial: true, + isRunning: false, +}; + +const isRollupIndex = () => false; +const getTags = () => []; +const searchClient = () => + new Observable((observer) => { + observer.next(successfulSearchResponse); + observer.complete(); + }) as any; const http = httpServiceMock.createStartContract(); -http.get.mockResolvedValue(successfulResponse); +http.get.mockResolvedValue(successfulResolveResponse); describe('getIndices', () => { it('should work in a basic case', async () => { - const result = await getIndices(http, mockIsRollupIndex, 'kibana', false); + const uncalledSearchClient = jest.fn(); + const result = await getIndices({ + http, + pattern: 'kibana', + searchClient: uncalledSearchClient, + isRollupIndex, + }); + expect(http.get).toHaveBeenCalled(); + expect(uncalledSearchClient).not.toHaveBeenCalled(); expect(result.length).toBe(3); expect(result[0].name).toBe('f-alias'); expect(result[1].name).toBe('foo'); }); + it('should make two calls in cross cluser case', async () => { + http.get.mockResolvedValue(successfulResolveResponse); + const result = await getIndices({ http, pattern: '*:kibana', searchClient, isRollupIndex }); + + expect(http.get).toHaveBeenCalled(); + expect(result.length).toBe(4); + expect(result[0].name).toBe('f-alias'); + expect(result[1].name).toBe('foo'); + expect(result[2].name).toBe('kibana_sample_data_ecommerce'); + expect(result[3].name).toBe('remoteCluster1:bar-01'); + }); + it('should ignore ccs query-all', async () => { - expect((await getIndices(http, mockIsRollupIndex, '*:', false)).length).toBe(0); + expect((await getIndices({ http, pattern: '*:', searchClient, isRollupIndex })).length).toBe(0); }); it('should ignore a single comma', async () => { - expect((await getIndices(http, mockIsRollupIndex, ',', false)).length).toBe(0); - expect((await getIndices(http, mockIsRollupIndex, ',*', false)).length).toBe(0); - expect((await getIndices(http, mockIsRollupIndex, ',foobar', false)).length).toBe(0); + expect((await getIndices({ http, pattern: ',', searchClient, isRollupIndex })).length).toBe(0); + expect((await getIndices({ http, pattern: ',*', searchClient, isRollupIndex })).length).toBe(0); + expect( + (await getIndices({ http, pattern: ',foobar', searchClient, isRollupIndex })).length + ).toBe(0); + }); + + it('should work with partial responses', async () => { + const searchClientPartialResponse = () => + new Observable((observer) => { + observer.next(partialSearchResponse); + observer.next(successfulSearchResponse); + observer.complete(); + }) as any; + const result = await getIndices({ + http, + pattern: '*:kibana', + searchClient: searchClientPartialResponse, + isRollupIndex, + }); + expect(result.length).toBe(4); }); it('response object to item array', () => { @@ -81,16 +158,37 @@ describe('getIndices', () => { }, ], }; - expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot(); - expect(responseToItemArray({}, mockGetTags)).toEqual([]); + expect(responseToItemArray(result, getTags)).toMatchSnapshot(); + expect(responseToItemArray({}, getTags)).toEqual([]); + }); + + it('matched items are deduped', () => { + const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[]; + const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[]; + expect(dedupeMatchedItems(setA, setB)).toHaveLength(3); }); describe('errors', () => { - it('should handle errors gracefully', async () => { + it('should handle thrown errors gracefully', async () => { http.get.mockImplementationOnce(() => { throw new Error('Test error'); }); - const result = await getIndices(http, mockIsRollupIndex, 'kibana', false); + const result = await getIndices({ http, pattern: 'kibana', searchClient, isRollupIndex }); + expect(result.length).toBe(0); + }); + + it('getIndicesViaSearch should handle error responses gracefully', async () => { + const searchClientErrorResponse = () => + new Observable((observer) => { + observer.next(errorSearchResponse); + observer.complete(); + }) as any; + const result = await getIndicesViaSearch({ + pattern: '*:kibana', + searchClient: searchClientErrorResponse, + showAllIndices: false, + isRollupIndex, + }); expect(result.length).toBe(0); }); }); diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.ts index 625e99ecbcdc5..00b5dcdfb9678 100644 --- a/src/plugins/index_pattern_editor/public/lib/get_indices.ts +++ b/src/plugins/index_pattern_editor/public/lib/get_indices.ts @@ -8,10 +8,18 @@ import { sortBy } from 'lodash'; import { HttpStart } from 'kibana/public'; +import { map, filter } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { Tag, INDEX_PATTERN_TYPE } from '../types'; -// todo move into this plugin, consider removing all ipm references import { MatchedItem, ResolveIndexResponse, ResolveIndexResponseItemIndexAttrs } from '../types'; +import { MAX_SEARCH_SIZE } from '../constants'; + +import { + DataPublicPluginStart, + IEsSearchResponse, + isErrorResponse, + isCompleteResponse, +} from '../../../data/public'; const aliasLabel = i18n.translate('indexPatternEditor.aliasLabel', { defaultMessage: 'Alias' }); const dataStreamLabel = i18n.translate('indexPatternEditor.dataStreamLabel', { @@ -41,13 +49,137 @@ const getIndexTags = (isRollupIndex: (indexName: string) => boolean) => (indexNa ] : []; -export async function getIndices( - http: HttpStart, - isRollupIndex: (indexName: string) => boolean, - rawPattern: string, +export const searchResponseToArray = ( + getTags: (indexName: string) => Tag[], showAllIndices: boolean -): Promise { +) => (response: IEsSearchResponse) => { + const { rawResponse } = response; + if (!rawResponse.aggregations) { + return []; + } else { + // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response + return rawResponse.aggregations.indices.buckets + .map((bucket: { key: string }) => { + return bucket.key; + }) + .filter((indexName: string) => { + if (showAllIndices) { + return true; + } else { + return !indexName.startsWith('.'); + } + }) + .map((indexName: string) => { + return { + name: indexName, + tags: getTags(indexName), + item: {}, + }; + }); + } +}; + +export const getIndicesViaSearch = async ({ + pattern, + searchClient, + showAllIndices, + isRollupIndex, +}: { + pattern: string; + searchClient: DataPublicPluginStart['search']['search']; + showAllIndices: boolean; + isRollupIndex: (indexName: string) => boolean; +}): Promise => + searchClient({ + params: { + ignoreUnavailable: true, + expand_wildcards: showAllIndices ? 'all' : 'open', + index: pattern, + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: MAX_SEARCH_SIZE, + }, + }, + }, + }, + }, + }) + .pipe( + filter((resp) => isCompleteResponse(resp) || isErrorResponse(resp)), + map(searchResponseToArray(getIndexTags(isRollupIndex), showAllIndices)) + ) + .toPromise() + .catch(() => []); + +export const getIndicesViaResolve = async ({ + http, + pattern, + showAllIndices, + isRollupIndex, +}: { + http: HttpStart; + pattern: string; + showAllIndices: boolean; + isRollupIndex: (indexName: string) => boolean; +}) => + http + .get(`/internal/index-pattern-management/resolve_index/${pattern}`, { + query: showAllIndices ? { expand_wildcards: 'all' } : undefined, + }) + .then((response) => { + if (!response) { + return []; + } else { + return responseToItemArray(response, getIndexTags(isRollupIndex)); + } + }); + +/** + * Takes two MatchedItem[]s and returns a merged set, with the second set prrioritized over the first based on name + * + * @param matchedA + * @param matchedB + */ + +export const dedupeMatchedItems = (matchedA: MatchedItem[], matchedB: MatchedItem[]) => { + const mergedMatchedItems = matchedA.reduce((col, item) => { + col[item.name] = item; + return col; + }, {} as Record); + + matchedB.reduce((col, item) => { + col[item.name] = item; + return col; + }, mergedMatchedItems); + + return Object.values(mergedMatchedItems).sort((a, b) => { + if (a.name > b.name) return 1; + if (b.name > a.name) return -1; + + return 0; + }); +}; + +export async function getIndices({ + http, + pattern: rawPattern, + showAllIndices = false, + searchClient, + isRollupIndex, +}: { + http: HttpStart; + pattern: string; + showAllIndices?: boolean; + searchClient: DataPublicPluginStart['search']['search']; + isRollupIndex: (indexName: string) => boolean; +}): Promise { const pattern = rawPattern.trim(); + const isCCS = pattern.indexOf(':') !== -1; + const requests: Array> = []; // Searching for `*:` fails for CCS environments. The search request // is worthless anyways as the we should only send a request @@ -67,20 +199,32 @@ export async function getIndices( return []; } - const query = showAllIndices ? { expand_wildcards: 'all' } : undefined; + const promiseResolve = getIndicesViaResolve({ + http, + pattern, + showAllIndices, + isRollupIndex, + }).catch(() => []); + requests.push(promiseResolve); - try { - const response = await http.get( - `/internal/index-pattern-management/resolve_index/${pattern}`, - { query } - ); - if (!response) { - return []; - } + if (isCCS) { + // CCS supports ±1 major version. We won't be able to expect resolve endpoint to exist until v9 + const promiseSearch = getIndicesViaSearch({ + pattern, + searchClient, + showAllIndices, + isRollupIndex, + }).catch(() => []); + requests.push(promiseSearch); + } - return responseToItemArray(response, getIndexTags(isRollupIndex)); - } catch { - return []; + const responses = await Promise.all(requests); + + if (responses.length === 2) { + const [resolveResponse, searchResponse] = responses; + return dedupeMatchedItems(searchResponse, resolveResponse); + } else { + return responses[0]; } }