From 5b5e152e2fa799e555b009856dc059fcf22a48c9 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Tue, 17 Aug 2021 12:52:07 -0500 Subject: [PATCH] [index pattern management] Restore cross cluster search functionality (#108756) * restore cross cluster search functionality --- ...gin-plugins-data-public.iserrorresponse.md | 2 +- src/plugins/data/common/search/utils.ts | 2 +- src/plugins/data/public/public.api.md | 2 +- .../empty_prompts/empty_prompts.tsx | 12 +- .../index_pattern_editor_flyout_content.tsx | 38 +++- .../index_pattern_editor/public/constants.ts | 7 + .../public/lib/get_indices.test.ts | 128 +++++++++++-- .../public/lib/get_indices.ts | 180 ++++++++++++++++-- .../public/open_editor.tsx | 4 +- .../index_pattern_editor/public/plugin.tsx | 2 + .../index_pattern_editor/public/types.ts | 1 + 11 files changed, 331 insertions(+), 47 deletions(-) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md index e4ac35f19e959..93dfdeb056f15 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md @@ -7,5 +7,5 @@ Signature: ```typescript -isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean | undefined +isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean ``` diff --git a/src/plugins/data/common/search/utils.ts b/src/plugins/data/common/search/utils.ts index e11957c6fa9fc..ea5ac28852d6a 100644 --- a/src/plugins/data/common/search/utils.ts +++ b/src/plugins/data/common/search/utils.ts @@ -12,7 +12,7 @@ import type { IKibanaSearchResponse } from './types'; * @returns true if response had an error while executing in ES */ export const isErrorResponse = (response?: IKibanaSearchResponse) => { - return !response || !response.rawResponse || (!response.isRunning && response.isPartial); + return !response || !response.rawResponse || (!response.isRunning && !!response.isPartial); }; /** diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 485dad1daea9d..7a28492a78657 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1568,7 +1568,7 @@ export interface ISearchStartSearchSource { // Warning: (ae-missing-release-tag) "isErrorResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean | undefined; +export const isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean; // Warning: (ae-missing-release-tag) "isEsError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx index 3b06fa1cff298..80224dbfb673f 100644 --- a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx +++ b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx @@ -36,7 +36,7 @@ export const EmptyPrompts: FC = ({ loadSources, }) => { const { - services: { docLinks, application, http }, + services: { docLinks, application, http, searchClient }, } = useKibana(); const [remoteClustersExist, setRemoteClustersExist] = useState(false); @@ -47,7 +47,13 @@ export const EmptyPrompts: FC = ({ useCallback(() => { let isMounted = true; if (!hasDataIndices) - getIndices(http, () => false, '*:*', false).then((dataSources) => { + getIndices({ + http, + isRollupIndex: () => false, + pattern: '*:*', + showAllIndices: false, + searchClient, + }).then((dataSources) => { if (isMounted) { setRemoteClustersExist(!!dataSources.filter(removeAliases).length); } @@ -55,7 +61,7 @@ export const EmptyPrompts: FC = ({ return () => { isMounted = false; }; - }, [http, hasDataIndices]); + }, [http, hasDataIndices, searchClient]); if (!hasExistingIndexPatterns && !goToForm) { if (!hasDataIndices && !remoteClustersExist) { diff --git a/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx b/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx index cabff9bfb009b..4f6f7708d90c0 100644 --- a/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx +++ b/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx @@ -70,7 +70,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ }: Props) => { const isMounted = useRef(false); const { - services: { http, indexPatternService, uiSettings }, + services: { http, indexPatternService, uiSettings, searchClient }, } = useKibana(); const { form } = useForm({ @@ -128,13 +128,19 @@ const IndexPatternEditorFlyoutContentComponent = ({ // load all data sources and set initial matchedIndices const loadSources = useCallback(() => { - getIndices(http, () => false, '*', allowHidden).then((dataSources) => { + getIndices({ + http, + isRollupIndex: () => false, + pattern: '*', + showAllIndices: allowHidden, + searchClient, + }).then((dataSources) => { setAllSources(dataSources); const matchedSet = getMatchedIndices(dataSources, [], [], allowHidden); setMatchedIndices(matchedSet); setIsLoadingSources(false); }); - }, [http, allowHidden]); + }, [http, allowHidden, searchClient]); // loading list of index patterns useEffect(() => { @@ -223,13 +229,31 @@ const IndexPatternEditorFlyoutContentComponent = ({ const indexRequests = []; if (query?.endsWith('*')) { - const exactMatchedQuery = getIndices(http, isRollupIndex, query, allowHidden); + const exactMatchedQuery = getIndices({ + http, + isRollupIndex, + pattern: query, + showAllIndices: allowHidden, + searchClient, + }); indexRequests.push(exactMatchedQuery); // provide default value when not making a request for the partialMatchQuery indexRequests.push(Promise.resolve([])); } else { - const exactMatchQuery = getIndices(http, isRollupIndex, query, allowHidden); - const partialMatchQuery = getIndices(http, isRollupIndex, `${query}*`, allowHidden); + const exactMatchQuery = getIndices({ + http, + isRollupIndex, + pattern: query, + showAllIndices: allowHidden, + searchClient, + }); + const partialMatchQuery = getIndices({ + http, + isRollupIndex, + pattern: `${query}*`, + showAllIndices: allowHidden, + searchClient, + }); indexRequests.push(exactMatchQuery); indexRequests.push(partialMatchQuery); @@ -264,7 +288,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ return fetchIndices(newTitle); }, - [http, allowHidden, allSources, type, rollupIndicesCapabilities] + [http, allowHidden, allSources, type, rollupIndicesCapabilities, searchClient] ); useEffect(() => { diff --git a/src/plugins/index_pattern_editor/public/constants.ts b/src/plugins/index_pattern_editor/public/constants.ts index ff74e0827fa50..8d325184353df 100644 --- a/src/plugins/index_pattern_editor/public/constants.ts +++ b/src/plugins/index_pattern_editor/public/constants.ts @@ -9,3 +9,10 @@ export const pluginName = 'index_pattern_editor'; export const MAX_NUMBER_OF_MATCHING_INDICES = 100; export const CONFIG_ROLLUPS = 'rollups:enableIndexPatterns'; + +// This isn't ideal. We want to avoid searching for 20 indices +// then filtering out the majority of them because they are system indices. +// We'd like to filter system indices out in the query +// so if we can accomplish that in the future, this logic can go away +export const ESTIMATED_NUMBER_OF_SYSTEM_INDICES = 100; +export const MAX_SEARCH_SIZE = MAX_NUMBER_OF_MATCHING_INDICES + ESTIMATED_NUMBER_OF_SYSTEM_INDICES; diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts index fc96482f0379f..d65cd27e090bb 100644 --- a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts +++ b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts @@ -6,11 +6,17 @@ * Side Public License, v 1. */ -import { getIndices, responseToItemArray } from './get_indices'; +import { + getIndices, + getIndicesViaSearch, + responseToItemArray, + dedupeMatchedItems, +} from './get_indices'; import { httpServiceMock } from '../../../../core/public/mocks'; -import { ResolveIndexResponseItemIndexAttrs } from '../types'; +import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types'; +import { Observable } from 'rxjs'; -export const successfulResponse = { +export const successfulResolveResponse = { indices: [ { name: 'remoteCluster1:bar-01', @@ -32,28 +38,99 @@ export const successfulResponse = { ], }; -const mockGetTags = () => []; -const mockIsRollupIndex = () => false; +const successfulSearchResponse = { + isPartial: false, + isRunning: false, + rawResponse: { + aggregations: { + indices: { + buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }], + }, + }, + }, +}; + +const partialSearchResponse = { + isPartial: true, + isRunning: true, + rawResponse: { + hits: { + total: 2, + hits: [], + }, + }, +}; + +const errorSearchResponse = { + isPartial: true, + isRunning: false, +}; + +const isRollupIndex = () => false; +const getTags = () => []; +const searchClient = () => + new Observable((observer) => { + observer.next(successfulSearchResponse); + observer.complete(); + }) as any; const http = httpServiceMock.createStartContract(); -http.get.mockResolvedValue(successfulResponse); +http.get.mockResolvedValue(successfulResolveResponse); describe('getIndices', () => { it('should work in a basic case', async () => { - const result = await getIndices(http, mockIsRollupIndex, 'kibana', false); + const uncalledSearchClient = jest.fn(); + const result = await getIndices({ + http, + pattern: 'kibana', + searchClient: uncalledSearchClient, + isRollupIndex, + }); + expect(http.get).toHaveBeenCalled(); + expect(uncalledSearchClient).not.toHaveBeenCalled(); expect(result.length).toBe(3); expect(result[0].name).toBe('f-alias'); expect(result[1].name).toBe('foo'); }); + it('should make two calls in cross cluser case', async () => { + http.get.mockResolvedValue(successfulResolveResponse); + const result = await getIndices({ http, pattern: '*:kibana', searchClient, isRollupIndex }); + + expect(http.get).toHaveBeenCalled(); + expect(result.length).toBe(4); + expect(result[0].name).toBe('f-alias'); + expect(result[1].name).toBe('foo'); + expect(result[2].name).toBe('kibana_sample_data_ecommerce'); + expect(result[3].name).toBe('remoteCluster1:bar-01'); + }); + it('should ignore ccs query-all', async () => { - expect((await getIndices(http, mockIsRollupIndex, '*:', false)).length).toBe(0); + expect((await getIndices({ http, pattern: '*:', searchClient, isRollupIndex })).length).toBe(0); }); it('should ignore a single comma', async () => { - expect((await getIndices(http, mockIsRollupIndex, ',', false)).length).toBe(0); - expect((await getIndices(http, mockIsRollupIndex, ',*', false)).length).toBe(0); - expect((await getIndices(http, mockIsRollupIndex, ',foobar', false)).length).toBe(0); + expect((await getIndices({ http, pattern: ',', searchClient, isRollupIndex })).length).toBe(0); + expect((await getIndices({ http, pattern: ',*', searchClient, isRollupIndex })).length).toBe(0); + expect( + (await getIndices({ http, pattern: ',foobar', searchClient, isRollupIndex })).length + ).toBe(0); + }); + + it('should work with partial responses', async () => { + const searchClientPartialResponse = () => + new Observable((observer) => { + observer.next(partialSearchResponse); + observer.next(successfulSearchResponse); + observer.complete(); + }) as any; + const result = await getIndices({ + http, + pattern: '*:kibana', + searchClient: searchClientPartialResponse, + isRollupIndex, + }); + expect(result.length).toBe(4); }); it('response object to item array', () => { @@ -81,16 +158,37 @@ describe('getIndices', () => { }, ], }; - expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot(); - expect(responseToItemArray({}, mockGetTags)).toEqual([]); + expect(responseToItemArray(result, getTags)).toMatchSnapshot(); + expect(responseToItemArray({}, getTags)).toEqual([]); + }); + + it('matched items are deduped', () => { + const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[]; + const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[]; + expect(dedupeMatchedItems(setA, setB)).toHaveLength(3); }); describe('errors', () => { - it('should handle errors gracefully', async () => { + it('should handle thrown errors gracefully', async () => { http.get.mockImplementationOnce(() => { throw new Error('Test error'); }); - const result = await getIndices(http, mockIsRollupIndex, 'kibana', false); + const result = await getIndices({ http, pattern: 'kibana', searchClient, isRollupIndex }); + expect(result.length).toBe(0); + }); + + it('getIndicesViaSearch should handle error responses gracefully', async () => { + const searchClientErrorResponse = () => + new Observable((observer) => { + observer.next(errorSearchResponse); + observer.complete(); + }) as any; + const result = await getIndicesViaSearch({ + pattern: '*:kibana', + searchClient: searchClientErrorResponse, + showAllIndices: false, + isRollupIndex, + }); expect(result.length).toBe(0); }); }); diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.ts index 625e99ecbcdc5..8d642174232ac 100644 --- a/src/plugins/index_pattern_editor/public/lib/get_indices.ts +++ b/src/plugins/index_pattern_editor/public/lib/get_indices.ts @@ -8,10 +8,18 @@ import { sortBy } from 'lodash'; import { HttpStart } from 'kibana/public'; +import { map, filter } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { Tag, INDEX_PATTERN_TYPE } from '../types'; -// todo move into this plugin, consider removing all ipm references import { MatchedItem, ResolveIndexResponse, ResolveIndexResponseItemIndexAttrs } from '../types'; +import { MAX_SEARCH_SIZE } from '../constants'; + +import { + DataPublicPluginStart, + IEsSearchResponse, + isErrorResponse, + isCompleteResponse, +} from '../../../data/public'; const aliasLabel = i18n.translate('indexPatternEditor.aliasLabel', { defaultMessage: 'Alias' }); const dataStreamLabel = i18n.translate('indexPatternEditor.dataStreamLabel', { @@ -41,13 +49,137 @@ const getIndexTags = (isRollupIndex: (indexName: string) => boolean) => (indexNa ] : []; -export async function getIndices( - http: HttpStart, - isRollupIndex: (indexName: string) => boolean, - rawPattern: string, +export const searchResponseToArray = ( + getTags: (indexName: string) => Tag[], showAllIndices: boolean -): Promise { +) => (response: IEsSearchResponse) => { + const { rawResponse } = response; + if (!rawResponse.aggregations) { + return []; + } else { + // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response + return rawResponse.aggregations.indices.buckets + .map((bucket: { key: string }) => { + return bucket.key; + }) + .filter((indexName: string) => { + if (showAllIndices) { + return true; + } else { + return !indexName.startsWith('.'); + } + }) + .map((indexName: string) => { + return { + name: indexName, + tags: getTags(indexName), + item: {}, + }; + }); + } +}; + +export const getIndicesViaSearch = async ({ + pattern, + searchClient, + showAllIndices, + isRollupIndex, +}: { + pattern: string; + searchClient: DataPublicPluginStart['search']['search']; + showAllIndices: boolean; + isRollupIndex: (indexName: string) => boolean; +}): Promise => + searchClient({ + params: { + ignoreUnavailable: true, + expand_wildcards: showAllIndices ? 'all' : 'open', + index: pattern, + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: MAX_SEARCH_SIZE, + }, + }, + }, + }, + }, + }) + .pipe( + filter((resp) => isCompleteResponse(resp) || isErrorResponse(resp)), + map(searchResponseToArray(getIndexTags(isRollupIndex), showAllIndices)) + ) + .toPromise() + .catch(() => []); + +export const getIndicesViaResolve = async ({ + http, + pattern, + showAllIndices, + isRollupIndex, +}: { + http: HttpStart; + pattern: string; + showAllIndices: boolean; + isRollupIndex: (indexName: string) => boolean; +}) => + http + .get(`/internal/index-pattern-management/resolve_index/${pattern}`, { + query: showAllIndices ? { expand_wildcards: 'all' } : undefined, + }) + .then((response) => { + if (!response) { + return []; + } else { + return responseToItemArray(response, getIndexTags(isRollupIndex)); + } + }); + +/** + * Takes two MatchedItem[]s and returns a merged set, with the second set prrioritized over the first based on name + * + * @param matchedA + * @param matchedB + */ + +export const dedupeMatchedItems = (matchedA: MatchedItem[], matchedB: MatchedItem[]) => { + const mergedMatchedItems = matchedA.reduce((col, item) => { + col[item.name] = item; + return col; + }, {} as Record); + + matchedB.reduce((col, item) => { + col[item.name] = item; + return col; + }, mergedMatchedItems); + + return Object.values(mergedMatchedItems).sort((a, b) => { + if (a.name > b.name) return 1; + if (b.name > a.name) return -1; + + return 0; + }); +}; + +export async function getIndices({ + http, + pattern: rawPattern = '', + showAllIndices = false, + searchClient, + isRollupIndex, +}: { + http: HttpStart; + pattern: string; + showAllIndices?: boolean; + searchClient: DataPublicPluginStart['search']['search']; + isRollupIndex: (indexName: string) => boolean; +}): Promise { const pattern = rawPattern.trim(); + const isCCS = pattern.indexOf(':') !== -1; + const requests: Array> = []; // Searching for `*:` fails for CCS environments. The search request // is worthless anyways as the we should only send a request @@ -67,20 +199,32 @@ export async function getIndices( return []; } - const query = showAllIndices ? { expand_wildcards: 'all' } : undefined; + const promiseResolve = getIndicesViaResolve({ + http, + pattern, + showAllIndices, + isRollupIndex, + }).catch(() => []); + requests.push(promiseResolve); - try { - const response = await http.get( - `/internal/index-pattern-management/resolve_index/${pattern}`, - { query } - ); - if (!response) { - return []; - } + if (isCCS) { + // CCS supports ±1 major version. We won't be able to expect resolve endpoint to exist until v9 + const promiseSearch = getIndicesViaSearch({ + pattern, + searchClient, + showAllIndices, + isRollupIndex, + }).catch(() => []); + requests.push(promiseSearch); + } - return responseToItemArray(response, getIndexTags(isRollupIndex)); - } catch { - return []; + const responses = await Promise.all(requests); + + if (responses.length === 2) { + const [resolveResponse, searchResponse] = responses; + return dedupeMatchedItems(searchResponse, resolveResponse); + } else { + return responses[0]; } } diff --git a/src/plugins/index_pattern_editor/public/open_editor.tsx b/src/plugins/index_pattern_editor/public/open_editor.tsx index ec62a1d6ec7c6..afeaff11f7403 100644 --- a/src/plugins/index_pattern_editor/public/open_editor.tsx +++ b/src/plugins/index_pattern_editor/public/open_editor.tsx @@ -23,9 +23,10 @@ import { IndexPatternEditorLazy } from './components/index_pattern_editor_lazy'; interface Dependencies { core: CoreStart; indexPatternService: DataPublicPluginStart['indexPatterns']; + searchClient: DataPublicPluginStart['search']['search']; } -export const getEditorOpener = ({ core, indexPatternService }: Dependencies) => ( +export const getEditorOpener = ({ core, indexPatternService, searchClient }: Dependencies) => ( options: IndexPatternEditorProps ): CloseEditor => { const { uiSettings, overlays, docLinks, notifications, http, application } = core; @@ -38,6 +39,7 @@ export const getEditorOpener = ({ core, indexPatternService }: Dependencies) => notifications, application, indexPatternService, + searchClient, }); let overlayRef: OverlayRef | null = null; diff --git a/src/plugins/index_pattern_editor/public/plugin.tsx b/src/plugins/index_pattern_editor/public/plugin.tsx index ca72249496e77..246386c5800e4 100644 --- a/src/plugins/index_pattern_editor/public/plugin.tsx +++ b/src/plugins/index_pattern_editor/public/plugin.tsx @@ -38,6 +38,7 @@ export class IndexPatternEditorPlugin openEditor: getEditorOpener({ core, indexPatternService: data.indexPatterns, + searchClient: data.search.search, }), /** * Index pattern editor flyout via react component @@ -53,6 +54,7 @@ export class IndexPatternEditorPlugin notifications, application, indexPatternService: data.indexPatterns, + searchClient: data.search.search, }} {...props} /> diff --git a/src/plugins/index_pattern_editor/public/types.ts b/src/plugins/index_pattern_editor/public/types.ts index 2a2abe249b330..8cc1779a804ba 100644 --- a/src/plugins/index_pattern_editor/public/types.ts +++ b/src/plugins/index_pattern_editor/public/types.ts @@ -27,6 +27,7 @@ export interface IndexPatternEditorContext { notifications: NotificationsStart; application: ApplicationStart; indexPatternService: DataPublicPluginStart['indexPatterns']; + searchClient: DataPublicPluginStart['search']['search']; } /** @public */