diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md
index e4ac35f19e959..93dfdeb056f15 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iserrorresponse.md
@@ -7,5 +7,5 @@
Signature:
```typescript
-isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean | undefined
+isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean
```
diff --git a/src/plugins/data/common/search/utils.ts b/src/plugins/data/common/search/utils.ts
index e11957c6fa9fc..ea5ac28852d6a 100644
--- a/src/plugins/data/common/search/utils.ts
+++ b/src/plugins/data/common/search/utils.ts
@@ -12,7 +12,7 @@ import type { IKibanaSearchResponse } from './types';
* @returns true if response had an error while executing in ES
*/
export const isErrorResponse = (response?: IKibanaSearchResponse) => {
- return !response || !response.rawResponse || (!response.isRunning && response.isPartial);
+ return !response || !response.rawResponse || (!response.isRunning && !!response.isPartial);
};
/**
diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
index 995a1e7a908c5..7ce029860c9f7 100644
--- a/src/plugins/data/public/public.api.md
+++ b/src/plugins/data/public/public.api.md
@@ -1559,7 +1559,7 @@ export interface ISearchStartSearchSource {
// Warning: (ae-missing-release-tag) "isErrorResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
-export const isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean | undefined;
+export const isErrorResponse: (response?: IKibanaSearchResponse | undefined) => boolean;
// Warning: (ae-missing-release-tag) "isEsError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
diff --git a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx
index 3b06fa1cff298..80224dbfb673f 100644
--- a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx
+++ b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_prompts.tsx
@@ -36,7 +36,7 @@ export const EmptyPrompts: FC = ({
loadSources,
}) => {
const {
- services: { docLinks, application, http },
+ services: { docLinks, application, http, searchClient },
} = useKibana();
const [remoteClustersExist, setRemoteClustersExist] = useState(false);
@@ -47,7 +47,13 @@ export const EmptyPrompts: FC = ({
useCallback(() => {
let isMounted = true;
if (!hasDataIndices)
- getIndices(http, () => false, '*:*', false).then((dataSources) => {
+ getIndices({
+ http,
+ isRollupIndex: () => false,
+ pattern: '*:*',
+ showAllIndices: false,
+ searchClient,
+ }).then((dataSources) => {
if (isMounted) {
setRemoteClustersExist(!!dataSources.filter(removeAliases).length);
}
@@ -55,7 +61,7 @@ export const EmptyPrompts: FC = ({
return () => {
isMounted = false;
};
- }, [http, hasDataIndices]);
+ }, [http, hasDataIndices, searchClient]);
if (!hasExistingIndexPatterns && !goToForm) {
if (!hasDataIndices && !remoteClustersExist) {
diff --git a/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx b/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx
index cabff9bfb009b..4f6f7708d90c0 100644
--- a/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx
+++ b/src/plugins/index_pattern_editor/public/components/index_pattern_editor_flyout_content.tsx
@@ -70,7 +70,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
}: Props) => {
const isMounted = useRef(false);
const {
- services: { http, indexPatternService, uiSettings },
+ services: { http, indexPatternService, uiSettings, searchClient },
} = useKibana();
const { form } = useForm({
@@ -128,13 +128,19 @@ const IndexPatternEditorFlyoutContentComponent = ({
// load all data sources and set initial matchedIndices
const loadSources = useCallback(() => {
- getIndices(http, () => false, '*', allowHidden).then((dataSources) => {
+ getIndices({
+ http,
+ isRollupIndex: () => false,
+ pattern: '*',
+ showAllIndices: allowHidden,
+ searchClient,
+ }).then((dataSources) => {
setAllSources(dataSources);
const matchedSet = getMatchedIndices(dataSources, [], [], allowHidden);
setMatchedIndices(matchedSet);
setIsLoadingSources(false);
});
- }, [http, allowHidden]);
+ }, [http, allowHidden, searchClient]);
// loading list of index patterns
useEffect(() => {
@@ -223,13 +229,31 @@ const IndexPatternEditorFlyoutContentComponent = ({
const indexRequests = [];
if (query?.endsWith('*')) {
- const exactMatchedQuery = getIndices(http, isRollupIndex, query, allowHidden);
+ const exactMatchedQuery = getIndices({
+ http,
+ isRollupIndex,
+ pattern: query,
+ showAllIndices: allowHidden,
+ searchClient,
+ });
indexRequests.push(exactMatchedQuery);
// provide default value when not making a request for the partialMatchQuery
indexRequests.push(Promise.resolve([]));
} else {
- const exactMatchQuery = getIndices(http, isRollupIndex, query, allowHidden);
- const partialMatchQuery = getIndices(http, isRollupIndex, `${query}*`, allowHidden);
+ const exactMatchQuery = getIndices({
+ http,
+ isRollupIndex,
+ pattern: query,
+ showAllIndices: allowHidden,
+ searchClient,
+ });
+ const partialMatchQuery = getIndices({
+ http,
+ isRollupIndex,
+ pattern: `${query}*`,
+ showAllIndices: allowHidden,
+ searchClient,
+ });
indexRequests.push(exactMatchQuery);
indexRequests.push(partialMatchQuery);
@@ -264,7 +288,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
return fetchIndices(newTitle);
},
- [http, allowHidden, allSources, type, rollupIndicesCapabilities]
+ [http, allowHidden, allSources, type, rollupIndicesCapabilities, searchClient]
);
useEffect(() => {
diff --git a/src/plugins/index_pattern_editor/public/constants.ts b/src/plugins/index_pattern_editor/public/constants.ts
index ff74e0827fa50..8d325184353df 100644
--- a/src/plugins/index_pattern_editor/public/constants.ts
+++ b/src/plugins/index_pattern_editor/public/constants.ts
@@ -9,3 +9,10 @@
export const pluginName = 'index_pattern_editor';
export const MAX_NUMBER_OF_MATCHING_INDICES = 100;
export const CONFIG_ROLLUPS = 'rollups:enableIndexPatterns';
+
+// This isn't ideal. We want to avoid searching for 20 indices
+// then filtering out the majority of them because they are system indices.
+// We'd like to filter system indices out in the query
+// so if we can accomplish that in the future, this logic can go away
+export const ESTIMATED_NUMBER_OF_SYSTEM_INDICES = 100;
+export const MAX_SEARCH_SIZE = MAX_NUMBER_OF_MATCHING_INDICES + ESTIMATED_NUMBER_OF_SYSTEM_INDICES;
diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts
index fc96482f0379f..d65cd27e090bb 100644
--- a/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts
+++ b/src/plugins/index_pattern_editor/public/lib/get_indices.test.ts
@@ -6,11 +6,17 @@
* Side Public License, v 1.
*/
-import { getIndices, responseToItemArray } from './get_indices';
+import {
+ getIndices,
+ getIndicesViaSearch,
+ responseToItemArray,
+ dedupeMatchedItems,
+} from './get_indices';
import { httpServiceMock } from '../../../../core/public/mocks';
-import { ResolveIndexResponseItemIndexAttrs } from '../types';
+import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types';
+import { Observable } from 'rxjs';
-export const successfulResponse = {
+export const successfulResolveResponse = {
indices: [
{
name: 'remoteCluster1:bar-01',
@@ -32,28 +38,99 @@ export const successfulResponse = {
],
};
-const mockGetTags = () => [];
-const mockIsRollupIndex = () => false;
+const successfulSearchResponse = {
+ isPartial: false,
+ isRunning: false,
+ rawResponse: {
+ aggregations: {
+ indices: {
+ buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }],
+ },
+ },
+ },
+};
+
+const partialSearchResponse = {
+ isPartial: true,
+ isRunning: true,
+ rawResponse: {
+ hits: {
+ total: 2,
+ hits: [],
+ },
+ },
+};
+
+const errorSearchResponse = {
+ isPartial: true,
+ isRunning: false,
+};
+
+const isRollupIndex = () => false;
+const getTags = () => [];
+const searchClient = () =>
+ new Observable((observer) => {
+ observer.next(successfulSearchResponse);
+ observer.complete();
+ }) as any;
const http = httpServiceMock.createStartContract();
-http.get.mockResolvedValue(successfulResponse);
+http.get.mockResolvedValue(successfulResolveResponse);
describe('getIndices', () => {
it('should work in a basic case', async () => {
- const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+ const uncalledSearchClient = jest.fn();
+ const result = await getIndices({
+ http,
+ pattern: 'kibana',
+ searchClient: uncalledSearchClient,
+ isRollupIndex,
+ });
+ expect(http.get).toHaveBeenCalled();
+ expect(uncalledSearchClient).not.toHaveBeenCalled();
expect(result.length).toBe(3);
expect(result[0].name).toBe('f-alias');
expect(result[1].name).toBe('foo');
});
+ it('should make two calls in cross cluser case', async () => {
+ http.get.mockResolvedValue(successfulResolveResponse);
+ const result = await getIndices({ http, pattern: '*:kibana', searchClient, isRollupIndex });
+
+ expect(http.get).toHaveBeenCalled();
+ expect(result.length).toBe(4);
+ expect(result[0].name).toBe('f-alias');
+ expect(result[1].name).toBe('foo');
+ expect(result[2].name).toBe('kibana_sample_data_ecommerce');
+ expect(result[3].name).toBe('remoteCluster1:bar-01');
+ });
+
it('should ignore ccs query-all', async () => {
- expect((await getIndices(http, mockIsRollupIndex, '*:', false)).length).toBe(0);
+ expect((await getIndices({ http, pattern: '*:', searchClient, isRollupIndex })).length).toBe(0);
});
it('should ignore a single comma', async () => {
- expect((await getIndices(http, mockIsRollupIndex, ',', false)).length).toBe(0);
- expect((await getIndices(http, mockIsRollupIndex, ',*', false)).length).toBe(0);
- expect((await getIndices(http, mockIsRollupIndex, ',foobar', false)).length).toBe(0);
+ expect((await getIndices({ http, pattern: ',', searchClient, isRollupIndex })).length).toBe(0);
+ expect((await getIndices({ http, pattern: ',*', searchClient, isRollupIndex })).length).toBe(0);
+ expect(
+ (await getIndices({ http, pattern: ',foobar', searchClient, isRollupIndex })).length
+ ).toBe(0);
+ });
+
+ it('should work with partial responses', async () => {
+ const searchClientPartialResponse = () =>
+ new Observable((observer) => {
+ observer.next(partialSearchResponse);
+ observer.next(successfulSearchResponse);
+ observer.complete();
+ }) as any;
+ const result = await getIndices({
+ http,
+ pattern: '*:kibana',
+ searchClient: searchClientPartialResponse,
+ isRollupIndex,
+ });
+ expect(result.length).toBe(4);
});
it('response object to item array', () => {
@@ -81,16 +158,37 @@ describe('getIndices', () => {
},
],
};
- expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot();
- expect(responseToItemArray({}, mockGetTags)).toEqual([]);
+ expect(responseToItemArray(result, getTags)).toMatchSnapshot();
+ expect(responseToItemArray({}, getTags)).toEqual([]);
+ });
+
+ it('matched items are deduped', () => {
+ const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[];
+ const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[];
+ expect(dedupeMatchedItems(setA, setB)).toHaveLength(3);
});
describe('errors', () => {
- it('should handle errors gracefully', async () => {
+ it('should handle thrown errors gracefully', async () => {
http.get.mockImplementationOnce(() => {
throw new Error('Test error');
});
- const result = await getIndices(http, mockIsRollupIndex, 'kibana', false);
+ const result = await getIndices({ http, pattern: 'kibana', searchClient, isRollupIndex });
+ expect(result.length).toBe(0);
+ });
+
+ it('getIndicesViaSearch should handle error responses gracefully', async () => {
+ const searchClientErrorResponse = () =>
+ new Observable((observer) => {
+ observer.next(errorSearchResponse);
+ observer.complete();
+ }) as any;
+ const result = await getIndicesViaSearch({
+ pattern: '*:kibana',
+ searchClient: searchClientErrorResponse,
+ showAllIndices: false,
+ isRollupIndex,
+ });
expect(result.length).toBe(0);
});
});
diff --git a/src/plugins/index_pattern_editor/public/lib/get_indices.ts b/src/plugins/index_pattern_editor/public/lib/get_indices.ts
index 625e99ecbcdc5..8d642174232ac 100644
--- a/src/plugins/index_pattern_editor/public/lib/get_indices.ts
+++ b/src/plugins/index_pattern_editor/public/lib/get_indices.ts
@@ -8,10 +8,18 @@
import { sortBy } from 'lodash';
import { HttpStart } from 'kibana/public';
+import { map, filter } from 'rxjs/operators';
import { i18n } from '@kbn/i18n';
import { Tag, INDEX_PATTERN_TYPE } from '../types';
-// todo move into this plugin, consider removing all ipm references
import { MatchedItem, ResolveIndexResponse, ResolveIndexResponseItemIndexAttrs } from '../types';
+import { MAX_SEARCH_SIZE } from '../constants';
+
+import {
+ DataPublicPluginStart,
+ IEsSearchResponse,
+ isErrorResponse,
+ isCompleteResponse,
+} from '../../../data/public';
const aliasLabel = i18n.translate('indexPatternEditor.aliasLabel', { defaultMessage: 'Alias' });
const dataStreamLabel = i18n.translate('indexPatternEditor.dataStreamLabel', {
@@ -41,13 +49,137 @@ const getIndexTags = (isRollupIndex: (indexName: string) => boolean) => (indexNa
]
: [];
-export async function getIndices(
- http: HttpStart,
- isRollupIndex: (indexName: string) => boolean,
- rawPattern: string,
+export const searchResponseToArray = (
+ getTags: (indexName: string) => Tag[],
showAllIndices: boolean
-): Promise {
+) => (response: IEsSearchResponse) => {
+ const { rawResponse } = response;
+ if (!rawResponse.aggregations) {
+ return [];
+ } else {
+ // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response
+ return rawResponse.aggregations.indices.buckets
+ .map((bucket: { key: string }) => {
+ return bucket.key;
+ })
+ .filter((indexName: string) => {
+ if (showAllIndices) {
+ return true;
+ } else {
+ return !indexName.startsWith('.');
+ }
+ })
+ .map((indexName: string) => {
+ return {
+ name: indexName,
+ tags: getTags(indexName),
+ item: {},
+ };
+ });
+ }
+};
+
+export const getIndicesViaSearch = async ({
+ pattern,
+ searchClient,
+ showAllIndices,
+ isRollupIndex,
+}: {
+ pattern: string;
+ searchClient: DataPublicPluginStart['search']['search'];
+ showAllIndices: boolean;
+ isRollupIndex: (indexName: string) => boolean;
+}): Promise =>
+ searchClient({
+ params: {
+ ignoreUnavailable: true,
+ expand_wildcards: showAllIndices ? 'all' : 'open',
+ index: pattern,
+ body: {
+ size: 0, // no hits
+ aggs: {
+ indices: {
+ terms: {
+ field: '_index',
+ size: MAX_SEARCH_SIZE,
+ },
+ },
+ },
+ },
+ },
+ })
+ .pipe(
+ filter((resp) => isCompleteResponse(resp) || isErrorResponse(resp)),
+ map(searchResponseToArray(getIndexTags(isRollupIndex), showAllIndices))
+ )
+ .toPromise()
+ .catch(() => []);
+
+export const getIndicesViaResolve = async ({
+ http,
+ pattern,
+ showAllIndices,
+ isRollupIndex,
+}: {
+ http: HttpStart;
+ pattern: string;
+ showAllIndices: boolean;
+ isRollupIndex: (indexName: string) => boolean;
+}) =>
+ http
+ .get(`/internal/index-pattern-management/resolve_index/${pattern}`, {
+ query: showAllIndices ? { expand_wildcards: 'all' } : undefined,
+ })
+ .then((response) => {
+ if (!response) {
+ return [];
+ } else {
+ return responseToItemArray(response, getIndexTags(isRollupIndex));
+ }
+ });
+
+/**
+ * Takes two MatchedItem[]s and returns a merged set, with the second set prrioritized over the first based on name
+ *
+ * @param matchedA
+ * @param matchedB
+ */
+
+export const dedupeMatchedItems = (matchedA: MatchedItem[], matchedB: MatchedItem[]) => {
+ const mergedMatchedItems = matchedA.reduce((col, item) => {
+ col[item.name] = item;
+ return col;
+ }, {} as Record);
+
+ matchedB.reduce((col, item) => {
+ col[item.name] = item;
+ return col;
+ }, mergedMatchedItems);
+
+ return Object.values(mergedMatchedItems).sort((a, b) => {
+ if (a.name > b.name) return 1;
+ if (b.name > a.name) return -1;
+
+ return 0;
+ });
+};
+
+export async function getIndices({
+ http,
+ pattern: rawPattern = '',
+ showAllIndices = false,
+ searchClient,
+ isRollupIndex,
+}: {
+ http: HttpStart;
+ pattern: string;
+ showAllIndices?: boolean;
+ searchClient: DataPublicPluginStart['search']['search'];
+ isRollupIndex: (indexName: string) => boolean;
+}): Promise {
const pattern = rawPattern.trim();
+ const isCCS = pattern.indexOf(':') !== -1;
+ const requests: Array> = [];
// Searching for `*:` fails for CCS environments. The search request
// is worthless anyways as the we should only send a request
@@ -67,20 +199,32 @@ export async function getIndices(
return [];
}
- const query = showAllIndices ? { expand_wildcards: 'all' } : undefined;
+ const promiseResolve = getIndicesViaResolve({
+ http,
+ pattern,
+ showAllIndices,
+ isRollupIndex,
+ }).catch(() => []);
+ requests.push(promiseResolve);
- try {
- const response = await http.get(
- `/internal/index-pattern-management/resolve_index/${pattern}`,
- { query }
- );
- if (!response) {
- return [];
- }
+ if (isCCS) {
+ // CCS supports ±1 major version. We won't be able to expect resolve endpoint to exist until v9
+ const promiseSearch = getIndicesViaSearch({
+ pattern,
+ searchClient,
+ showAllIndices,
+ isRollupIndex,
+ }).catch(() => []);
+ requests.push(promiseSearch);
+ }
- return responseToItemArray(response, getIndexTags(isRollupIndex));
- } catch {
- return [];
+ const responses = await Promise.all(requests);
+
+ if (responses.length === 2) {
+ const [resolveResponse, searchResponse] = responses;
+ return dedupeMatchedItems(searchResponse, resolveResponse);
+ } else {
+ return responses[0];
}
}
diff --git a/src/plugins/index_pattern_editor/public/open_editor.tsx b/src/plugins/index_pattern_editor/public/open_editor.tsx
index ec62a1d6ec7c6..afeaff11f7403 100644
--- a/src/plugins/index_pattern_editor/public/open_editor.tsx
+++ b/src/plugins/index_pattern_editor/public/open_editor.tsx
@@ -23,9 +23,10 @@ import { IndexPatternEditorLazy } from './components/index_pattern_editor_lazy';
interface Dependencies {
core: CoreStart;
indexPatternService: DataPublicPluginStart['indexPatterns'];
+ searchClient: DataPublicPluginStart['search']['search'];
}
-export const getEditorOpener = ({ core, indexPatternService }: Dependencies) => (
+export const getEditorOpener = ({ core, indexPatternService, searchClient }: Dependencies) => (
options: IndexPatternEditorProps
): CloseEditor => {
const { uiSettings, overlays, docLinks, notifications, http, application } = core;
@@ -38,6 +39,7 @@ export const getEditorOpener = ({ core, indexPatternService }: Dependencies) =>
notifications,
application,
indexPatternService,
+ searchClient,
});
let overlayRef: OverlayRef | null = null;
diff --git a/src/plugins/index_pattern_editor/public/plugin.tsx b/src/plugins/index_pattern_editor/public/plugin.tsx
index ca72249496e77..246386c5800e4 100644
--- a/src/plugins/index_pattern_editor/public/plugin.tsx
+++ b/src/plugins/index_pattern_editor/public/plugin.tsx
@@ -38,6 +38,7 @@ export class IndexPatternEditorPlugin
openEditor: getEditorOpener({
core,
indexPatternService: data.indexPatterns,
+ searchClient: data.search.search,
}),
/**
* Index pattern editor flyout via react component
@@ -53,6 +54,7 @@ export class IndexPatternEditorPlugin
notifications,
application,
indexPatternService: data.indexPatterns,
+ searchClient: data.search.search,
}}
{...props}
/>
diff --git a/src/plugins/index_pattern_editor/public/types.ts b/src/plugins/index_pattern_editor/public/types.ts
index 2a2abe249b330..8cc1779a804ba 100644
--- a/src/plugins/index_pattern_editor/public/types.ts
+++ b/src/plugins/index_pattern_editor/public/types.ts
@@ -27,6 +27,7 @@ export interface IndexPatternEditorContext {
notifications: NotificationsStart;
application: ApplicationStart;
indexPatternService: DataPublicPluginStart['indexPatterns'];
+ searchClient: DataPublicPluginStart['search']['search'];
}
/** @public */