From 85d8231d137ab5ffc0ffb4f49d4e1abfa148da65 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Thu, 12 Oct 2023 23:40:54 -0500 Subject: [PATCH] [data views] Allow data views created on hidden and system indices (#168174) ## Summary Previously, the 'Allow hidden and system indices' advanced option when creating a data view was only a UI convenience. It allowed you to see which hidden and system indices you were matching but they would be would be selected just the same once the data view was loaded. At some point something changed and now there are system and hidden indices that require `expandWildcards: hidden` to be passed to field caps in order to see anything. `allowHidden: boolean` is added to the DataView and DataViewSpec and passed through when field caps requests are made. This is primarily a tool for troubleshooting. For instance, instead of hitting a full data stream across a number of data tiers you can select a specific index to compare its performance. Closes: https://github.com/elastic/kibana/issues/164652 --- packages/kbn-optimizer/limits.yml | 2 +- .../data_view_editor_flyout_content.tsx | 1 + .../__snapshots__/data_views.test.ts.snap | 1 + .../data_views/common/data_views/data_view.ts | 5 ++++ .../common/data_views/data_views.ts | 4 +++ src/plugins/data_views/common/types.ts | 9 ++++++ .../server/fetcher/index_patterns_fetcher.ts | 14 +++++++++- .../data_views/server/fetcher/lib/es_api.ts | 2 ++ .../field_capabilities.test.js | 1 + .../field_capabilities/field_capabilities.ts | 4 +++ .../rest_api_routes/internal/fields_for.ts | 4 +++ .../data_views/_data_view_create_delete.ts | 28 +++++++++++++++++++ test/functional/page_objects/settings_page.ts | 13 ++++++++- .../log_views/log_views_client.test.ts | 2 ++ 14 files changed, 87 insertions(+), 3 deletions(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 1bbc4d08247a2..d3bf6db92d0ad 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -31,7 +31,7 @@ pageLoadAssetSize: dataViewEditor: 28082 dataViewFieldEditor: 27000 dataViewManagement: 5000 - dataViews: 47300 + dataViews: 48300 dataVisualizer: 27530 devTools: 38637 discover: 99999 diff --git a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx index ad344d482c0cf..c889f2e0dcd06 100644 --- a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx +++ b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx @@ -124,6 +124,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ timeFieldName: formData.timestampField?.value, id: formData.id, name: formData.name, + allowHidden: formData.allowHidden, }; if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) { diff --git a/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap b/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap index 0d646f4afc28d..53b77e28a416c 100644 --- a/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap +++ b/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap @@ -28,6 +28,7 @@ exports[`IndexPatterns delete will throw if insufficient access 1`] = `[DataView exports[`IndexPatterns savedObjectToSpec 1`] = ` Object { + "allowHidden": undefined, "allowNoIndex": undefined, "fieldAttrs": Object { "aRuntimeField": Object { diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index ffda65af2a895..3cea4505b572c 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -152,6 +152,8 @@ export class DataView implements DataViewBase { */ public matchedIndices: string[] = []; + private allowHidden: boolean = false; + /** * constructor * @param config - config data and dependencies @@ -187,6 +189,7 @@ export class DataView implements DataViewBase { this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {}; this.namespaces = spec.namespaces || []; this.name = spec.name || ''; + this.allowHidden = spec.allowHidden || false; } /** @@ -201,6 +204,8 @@ export class DataView implements DataViewBase { getIndexPattern = () => this.title; + getAllowHidden = () => this.allowHidden; + /** * Set index pattern * @param string index pattern string diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 8bac81f19ef62..d5509f03db580 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -584,6 +584,7 @@ export class DataViewsService { allowNoIndex: true, pattern: dataView.getIndexPattern(), metaFields, + allowHidden: dataView.getAllowHidden(), }); }; @@ -596,6 +597,7 @@ export class DataViewsService { rollupIndex: options.rollupIndex, allowNoIndex: true, indexFilter: options.indexFilter, + allowHidden: options.allowHidden, }); }; @@ -747,6 +749,7 @@ export class DataViewsService { fieldAttrs, allowNoIndex, name, + allowHidden, }, } = savedObject; @@ -774,6 +777,7 @@ export class DataViewsService { allowNoIndex, runtimeFieldMap: parsedRuntimeFieldMap, name, + allowHidden, }; }; diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 9061c4643dce9..522bec8873534 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -157,6 +157,10 @@ export interface DataViewAttributes { * Name of the data view. Human readable name used to differentiate data view. */ name?: string; + /** + * Allow hidden and system indices when loading field list + */ + allowHidden?: boolean; } /** @@ -309,6 +313,7 @@ export interface GetFieldsOptions { indexFilter?: QueryDslQueryContainer; includeUnmapped?: boolean; fields?: string[]; + allowHidden?: boolean; } /** @@ -512,6 +517,10 @@ export type DataViewSpec = { * Name of the data view. Human readable name used to differentiate data view. */ name?: string; + /** + * Whether the data view is hidden from the user + */ + allowHidden?: boolean; }; // eslint-disable-next-line @typescript-eslint/consistent-type-definitions diff --git a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts index b1d22dc5523c8..f23933bc5775e 100644 --- a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts @@ -69,12 +69,23 @@ export class IndexPatternsFetcher { rollupIndex?: string; indexFilter?: QueryDslQueryContainer; fields?: string[]; + allowHidden?: boolean; }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> { - const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options; + const { + pattern, + metaFields = [], + fieldCapsOptions, + type, + rollupIndex, + indexFilter, + allowHidden, + } = options; const allowNoIndices = fieldCapsOptions ? fieldCapsOptions.allow_no_indices : this.allowNoIndices; + const expandWildcards = allowHidden ? 'all' : 'open'; + const fieldCapsResponse = await getFieldCapabilities({ callCluster: this.elasticsearchClient, indices: pattern, @@ -85,6 +96,7 @@ export class IndexPatternsFetcher { }, indexFilter, fields: options.fields || ['*'], + expandWildcards, }); if (this.rollupsEnabled && type === 'rollup' && rollupIndex) { diff --git a/src/plugins/data_views/server/fetcher/lib/es_api.ts b/src/plugins/data_views/server/fetcher/lib/es_api.ts index 988e4a4ec28c8..2128e52da537b 100644 --- a/src/plugins/data_views/server/fetcher/lib/es_api.ts +++ b/src/plugins/data_views/server/fetcher/lib/es_api.ts @@ -7,6 +7,7 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; +import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { QueryDslQueryContainer } from '../../../common/types'; import { convertEsError } from './errors'; @@ -45,6 +46,7 @@ interface FieldCapsApiParams { fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean }; indexFilter?: QueryDslQueryContainer; fields?: string[]; + expandWildcard?: ExpandWildcard; } /** diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js index 1b3be374bbd7c..f41c71498e81e 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js @@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => { const fillUndefinedParams = (args) => ({ callCluster: undefined, indices: undefined, + expandWildcard: undefined, fieldCapsOptions: undefined, indexFilter: undefined, fields: undefined, diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index 73e550ebd68ce..6bef117151609 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -9,6 +9,7 @@ import { defaults, keyBy, sortBy } from 'lodash'; import { ElasticsearchClient } from '@kbn/core/server'; +import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { callFieldCapsApi } from '../es_api'; import { readFieldCapsResponse } from './field_caps_response'; import { mergeOverrides } from './overrides'; @@ -22,6 +23,7 @@ interface FieldCapabilitiesParams { fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean }; indexFilter?: QueryDslQueryContainer; fields?: string[]; + expandWildcards?: ExpandWildcard; } /** @@ -42,6 +44,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { indexFilter, metaFields = [], fields, + expandWildcards, } = params; const esFieldCaps = await callFieldCapsApi({ callCluster, @@ -49,6 +52,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { fieldCapsOptions, indexFilter, fields, + expandWildcard: expandWildcards, }); const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body); const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name'); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 15d761935c0a7..7d39b41d5caee 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -50,6 +50,7 @@ interface IQuery { allow_no_index?: boolean; include_unmapped?: boolean; fields?: string[]; + allow_hidden?: boolean; } const querySchema = schema.object({ @@ -62,6 +63,7 @@ const querySchema = schema.object({ allow_no_index: schema.maybe(schema.boolean()), include_unmapped: schema.maybe(schema.boolean()), fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), + allow_hidden: schema.maybe(schema.boolean()), }); const fieldSubTypeSchema = schema.object({ @@ -122,6 +124,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I rollup_index: rollupIndex, allow_no_index: allowNoIndex, include_unmapped: includeUnmapped, + allow_hidden: allowHidden, } = request.query; // not available to get request @@ -147,6 +150,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I includeUnmapped, }, indexFilter, + allowHidden, ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), }); diff --git a/test/functional/apps/management/data_views/_data_view_create_delete.ts b/test/functional/apps/management/data_views/_data_view_create_delete.ts index edf2f000fcb27..e55afd799a9b8 100644 --- a/test/functional/apps/management/data_views/_data_view_create_delete.ts +++ b/test/functional/apps/management/data_views/_data_view_create_delete.ts @@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const testSubjects = getService('testSubjects'); const find = getService('find'); + const es = getService('es'); const PageObjects = getPageObjects(['settings', 'common', 'header']); describe('creating and deleting default data view', function describeIndexTests() { @@ -250,5 +251,32 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); }); + + describe('hidden index support', () => { + it('can create data view against hidden index', async () => { + const pattern = 'logstash-2015.09.21'; + + await es.transport.request({ + path: '/logstash-2015.09.21/_settings', + method: 'PUT', + body: { + index: { + hidden: true, + }, + }, + }); + + await PageObjects.settings.createIndexPattern( + pattern, + undefined, + undefined, + undefined, + undefined, + true + ); + const patternName = await PageObjects.settings.getIndexPageHeading(); + expect(patternName).to.be(pattern); + }); + }); }); } diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d2fba8e620153..568c45beb0dfb 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -470,13 +470,19 @@ export class SettingsPageObject extends FtrService { await customDataViewIdInput.type(value); } + async allowHiddenClick() { + await this.testSubjects.click('toggleAdvancedSetting'); + await this.testSubjects.click('allowHiddenField'); + } + async createIndexPattern( indexPatternName: string, // null to bypass default value timefield: string | null = '@timestamp', isStandardIndexPattern = true, customDataViewId?: string, - dataViewName?: string + dataViewName?: string, + allowHidden?: boolean ) { await this.retry.try(async () => { await this.header.waitUntilLoadingHasFinished(); @@ -489,6 +495,11 @@ export class SettingsPageObject extends FtrService { } else { await this.clickAddNewIndexPatternButton(); } + + if (allowHidden) { + await this.allowHiddenClick(); + } + await this.header.waitUntilLoadingHasFinished(); if (!isStandardIndexPattern) { await this.selectRollupIndexPatternType(); diff --git a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts index 5efdf9e125deb..b9f5037ced6b1 100644 --- a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts @@ -249,6 +249,7 @@ describe('LogViewsClient class', () => { }, ], "dataViewReference": DataView { + "allowHidden": false, "allowNoIndex": false, "deleteFieldFormat": [Function], "fieldAttrs": Object {}, @@ -273,6 +274,7 @@ describe('LogViewsClient class', () => { }, "fields": FldList [], "flattenHit": [Function], + "getAllowHidden": [Function], "getFieldAttrs": [Function], "getIndexPattern": [Function], "getName": [Function],