diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 1bbc4d08247a2..d3bf6db92d0ad 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -31,7 +31,7 @@ pageLoadAssetSize: dataViewEditor: 28082 dataViewFieldEditor: 27000 dataViewManagement: 5000 - dataViews: 47300 + dataViews: 48300 dataVisualizer: 27530 devTools: 38637 discover: 99999 diff --git a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx index ad344d482c0cf..c889f2e0dcd06 100644 --- a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx +++ b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx @@ -124,6 +124,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ timeFieldName: formData.timestampField?.value, id: formData.id, name: formData.name, + allowHidden: formData.allowHidden, }; if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) { diff --git a/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap b/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap index 0d646f4afc28d..53b77e28a416c 100644 --- a/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap +++ b/src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap @@ -28,6 +28,7 @@ exports[`IndexPatterns delete will throw if insufficient access 1`] = `[DataView exports[`IndexPatterns savedObjectToSpec 1`] = ` Object { + "allowHidden": undefined, "allowNoIndex": undefined, "fieldAttrs": Object { "aRuntimeField": Object { diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index ffda65af2a895..3cea4505b572c 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -152,6 +152,8 @@ export class DataView implements DataViewBase { */ public matchedIndices: string[] = []; + private allowHidden: boolean = false; + /** * constructor * @param config - config data and dependencies @@ -187,6 +189,7 @@ export class DataView implements DataViewBase { this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {}; this.namespaces = spec.namespaces || []; this.name = spec.name || ''; + this.allowHidden = spec.allowHidden || false; } /** @@ -201,6 +204,8 @@ export class DataView implements DataViewBase { getIndexPattern = () => this.title; + getAllowHidden = () => this.allowHidden; + /** * Set index pattern * @param string index pattern string diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 8bac81f19ef62..d5509f03db580 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -584,6 +584,7 @@ export class DataViewsService { allowNoIndex: true, pattern: dataView.getIndexPattern(), metaFields, + allowHidden: dataView.getAllowHidden(), }); }; @@ -596,6 +597,7 @@ export class DataViewsService { rollupIndex: options.rollupIndex, allowNoIndex: true, indexFilter: options.indexFilter, + allowHidden: options.allowHidden, }); }; @@ -747,6 +749,7 @@ export class DataViewsService { fieldAttrs, allowNoIndex, name, + allowHidden, }, } = savedObject; @@ -774,6 +777,7 @@ export class DataViewsService { allowNoIndex, runtimeFieldMap: parsedRuntimeFieldMap, name, + allowHidden, }; }; diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 9061c4643dce9..522bec8873534 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -157,6 +157,10 @@ export interface DataViewAttributes { * Name of the data view. Human readable name used to differentiate data view. */ name?: string; + /** + * Allow hidden and system indices when loading field list + */ + allowHidden?: boolean; } /** @@ -309,6 +313,7 @@ export interface GetFieldsOptions { indexFilter?: QueryDslQueryContainer; includeUnmapped?: boolean; fields?: string[]; + allowHidden?: boolean; } /** @@ -512,6 +517,10 @@ export type DataViewSpec = { * Name of the data view. Human readable name used to differentiate data view. */ name?: string; + /** + * Whether the data view is hidden from the user + */ + allowHidden?: boolean; }; // eslint-disable-next-line @typescript-eslint/consistent-type-definitions diff --git a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts index b1d22dc5523c8..f23933bc5775e 100644 --- a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts @@ -69,12 +69,23 @@ export class IndexPatternsFetcher { rollupIndex?: string; indexFilter?: QueryDslQueryContainer; fields?: string[]; + allowHidden?: boolean; }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> { - const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options; + const { + pattern, + metaFields = [], + fieldCapsOptions, + type, + rollupIndex, + indexFilter, + allowHidden, + } = options; const allowNoIndices = fieldCapsOptions ? fieldCapsOptions.allow_no_indices : this.allowNoIndices; + const expandWildcards = allowHidden ? 'all' : 'open'; + const fieldCapsResponse = await getFieldCapabilities({ callCluster: this.elasticsearchClient, indices: pattern, @@ -85,6 +96,7 @@ export class IndexPatternsFetcher { }, indexFilter, fields: options.fields || ['*'], + expandWildcards, }); if (this.rollupsEnabled && type === 'rollup' && rollupIndex) { diff --git a/src/plugins/data_views/server/fetcher/lib/es_api.ts b/src/plugins/data_views/server/fetcher/lib/es_api.ts index 988e4a4ec28c8..2128e52da537b 100644 --- a/src/plugins/data_views/server/fetcher/lib/es_api.ts +++ b/src/plugins/data_views/server/fetcher/lib/es_api.ts @@ -7,6 +7,7 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; +import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { QueryDslQueryContainer } from '../../../common/types'; import { convertEsError } from './errors'; @@ -45,6 +46,7 @@ interface FieldCapsApiParams { fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean }; indexFilter?: QueryDslQueryContainer; fields?: string[]; + expandWildcard?: ExpandWildcard; } /** diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js index 1b3be374bbd7c..f41c71498e81e 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js @@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => { const fillUndefinedParams = (args) => ({ callCluster: undefined, indices: undefined, + expandWildcard: undefined, fieldCapsOptions: undefined, indexFilter: undefined, fields: undefined, diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index 73e550ebd68ce..6bef117151609 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -9,6 +9,7 @@ import { defaults, keyBy, sortBy } from 'lodash'; import { ElasticsearchClient } from '@kbn/core/server'; +import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { callFieldCapsApi } from '../es_api'; import { readFieldCapsResponse } from './field_caps_response'; import { mergeOverrides } from './overrides'; @@ -22,6 +23,7 @@ interface FieldCapabilitiesParams { fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean }; indexFilter?: QueryDslQueryContainer; fields?: string[]; + expandWildcards?: ExpandWildcard; } /** @@ -42,6 +44,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { indexFilter, metaFields = [], fields, + expandWildcards, } = params; const esFieldCaps = await callFieldCapsApi({ callCluster, @@ -49,6 +52,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { fieldCapsOptions, indexFilter, fields, + expandWildcard: expandWildcards, }); const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body); const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name'); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 15d761935c0a7..7d39b41d5caee 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -50,6 +50,7 @@ interface IQuery { allow_no_index?: boolean; include_unmapped?: boolean; fields?: string[]; + allow_hidden?: boolean; } const querySchema = schema.object({ @@ -62,6 +63,7 @@ const querySchema = schema.object({ allow_no_index: schema.maybe(schema.boolean()), include_unmapped: schema.maybe(schema.boolean()), fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), + allow_hidden: schema.maybe(schema.boolean()), }); const fieldSubTypeSchema = schema.object({ @@ -122,6 +124,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I rollup_index: rollupIndex, allow_no_index: allowNoIndex, include_unmapped: includeUnmapped, + allow_hidden: allowHidden, } = request.query; // not available to get request @@ -147,6 +150,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I includeUnmapped, }, indexFilter, + allowHidden, ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), }); diff --git a/test/functional/apps/management/data_views/_data_view_create_delete.ts b/test/functional/apps/management/data_views/_data_view_create_delete.ts index edf2f000fcb27..e55afd799a9b8 100644 --- a/test/functional/apps/management/data_views/_data_view_create_delete.ts +++ b/test/functional/apps/management/data_views/_data_view_create_delete.ts @@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const testSubjects = getService('testSubjects'); const find = getService('find'); + const es = getService('es'); const PageObjects = getPageObjects(['settings', 'common', 'header']); describe('creating and deleting default data view', function describeIndexTests() { @@ -250,5 +251,32 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); }); + + describe('hidden index support', () => { + it('can create data view against hidden index', async () => { + const pattern = 'logstash-2015.09.21'; + + await es.transport.request({ + path: '/logstash-2015.09.21/_settings', + method: 'PUT', + body: { + index: { + hidden: true, + }, + }, + }); + + await PageObjects.settings.createIndexPattern( + pattern, + undefined, + undefined, + undefined, + undefined, + true + ); + const patternName = await PageObjects.settings.getIndexPageHeading(); + expect(patternName).to.be(pattern); + }); + }); }); } diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d2fba8e620153..568c45beb0dfb 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -470,13 +470,19 @@ export class SettingsPageObject extends FtrService { await customDataViewIdInput.type(value); } + async allowHiddenClick() { + await this.testSubjects.click('toggleAdvancedSetting'); + await this.testSubjects.click('allowHiddenField'); + } + async createIndexPattern( indexPatternName: string, // null to bypass default value timefield: string | null = '@timestamp', isStandardIndexPattern = true, customDataViewId?: string, - dataViewName?: string + dataViewName?: string, + allowHidden?: boolean ) { await this.retry.try(async () => { await this.header.waitUntilLoadingHasFinished(); @@ -489,6 +495,11 @@ export class SettingsPageObject extends FtrService { } else { await this.clickAddNewIndexPatternButton(); } + + if (allowHidden) { + await this.allowHiddenClick(); + } + await this.header.waitUntilLoadingHasFinished(); if (!isStandardIndexPattern) { await this.selectRollupIndexPatternType(); diff --git a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts index 5efdf9e125deb..b9f5037ced6b1 100644 --- a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts @@ -249,6 +249,7 @@ describe('LogViewsClient class', () => { }, ], "dataViewReference": DataView { + "allowHidden": false, "allowNoIndex": false, "deleteFieldFormat": [Function], "fieldAttrs": Object {}, @@ -273,6 +274,7 @@ describe('LogViewsClient class', () => { }, "fields": FldList [], "flattenHit": [Function], + "getAllowHidden": [Function], "getFieldAttrs": [Function], "getIndexPattern": [Function], "getName": [Function],