[data views] Allow data views created on hidden and system indices

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -124,6 +124,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
         timeFieldName: formData.timestampField?.value,
         id: formData.id,
         name: formData.name,
+        allowHidden: formData.allowHidden,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {

src/plugins/data_views/common/data_views/data_view.ts

@@ -152,6 +152,8 @@ export class DataView implements DataViewBase {
    */
   public matchedIndices: string[] = [];
 
+  private allowHidden: boolean = false;
+
   /**
    * constructor
    * @param config - config data and dependencies
@@ -187,6 +189,7 @@ export class DataView implements DataViewBase {
     this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {};
     this.namespaces = spec.namespaces || [];
     this.name = spec.name || '';
+    this.allowHidden = spec.allowHidden || false;
   }
 
   /**
@@ -201,6 +204,8 @@ export class DataView implements DataViewBase {
 
   getIndexPattern = () => this.title;
 
+  getAllowHidden = () => this.allowHidden;
+
   /**
    * Set index pattern
    * @param string index pattern string

src/plugins/data_views/common/data_views/data_views.ts

@@ -584,6 +584,7 @@ export class DataViewsService {
       allowNoIndex: true,
       pattern: dataView.getIndexPattern(),
       metaFields,
+      allowHidden: dataView.getAllowHidden(),
     });
   };
 
@@ -596,6 +597,7 @@ export class DataViewsService {
       rollupIndex: options.rollupIndex,
       allowNoIndex: true,
       indexFilter: options.indexFilter,
+      allowHidden: options.allowHidden,
     });
   };
 
@@ -747,6 +749,7 @@ export class DataViewsService {
         fieldAttrs,
         allowNoIndex,
         name,
+        allowHidden,
       },
     } = savedObject;
 
@@ -774,6 +777,7 @@ export class DataViewsService {
       allowNoIndex,
       runtimeFieldMap: parsedRuntimeFieldMap,
       name,
+      allowHidden,
     };
   };
 

src/plugins/data_views/common/types.ts

@@ -157,6 +157,10 @@ export interface DataViewAttributes {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Allow hidden and system indices when loading field list
+   */
+  allowHidden?: boolean;
 }
 
 /**
@@ -309,6 +313,7 @@ export interface GetFieldsOptions {
   indexFilter?: QueryDslQueryContainer;
   includeUnmapped?: boolean;
   fields?: string[];
+  allowHidden?: boolean;
 }
 
 /**
@@ -512,6 +517,10 @@ export type DataViewSpec = {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Whether the data view is hidden from the user
+   */
+  allowHidden?: boolean;
 };
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -69,12 +69,23 @@ export class IndexPatternsFetcher {
     rollupIndex?: string;
     indexFilter?: QueryDslQueryContainer;
     fields?: string[];
+    allowHidden?: boolean;
   }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
-    const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options;
+    const {
+      pattern,
+      metaFields = [],
+      fieldCapsOptions,
+      type,
+      rollupIndex,
+      indexFilter,
+      allowHidden,
+    } = options;
     const allowNoIndices = fieldCapsOptions
       ? fieldCapsOptions.allow_no_indices
       : this.allowNoIndices;
 
+    const expandWildcards = allowHidden ? 'all' : 'open';
+
     const fieldCapsResponse = await getFieldCapabilities({
       callCluster: this.elasticsearchClient,
       indices: pattern,
@@ -85,6 +96,7 @@ export class IndexPatternsFetcher {
       },
       indexFilter,
       fields: options.fields || ['*'],
+      expandWildcards,
     });
 
     if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {

src/plugins/data_views/server/fetcher/lib/es_api.ts

@@ -7,6 +7,7 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '../../../common/types';
 import { convertEsError } from './errors';
 
@@ -45,6 +46,7 @@ interface FieldCapsApiParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
+  expandWildcard?: ExpandWildcard;
 }
 
 /**

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js

@@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   const fillUndefinedParams = (args) => ({
     callCluster: undefined,
     indices: undefined,
+    expandWildcard: undefined,
     fieldCapsOptions: undefined,
     indexFilter: undefined,
     fields: undefined,

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -9,6 +9,7 @@
 import { defaults, keyBy, sortBy } from 'lodash';
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { callFieldCapsApi } from '../es_api';
 import { readFieldCapsResponse } from './field_caps_response';
 import { mergeOverrides } from './overrides';
@@ -22,6 +23,7 @@ interface FieldCapabilitiesParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
+  expandWildcards?: ExpandWildcard;
 }
 
 /**
@@ -42,13 +44,15 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     indexFilter,
     metaFields = [],
     fields,
+    expandWildcards,
   } = params;
   const esFieldCaps = await callFieldCapsApi({
     callCluster,
     indices,
     fieldCapsOptions,
     indexFilter,
     fields,
+    expandWildcard: expandWildcards,
   });
   const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
   const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -50,6 +50,7 @@ interface IQuery {
   allow_no_index?: boolean;
   include_unmapped?: boolean;
   fields?: string[];
+  allow_hidden?: boolean;
 }
 
 const querySchema = schema.object({
@@ -62,6 +63,7 @@ const querySchema = schema.object({
   allow_no_index: schema.maybe(schema.boolean()),
   include_unmapped: schema.maybe(schema.boolean()),
   fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
+  allow_hidden: schema.maybe(schema.boolean()),
 });
 
 const fieldSubTypeSchema = schema.object({
@@ -122,6 +124,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       rollup_index: rollupIndex,
       allow_no_index: allowNoIndex,
       include_unmapped: includeUnmapped,
+      allow_hidden: allowHidden,
     } = request.query;
 
     // not available to get request
@@ -147,6 +150,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           includeUnmapped,
         },
         indexFilter,
+        allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

test/functional/apps/management/data_views/_data_view_create_delete.ts

@@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const testSubjects = getService('testSubjects');
   const find = getService('find');
+  const es = getService('es');
   const PageObjects = getPageObjects(['settings', 'common', 'header']);
 
   describe('creating and deleting default data view', function describeIndexTests() {
@@ -250,5 +251,32 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         });
       });
     });
+
+    describe('hidden index support', () => {
+      it('can create data view against hidden index', async () => {
+        const pattern = 'logstash-2015.09.21';
+
+        await es.transport.request({
+          path: '/logstash-2015.09.21/_settings',
+          method: 'PUT',
+          body: {
+            index: {
+              hidden: true,
+            },
+          },
+        });
+
+        await PageObjects.settings.createIndexPattern(
+          pattern,
+          undefined,
+          undefined,
+          undefined,
+          undefined,
+          true
+        );
+        const patternName = await PageObjects.settings.getIndexPageHeading();
+        expect(patternName).to.be(pattern);
+      });
+    });
   });
 }

test/functional/page_objects/settings_page.ts

@@ -470,13 +470,19 @@ export class SettingsPageObject extends FtrService {
     await customDataViewIdInput.type(value);
   }
 
+  async allowHiddenClick() {
+    await this.testSubjects.click('toggleAdvancedSetting');
+    await this.testSubjects.click('allowHiddenField');
+  }
+
   async createIndexPattern(
     indexPatternName: string,
     // null to bypass default value
     timefield: string | null = '@timestamp',
     isStandardIndexPattern = true,
     customDataViewId?: string,
-    dataViewName?: string
+    dataViewName?: string,
+    allowHidden?: boolean
   ) {
     await this.retry.try(async () => {
       await this.header.waitUntilLoadingHasFinished();
@@ -489,6 +495,11 @@ export class SettingsPageObject extends FtrService {
       } else {
         await this.clickAddNewIndexPatternButton();
       }
+
+      if (allowHidden) {
+        await this.allowHiddenClick();
+      }
+
       await this.header.waitUntilLoadingHasFinished();
       if (!isStandardIndexPattern) {
         await this.selectRollupIndexPatternType();

x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts

@@ -249,6 +249,7 @@ describe('LogViewsClient class', () => {
           },
         ],
         "dataViewReference": DataView {
+          "allowHidden": false,
           "allowNoIndex": false,
           "deleteFieldFormat": [Function],
           "fieldAttrs": Object {},
@@ -273,6 +274,7 @@ describe('LogViewsClient class', () => {
           },
           "fields": FldList [],
           "flattenHit": [Function],
+          "getAllowHidden": [Function],
           "getFieldAttrs": [Function],
           "getIndexPattern": [Function],
           "getName": [Function],