Show _ignored field in Discover

packages/kbn-field-utils/src/utils/get_field_icon_type.ts

@@ -23,7 +23,7 @@ export function getFieldIconType<T extends FieldBase = DataViewField>(
 ): string {
   const type = getCustomFieldType ? getCustomFieldType(field) : getFieldType<T>(field);
   const esType = field.esTypes?.[0] || null;
-  if (esType && ['_id', '_index'].includes(esType) && type === 'string') {
+  if (esType && ['_id', '_index', '_ignored'].includes(esType) && type === 'string') {
     return 'keyword';
   }
   if (type === 'unknown' && esType && isKnownFieldType(esType)) {

src/plugins/data/server/ui_settings.ts

@@ -40,7 +40,7 @@ export function getUiSettings(
       name: i18n.translate('data.advancedSettings.metaFieldsTitle', {
         defaultMessage: 'Meta fields',
       }),
-      value: ['_source', '_id', '_index', '_score'],
+      value: ['_source', '_id', '_index', '_score', '_ignored'],
       description: i18n.translate('data.advancedSettings.metaFieldsText', {
         defaultMessage:
           'Fields that exist outside of _source to merge into our document when displaying it',

src/plugins/data_views/server/fetcher/lib/field_capabilities/overrides.ts

@@ -13,6 +13,7 @@ const OVERRIDES: Record<string, Partial<FieldDescriptor>> = {
   _source: { type: '_source' },
   _index: { type: 'string' },
   _type: { type: 'string' },
+  _ignored: { type: 'string' },
   _id: { type: 'string' },
   _score: {
     type: 'number',

test/examples/unified_field_list_examples/existing_fields.ts

@@ -11,7 +11,7 @@ import type { FtrProviderContext } from '../../functional/ftr_provider_context';
 
 const TEST_START_TIME = 'Jan 2, 2021 @ 00:00:00.000';
 const TEST_END_TIME = 'Jan 2, 2022 @ 00:00:00.000';
-const metaFields = ['_id', '_index', '_score'];
+const metaFields = ['_id', '_index', '_score', '_ignored'];
 
 const fieldsWithData = [
   'ts',

test/functional/apps/discover/group2_data_grid2/_data_grid_field_tokens.ts

@@ -96,6 +96,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       expect(await findFirstColumnTokens()).to.eql([]);
 
       expect(await findFirstDocViewerTokens()).to.eql([
+        'Keyword',
         'Keyword',
         'Keyword',
         'Number',
@@ -105,7 +106,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         'Text',
         'Number',
         'IP address',
-        'Text',
       ]);
     });
 
@@ -118,6 +118,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       expect(await findFirstColumnTokens()).to.eql(['Number', 'Text', 'IP address', 'Geo point']);
 
       expect(await findFirstDocViewerTokens()).to.eql([
+        'Keyword',
         'Keyword',
         'Keyword',
         'Number',
@@ -127,7 +128,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         'Text',
         'Number',
         'IP address',
-        'Text',
       ]);
     });
 
@@ -173,6 +173,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       expect(await findFirstColumnTokens()).to.eql(['Number', 'Text', 'Geo point', 'Date']);
 
       expect(await findFirstDocViewerTokens()).to.eql([
+        'Keyword',
         'Keyword',
         'Keyword',
         'Number',
@@ -182,7 +183,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         'Text',
         'Number',
         'IP address',
-        'Text',
       ]);
     });
 

test/functional/apps/discover/group3/_drag_drop.ts

@@ -48,7 +48,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '48 available fields. 5 empty fields. 3 meta fields.'
+          '48 available fields. 5 empty fields. 4 meta fields.'
         );
         expect((await PageObjects.discover.getColumnHeaders()).join(', ')).to.be(
           '@timestamp, Document'
@@ -72,7 +72,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '48 available fields. 5 empty fields. 3 meta fields.'
+          '48 available fields. 5 empty fields. 4 meta fields.'
         );
         expect((await PageObjects.discover.getColumnHeaders()).join(', ')).to.be(
           '@timestamp, Document'

test/functional/apps/discover/group6/_sidebar.ts

@@ -29,7 +29,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const dataViews = getService('dataViews');
   const retry = getService('retry');
   const dataGrid = getService('dataGrid');
-  const INITIAL_FIELD_LIST_SUMMARY = '48 available fields. 5 empty fields. 3 meta fields.';
+  const INITIAL_FIELD_LIST_SUMMARY = '48 available fields. 5 empty fields. 4 meta fields.';
 
   describe('discover sidebar', function describeIndexTests() {
     before(async function () {
@@ -73,7 +73,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await retry.waitFor('first updates', async () => {
           return (
             (await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
-            '6 available fields. 1 empty field. 2 meta fields.'
+            '6 available fields. 1 empty field. 3 meta fields.'
           );
         });
 
@@ -82,7 +82,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await retry.waitFor('second updates', async () => {
           return (
             (await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
-            '10 available fields. 3 empty fields. 3 meta fields.'
+            '10 available fields. 3 empty fields. 4 meta fields.'
           );
         });
 
@@ -173,7 +173,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await retry.waitFor('first updates', async () => {
           return (
             (await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
-            '28 available fields. 2 empty fields. 2 meta fields.'
+            '28 available fields. 2 empty fields. 3 meta fields.'
           );
         });
 
@@ -336,7 +336,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.toggleSidebarSection('meta');
         expect(
           (await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('meta')).join(', ')
-        ).to.be('_id, _index, _score');
+        ).to.be('_id, _ignored, _index, _score');
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
           INITIAL_FIELD_LIST_SUMMARY
@@ -373,7 +373,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.toggleSidebarSection('meta');
         expect(
           (await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('meta')).join(', ')
-        ).to.be('_id, _index, _score');
+        ).to.be('_id, _ignored, _index, _score');
 
         // Expand Unmapped section
         await PageObjects.unifiedFieldList.toggleSidebarSection('unmapped');
@@ -382,7 +382,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         ).to.be('relatedContent');
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '48 available fields. 1 unmapped field. 5 empty fields. 3 meta fields.'
+          '48 available fields. 1 unmapped field. 5 empty fields. 4 meta fields.'
         );
       });
 
@@ -403,7 +403,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(availableFields.includes('@message')).to.be(true);
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '2 selected fields. 2 popular fields. 48 available fields. 5 empty fields. 3 meta fields.'
+          '2 selected fields. 2 popular fields. 48 available fields. 5 empty fields. 4 meta fields.'
         );
 
         await PageObjects.unifiedFieldList.clickFieldListItemRemove('@message');
@@ -423,7 +423,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         ).to.be('@message, _id, extension');
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '3 selected fields. 3 popular fields. 48 available fields. 5 empty fields. 3 meta fields.'
+          '3 selected fields. 3 popular fields. 48 available fields. 5 empty fields. 4 meta fields.'
         );
       });
 
@@ -475,7 +475,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '48 available fields. 5 empty fields. 3 meta fields.'
+          '48 available fields. 5 empty fields. 4 meta fields.'
         );
       });
 
@@ -542,7 +542,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '6 available fields. 3 meta fields.'
+          '6 available fields. 4 meta fields.'
         );
 
         await dataViews.switchToAndValidate('with-timefield');
@@ -551,7 +551,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '0 available fields. 7 empty fields. 3 meta fields.'
+          '0 available fields. 7 empty fields. 4 meta fields.'
         );
         await testSubjects.existOrFail(
           `${PageObjects.unifiedFieldList.getSidebarSectionSelector(
@@ -621,7 +621,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '6873 available fields. 3 meta fields.'
+          '6873 available fields. 4 meta fields.'
         );
 
         await dataViews.switchToAndValidate('logstash-*');
@@ -665,7 +665,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '49 available fields. 5 empty fields. 3 meta fields.'
+          '49 available fields. 5 empty fields. 4 meta fields.'
         );
 
         let allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
@@ -684,7 +684,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '49 available fields. 5 empty fields. 3 meta fields.'
+          '49 available fields. 5 empty fields. 4 meta fields.'
         );
 
         allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
@@ -721,7 +721,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
         // check that the sidebar is rendered
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '49 available fields. 5 empty fields. 3 meta fields.'
+          '49 available fields. 5 empty fields. 4 meta fields.'
         );
         let allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
         expect(allFields.includes('_invalid-runtimefield')).to.be(true);
@@ -761,7 +761,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '0 available fields. 7 empty fields. 3 meta fields.'
+          '0 available fields. 7 empty fields. 4 meta fields.'
         );
         await testSubjects.existOrFail(
           `${PageObjects.unifiedFieldList.getSidebarSectionSelector(
@@ -778,7 +778,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
         expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
-          '7 available fields. 3 meta fields.'
+          '7 available fields. 4 meta fields.'
         );
 
         await kibanaServer.importExport.unload(

test/functional/apps/management/data_views/_data_view_create_delete.ts

@@ -253,7 +253,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.settings.editIndexPattern('logstash-*', '@timestamp', undefined, true);
         await retry.try(async () => {
           // verify updated field list
-          expect(await testSubjects.exists('field-name-agent')).to.be(true);
+          expect(await testSubjects.exists('field-name-@message')).to.be(true);
         });
       });
 

test/functional/apps/management/data_views/_exclude_index_pattern.ts

@@ -35,7 +35,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     it('data view creation with exclusion', async () => {
       const fieldCount = await PageObjects.settings.getFieldsTabCount();
       // five metafields plus keyword and text version of 'user' field
-      expect(parseInt(fieldCount, 10)).to.be(6);
+      expect(parseInt(fieldCount, 10)).to.be(7);
     });
 
     after(async () => {

test/functional/apps/management/data_views/_index_pattern_filter.ts

@@ -78,10 +78,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         '@tags.raw',
         '@timestamp',
         '_id',
+        '_ignored',
         '_index',
         '_score',
         '_source',
-        '_test',
       ];
 
       expect(await PageObjects.settings.getFieldNames()).to.eql(unfilteredFields);
@@ -107,10 +107,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         '@tags.raw',
         '@timestamp',
         '_id',
+        '_ignored',
         '_index',
         '_score',
         '_source',
-        'agent',
       ];
 
       expect(await PageObjects.settings.getFieldNames()).to.eql(unfilteredFields);
@@ -181,10 +181,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         'keyword',
         'date',
         '_id',
+        '_ignored',
         '_index',
         '',
         '_source',
-        'text',
       ]);
 
       // set other filters to check if they get reset after pressing the button

test/functional/apps/management/data_views/_index_pattern_results_sort.ts

@@ -65,7 +65,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       });
     });
     describe('field list pagination', function () {
-      const EXPECTED_FIELD_COUNT = 85;
+      const EXPECTED_FIELD_COUNT = 86;
       it('makelogs data should have expected number of fields', async function () {
         await retry.try(async function () {
           const TabCount = await PageObjects.settings.getFieldsTabCount();

test/functional/apps/management/data_views/_test_huge_fields.ts

@@ -17,7 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   describe('test large number of fields', function () {
     this.tags(['skipCloud']);
 
-    const EXPECTED_FIELD_COUNT = '10005';
+    const EXPECTED_FIELD_COUNT = '10006';
     before(async function () {
       await security.testUser.setRoles(['kibana_admin', 'test_testhuge_reader']);
       await esArchiver.emptyKibanaIndex();

test/plugin_functional/test_suites/data_plugin/index_patterns.ts

@@ -34,7 +34,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
       indexPatternId = body.id;
       expect(body.id).not.empty();
       expect(body.title).to.equal(title);
-      expect(body.fields.length).to.equal(14);
+      expect(body.fields.length).to.equal(15);
       expect(body.fieldFormatMap).to.eql(fieldFormats);
     });
 

x-pack/packages/ml/anomaly_utils/field_types.ts

@@ -52,4 +52,5 @@ export const OMIT_FIELDS: string[] = [
   '_version',
   '_score',
   '_tier',
+  '_ignored',
 ];

x-pack/plugins/data_visualizer/common/constants.ts

@@ -59,6 +59,7 @@ export const OMIT_FIELDS: string[] = [
   '_version',
   '_score',
   '_tier',
+  '_ignored',
 ];
 
 export const NON_AGGREGATABLE_FIELD_TYPES = new Set<string>([

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts

@@ -69,7 +69,7 @@ export function getPivotDropdownOptions(
   const aggOptions: DropDownOptionWithField[] = [];
   const aggOptionsData: PivotAggsConfigWithUiSupportDict = {};
 
-  const ignoreFieldNames = ['_id', '_index', '_type'];
+  const ignoreFieldNames = ['_id', '_index', '_type', '_ignored'];
   const dataViewFields = dataView.fields
     .filter(
       (field) =>

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts

@@ -49,7 +49,15 @@ function getOptions(
         .filter((ip) => ip.runtimeField === undefined)
     : [];
 
-  const ignoreFieldNames = new Set(['_source', '_type', '_index', '_id', '_version', '_score']);
+  const ignoreFieldNames = new Set([
+    '_source',
+    '_type',
+    '_index',
+    '_id',
+    '_version',
+    '_score',
+    '_ignored',
+  ]);
 
   const runtimeFieldsOptions = runtimeMappings
     ? Object.entries(runtimeMappings).map(([fieldName, fieldMapping]) => ({