[Discover] Show correct data for top level object columns

src/plugins/discover/public/__mocks__/index_pattern.ts

@@ -52,6 +52,13 @@ const fields = [
     scripted: true,
     filterable: false,
   },
+  {
+    name: 'object.value',
+    type: 'number',
+    scripted: false,
+    filterable: true,
+    aggregatable: true,
+  },
 ] as IIndexPatternFieldList;
 
 fields.getByName = (name: string) => {
@@ -64,13 +71,14 @@ const indexPattern = ({
   metaFields: ['_index', '_score'],
   formatField: jest.fn(),
   flattenHit: undefined,
-  formatHit: jest.fn((hit) => hit._source),
+  formatHit: jest.fn((hit) => (hit.fields ? hit.fields : hit._source)),
   fields,
   getComputedFields: () => ({ docvalueFields: [], scriptFields: {}, storedFields: ['*'] }),
   getSourceFiltering: () => ({}),
-  getFieldByName: () => ({}),
+  getFieldByName: jest.fn(() => ({})),
   timeFieldName: '',
   docvalueFields: [],
+  getFormatterForField: () => ({ convert: () => 'formatted' }),
 } as unknown) as IndexPattern;
 
 indexPattern.flattenHit = indexPatterns.flattenHitWrapper(indexPattern, indexPattern.metaFields);

src/plugins/discover/public/application/angular/doc_table/components/table_row.ts

@@ -16,7 +16,7 @@ import cellTemplateHtml from '../components/table_row/cell.html';
 import truncateByHeightTemplateHtml from '../components/table_row/truncate_by_height.html';
 import { getServices } from '../../../../kibana_services';
 import { getContextUrl } from '../../../helpers/get_context_url';
-import { formatRow } from '../../helpers';
+import { formatRow, formatTopLevelObject } from '../../helpers';
 
 const TAGS_WITH_WS = />\s+</g;
 
@@ -145,16 +145,32 @@ export function createTableRowDirective($compile: ng.ICompileService) {
         } else {
           $scope.columns.forEach(function (column: string) {
             const isFilterable = mapping(column) && mapping(column).filterable && $scope.filter;
-
-            newHtmls.push(
-              cellTemplate({
-                timefield: false,
-                sourcefield: column === '_source',
-                formatted: _displayField(row, column, true),
-                filterable: isFilterable,
-                column,
-              })
-            );
+            if ($scope.useNewFieldsApi && !mapping(column) && !row.fields[column]) {
+              const innerColumns = Object.fromEntries(
+                Object.entries(row.fields).filter(([key]) => {
+                  return key.indexOf(`${column}.`) === 0;
+                })
+              );
+              newHtmls.push(
+                cellTemplate({
+                  timefield: false,
+                  sourcefield: true,
+                  formatted: formatTopLevelObject(row, innerColumns, indexPattern),
+                  filterable: false,
+                  column,
+                })
+              );
+            } else {
+              newHtmls.push(
+                cellTemplate({
+                  timefield: false,
+                  sourcefield: column === '_source',
+                  formatted: _displayField(row, column, true),
+                  filterable: isFilterable,
+                  column,
+                })
+              );
+            }
           });
         }
 

src/plugins/discover/public/application/angular/helpers/index.ts

@@ -7,5 +7,5 @@
  */
 
 export { buildPointSeriesData } from './point_series';
-export { formatRow } from './row_formatter';
+export { formatRow, formatTopLevelObject } from './row_formatter';
 export { handleSourceColumnState } from './state_helpers';

src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { formatRow } from './row_formatter';
+import { formatRow, formatTopLevelObject } from './row_formatter';
 import { stubbedSavedObjectIndexPattern } from '../../../__mocks__/stubbed_saved_object_index_pattern';
 import { IndexPattern } from '../../../../../data/common/index_patterns/index_patterns';
 import { fieldFormatsMock } from '../../../../../data/common/field_formats/mocks';
@@ -43,16 +43,97 @@ describe('Row formatter', () => {
     foo: 'bar',
     hello: '<h1>World</h1>',
   };
-  const formatHitMock = jest.fn().mockReturnValueOnce(formatHitReturnValue);
+  const formatHitMock = jest.fn().mockReturnValue(formatHitReturnValue);
 
   beforeEach(() => {
-    // @ts-ignore
+    // @ts-expect-error
     indexPattern.formatHit = formatHitMock;
   });
 
   it('formats document properly', () => {
-    expect(formatRow(hit, indexPattern).trim()).toBe(
-      '<dl class="source truncate-by-height"><dt>also:</dt><dd>with \\&quot;quotes\\&quot; or &#39;single qoutes&#39;</dd> <dt>number:</dt><dd>42</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>'
+    expect(formatRow(hit, indexPattern).trim()).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>also:</dt><dd>with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;</dd> <dt>number:</dt><dd>42</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>"`
+    );
+  });
+
+  it('formats document with highlighted fields first', () => {
+    expect(
+      formatRow({ ...hit, highlight: { number: '42' } }, indexPattern).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>number:</dt><dd>42</dd> <dt>also:</dt><dd>with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects using formatter', () => {
+    indexPattern.getFieldByName = jest.fn().mockReturnValue({
+      name: 'subfield',
+    });
+    indexPattern.getFormatterForField = jest.fn().mockReturnValue({
+      convert: () => 'formatted',
+    });
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+          },
+        },
+        {
+          'object.value': [5, 10],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.value:</dt><dd>formatted, formatted</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects with subfields and highlights', () => {
+    indexPattern.getFieldByName = jest.fn().mockReturnValue({
+      name: 'subfield',
+    });
+    indexPattern.getFormatterForField = jest.fn().mockReturnValue({
+      convert: () => 'formatted',
+    });
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+            'object.keys': ['a', 'b'],
+          },
+          highlight: {
+            'object.keys': 'a',
+          },
+        },
+        {
+          'object.value': [5, 10],
+          'object.keys': ['a', 'b'],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.keys:</dt><dd>formatted, formatted</dd> <dt>object.value:</dt><dd>formatted, formatted</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects, converting unknown fields to string', () => {
+    indexPattern.getFieldByName = jest.fn();
+    indexPattern.getFormatterForField = jest.fn();
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+          },
+        },
+        {
+          'object.value': [5, 10],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.value:</dt><dd>5, 10</dd> </dl>"`
     );
   });
 });

src/plugins/discover/public/application/angular/helpers/row_formatter.ts

@@ -35,3 +35,31 @@ export const formatRow = (hit: Record<string, any>, indexPattern: IndexPattern)
   });
   return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
 };
+
+export const formatTopLevelObject = (
+  row: Record<string, any>,
+  fields: Record<string, any>,
+  indexPattern: IndexPattern
+) => {
+  const highlights = row.highlight ?? {};
+  const highlightPairs: Array<[string, unknown]> = [];
+  const sourcePairs: Array<[string, unknown]> = [];
+  Object.entries(fields).forEach(([key, values]) => {
+    const field = indexPattern.getFieldByName(key);
+    const formatter = field
+      ? indexPattern.getFormatterForField(field)
+      : { convert: (v: string, ...rest: unknown[]) => v.toString() };
+    const formatted = values
+      .map((val: unknown) =>
+        formatter.convert(val, 'html', {
+          field,
+          hit: row,
+          indexPattern,
+        })
+      )
+      .join(', ');
+    const pairs = highlights[key] ? highlightPairs : sourcePairs;
+    pairs.push([key, formatted]);
+  });
+  return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
+};

src/plugins/discover/public/application/components/discover.tsx

@@ -411,6 +411,7 @@ export function Discover({
                               onSetColumns={onSetColumns}
                               onSort={onSort}
                               onResize={onResize}
+                              useNewFieldsApi={useNewFieldsApi}
                             />
                           </div>
                         )}

src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx

@@ -119,6 +119,10 @@ export interface DiscoverGridProps {
    * Current sort setting
    */
   sort: SortPairArr[];
+  /**
+   * How the data is fetched
+   */
+  useNewFieldsApi: boolean;
 }
 
 export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => {
@@ -145,6 +149,7 @@ export const DiscoverGrid = ({
   settings,
   showTimeCol,
   sort,
+  useNewFieldsApi,
 }: DiscoverGridProps) => {
   const defaultColumns = columns.includes('_source');
 
@@ -195,9 +200,10 @@ export const DiscoverGrid = ({
       getRenderCellValueFn(
         indexPattern,
         rows,
-        rows ? rows.map((hit) => indexPattern.flattenHit(hit)) : []
+        rows ? rows.map((hit) => indexPattern.flattenHit(hit)) : [],
+        useNewFieldsApi
       ),
-    [rows, indexPattern]
+    [rows, indexPattern, useNewFieldsApi]
   );
 
   /**