[Lens] Support index pattern runtime fields in existence and field stats API

x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx

@@ -116,27 +116,6 @@ describe('IndexPattern Field Item', () => {
     );
   });
 
-  it('should request field stats without a time field, if the index pattern has none', async () => {
-    indexPattern.timeFieldName = undefined;
-    core.http.post.mockImplementationOnce(() => {
-      return Promise.resolve({});
-    });
-    const wrapper = mountWithIntl(<InnerFieldItem {...defaultProps} />);
-
-    await act(async () => {
-      clickField(wrapper, 'bytes');
-    });
-
-    expect(core.http.post).toHaveBeenCalledWith(
-      '/api/lens/index_stats/my-fake-index-pattern/field',
-      expect.anything()
-    );
-    // Function argument types not detected correctly (https://github.com/microsoft/TypeScript/issues/26591)
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const { body } = (core.http.post.mock.calls[0] as any)[1];
-    expect(JSON.parse(body)).not.toHaveProperty('timeFieldName');
-  });
-
   it('should request field stats every time the button is clicked', async () => {
     let resolveFunction: (arg: unknown) => void;
 
@@ -150,31 +129,21 @@ describe('IndexPattern Field Item', () => {
 
     clickField(wrapper, 'bytes');
 
-    expect(core.http.post).toHaveBeenCalledWith(
-      `/api/lens/index_stats/my-fake-index-pattern/field`,
-      {
-        body: JSON.stringify({
-          dslQuery: {
-            bool: {
-              must: [{ match_all: {} }],
-              filter: [],
-              should: [],
-              must_not: [],
-            },
+    expect(core.http.post).toHaveBeenCalledWith(`/api/lens/index_stats/1/field`, {
+      body: JSON.stringify({
+        dslQuery: {
+          bool: {
+            must: [{ match_all: {} }],
+            filter: [],
+            should: [],
+            must_not: [],
           },
-          fromDate: 'now-7d',
-          toDate: 'now',
-          timeFieldName: 'timestamp',
-          field: {
-            name: 'bytes',
-            displayName: 'bytesLabel',
-            type: 'number',
-            aggregatable: true,
-            searchable: true,
-          },
-        }),
-      }
-    );
+        },
+        fromDate: 'now-7d',
+        toDate: 'now',
+        fieldName: 'bytes',
+      }),
+    });
 
     expect(wrapper.find(EuiPopover).prop('isOpen')).toEqual(true);
 
@@ -227,40 +196,30 @@ describe('IndexPattern Field Item', () => {
     clickField(wrapper, 'bytes');
 
     expect(core.http.post).toHaveBeenCalledTimes(2);
-    expect(core.http.post).toHaveBeenLastCalledWith(
-      `/api/lens/index_stats/my-fake-index-pattern/field`,
-      {
-        body: JSON.stringify({
-          dslQuery: {
-            bool: {
-              must: [],
-              filter: [
-                {
-                  bool: {
-                    should: [{ match_phrase: { 'geo.src': 'US' } }],
-                    minimum_should_match: 1,
-                  },
-                },
-                {
-                  match: { phrase: { 'geo.dest': 'US' } },
+    expect(core.http.post).toHaveBeenLastCalledWith(`/api/lens/index_stats/1/field`, {
+      body: JSON.stringify({
+        dslQuery: {
+          bool: {
+            must: [],
+            filter: [
+              {
+                bool: {
+                  should: [{ match_phrase: { 'geo.src': 'US' } }],
+                  minimum_should_match: 1,
                 },
-              ],
-              should: [],
-              must_not: [],
-            },
+              },
+              {
+                match: { phrase: { 'geo.dest': 'US' } },
+              },
+            ],
+            should: [],
+            must_not: [],
           },
-          fromDate: 'now-14d',
-          toDate: 'now-7d',
-          timeFieldName: 'timestamp',
-          field: {
-            name: 'bytes',
-            displayName: 'bytesLabel',
-            type: 'number',
-            aggregatable: true,
-            searchable: true,
-          },
-        }),
-      }
-    );
+        },
+        fromDate: 'now-14d',
+        toDate: 'now-7d',
+        fieldName: 'bytes',
+      }),
+    });
   });
 });

x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx

@@ -129,7 +129,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
     setState((s) => ({ ...s, isLoading: true }));
 
     core.http
-      .post(`/api/lens/index_stats/${indexPattern.title}/field`, {
+      .post(`/api/lens/index_stats/${indexPattern.id}/field`, {
         body: JSON.stringify({
           dslQuery: esQuery.buildEsQuery(
             indexPattern as IIndexPattern,
@@ -139,8 +139,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
           ),
           fromDate: dateRange.fromDate,
           toDate: dateRange.toDate,
-          timeFieldName: indexPattern.timeFieldName,
-          field,
+          fieldName: field.name,
         }),
       })
       .then((results: FieldStatsResponse<string | number>) => {

x-pack/plugins/lens/server/routes/existing_fields.test.ts

@@ -61,6 +61,20 @@ describe('existingFields', () => {
     expect(result).toEqual(['bar']);
   });
 
+  it('supports runtime fields', () => {
+    const result = existingFields(
+      [searchResults({ runtime_foo: ['scriptvalue'] })],
+      [
+        field({
+          name: 'runtime_foo',
+          runtimeField: { type: 'long', script: { source: '2+2' } },
+        }),
+      ]
+    );
+
+    expect(result).toEqual(['runtime_foo']);
+  });
+
   it('supports meta fields', () => {
     const result = existingFields(
       [{ _mymeta: 'abc', ...searchResults({ bar: ['scriptvalue'] }) }],
@@ -78,6 +92,11 @@ describe('buildFieldList', () => {
     typeMeta: 'typemeta',
     fields: [
       { name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
+      {
+        name: 'runtime_foo',
+        isMapped: false,
+        runtimeField: { type: 'long', script: { source: '2+2' } },
+      },
       { name: 'bar' },
       { name: '@bar' },
       { name: 'baz' },
@@ -95,6 +114,14 @@ describe('buildFieldList', () => {
     });
   });
 
+  it('supports runtime fields', () => {
+    const fields = buildFieldList((indexPattern as unknown) as IndexPattern, []);
+    expect(fields.find((f) => f.runtimeField)).toMatchObject({
+      name: 'runtime_foo',
+      runtimeField: { type: 'long', script: { source: '2+2' } },
+    });
+  });
+
   it('supports meta fields', () => {
     const fields = buildFieldList((indexPattern as unknown) as IndexPattern, ['_mymeta']);
     expect(fields.find((f) => f.isMeta)).toMatchObject({

x-pack/plugins/lens/server/routes/existing_fields.ts

@@ -10,7 +10,7 @@ import { errors } from '@elastic/elasticsearch';
 import { schema } from '@kbn/config-schema';
 import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server';
 import { CoreSetup, Logger } from 'src/core/server';
-import { IndexPattern, IndexPatternsService } from 'src/plugins/data/common';
+import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common';
 import { BASE_API_URL } from '../../common';
 import { UI_SETTINGS } from '../../../../../src/plugins/data/server';
 import { PluginStartContract } from '../plugin';
@@ -30,6 +30,7 @@ export interface Field {
   isMeta: boolean;
   lang?: string;
   script?: string;
+  runtimeField?: RuntimeField;
 }
 
 export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>, logger: Logger) {
@@ -138,6 +139,7 @@ export function buildFieldList(indexPattern: IndexPattern, metaFields: string[])
       // id is a special case - it doesn't show up in the meta field list,
       // but as it's not part of source, it has to be handled separately.
       isMeta: metaFields.includes(field.name) || field.name === '_id',
+      runtimeField: !field.isMapped ? field.runtimeField : undefined,
     };
   });
 }
@@ -181,6 +183,7 @@ async function fetchIndexPatternStats({
   };
 
   const scriptedFields = fields.filter((f) => f.isScript);
+  const runtimeFields = fields.filter((f) => f.runtimeField);
   const { body: result } = await client.search({
     index,
     body: {
@@ -189,6 +192,11 @@ async function fetchIndexPatternStats({
       sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [],
       fields: ['*'],
       _source: false,
+      runtime_mappings: runtimeFields.reduce((acc, field) => {
+        if (!field.runtimeField) return acc;
+        acc[field.name] = field.runtimeField;
+        return acc;
+      }, {} as Record<string, unknown>),
       script_fields: scriptedFields.reduce((acc, field) => {
         acc[field.name] = {
           script: {

x-pack/plugins/lens/server/routes/field_stats.ts

@@ -11,6 +11,7 @@ import DateMath from '@elastic/datemath';
 import { schema } from '@kbn/config-schema';
 import { CoreSetup } from 'src/core/server';
 import { IFieldType } from 'src/plugins/data/common';
+import { SavedObjectNotFound } from '../../../../../src/plugins/kibana_utils/common';
 import { ESSearchResponse } from '../../../../typings/elasticsearch';
 import { FieldStatsResponse, BASE_API_URL } from '../../common';
 import { PluginStartContract } from '../plugin';
@@ -21,38 +22,44 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
   const router = setup.http.createRouter();
   router.post(
     {
-      path: `${BASE_API_URL}/index_stats/{indexPatternTitle}/field`,
+      path: `${BASE_API_URL}/index_stats/{indexPatternId}/field`,
       validate: {
         params: schema.object({
-          indexPatternTitle: schema.string(),
+          indexPatternId: schema.string(),
         }),
         body: schema.object(
           {
             dslQuery: schema.object({}, { unknowns: 'allow' }),
             fromDate: schema.string(),
             toDate: schema.string(),
-            timeFieldName: schema.maybe(schema.string()),
-            field: schema.object(
-              {
-                name: schema.string(),
-                type: schema.string(),
-                esTypes: schema.maybe(schema.arrayOf(schema.string())),
-                scripted: schema.maybe(schema.boolean()),
-                lang: schema.maybe(schema.string()),
-                script: schema.maybe(schema.string()),
-              },
-              { unknowns: 'allow' }
-            ),
+            fieldName: schema.string(),
           },
           { unknowns: 'allow' }
         ),
       },
     },
     async (context, req, res) => {
       const requestClient = context.core.elasticsearch.client.asCurrentUser;
-      const { fromDate, toDate, timeFieldName, field, dslQuery } = req.body;
+      const { fromDate, toDate, fieldName, dslQuery } = req.body;
+
+      const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices();
+      const savedObjectsClient = savedObjects.getScopedClient(req);
+      const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
+      const indexPatternsService = await data.indexPatterns.indexPatternsServiceFactory(
+        savedObjectsClient,
+        esClient
+      );
 
       try {
+        const indexPattern = await indexPatternsService.get(req.params.indexPatternId);
+
+        const timeFieldName = indexPattern.timeFieldName;
+        const field = indexPattern.fields.find((f) => f.name === fieldName);
+
+        if (!field) {
+          throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`);
+        }
+
         const filter = timeFieldName
           ? [
               {
@@ -75,11 +82,13 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
 
         const search = async (aggs: unknown) => {
           const { body: result } = await requestClient.search({
-            index: req.params.indexPatternTitle,
+            index: indexPattern.title,
             track_total_hits: true,
             body: {
               query,
               aggs,
+              runtime_mappings:
+                !field.isMapped && field.runtimeField ? { [fieldName]: field.runtimeField } : {},
             },
             size: 0,
           });
@@ -100,6 +109,9 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
           body: await getStringSamples(search, field),
         });
       } catch (e) {
+        if (e instanceof SavedObjectNotFound) {
+          return res.notFound();
+        }
         if (e instanceof errors.ResponseError && e.statusCode === 404) {
           return res.notFound();
         }